Compare commits

..
70 changed files with 443 additions and 6866 deletions
-7
View File
@@ -5,13 +5,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/).
Starting from 0.2.0, CLI / Extension / DSH Plugin share the same version number.
## [Unreleased]
### Added
- [Scroll-to element primitive](docs/scroll-to.md) across CLI, Extension and DSH Plugin,
with ancestor-clipped visible bounds, iframe support and cooperative cancellation
## [0.2.1] - 2026-09-09
### Changed
-3
View File
@@ -207,9 +207,6 @@ through the [plugin](#deepseek-harness-plugin): the agent calls injected
## For Developers
The [scroll-to primitive reference](docs/scroll-to.md) covers its CLI, protocol
and plugin entry points, visible bounds and interruption behavior.
The repository is a Cargo + pnpm workspace:
- `crates/bsk-cli` — `bsk` CLI and local daemon
-2
View File
@@ -177,8 +177,6 @@ Agent 不直接与浏览器通信。它通过 `bsk` CLI 下发浏览器任务;
## 面向开发者
[scroll-to 原语说明](docs/scroll-to.md)介绍 CLI、协议和插件入口,以及可见区域、错误和中断语义。
本仓库是 Cargo + pnpm workspace:
- `crates/bsk-cli` — `bsk` CLI 与本地 daemon
+22 -42
View File
@@ -1,11 +1,19 @@
import type { VisualCandidate } from "@/tools/vom/visual-discovery";
/** Session-local refs describe the latest observation. Reusing eN does not identify
* which observation a caller read; generation is internal bookkeeping only. */
/**
* Per-session map from `@e<N>` snapshot refs to a CDP node address.
*
* Each fresh `tool.snapshot` resets the store: M6 will call
* `replace(...)` with the new ref → node address pairs. A node address
* includes the owning flat CDP session/frame when the element lives in
* an OOPIF; tools resolve live geometry from that identity at call time.
*
* Refs are session-scoped (§7): looking up a ref in the wrong session
* returns `null`, never silently leaks. Storing values in different
* sessions is fine; they live in independent `RefStore` instances
* inside the `SessionContext`.
*/
export type BackendNodeId = number;
export interface DomRefEntry {
readonly kind: "dom";
export interface RefEntry {
backendNodeId: BackendNodeId;
tabId: number | null;
frameId?: string;
@@ -13,15 +21,7 @@ export interface DomRefEntry {
generation: number;
}
export interface VisualRefInput {
readonly kind: "visual-region";
readonly candidate: VisualCandidate;
}
export type RefEntry = DomRefEntry | (VisualRefInput & { readonly generation: number });
export type RefInput =
| VisualRefInput
| BackendNodeId
| {
backendNodeId: BackendNodeId;
@@ -31,7 +31,7 @@ export type RefInput =
};
export class RefStore {
private map = new Map<string, RefEntry>();
private readonly map = new Map<string, RefEntry>();
private generation = 0;
size(): number {
@@ -44,7 +44,7 @@ export class RefStore {
resolve(ref: string, opts: { tabId?: number } = {}): BackendNodeId | null {
const entry = this.map.get(normaliseRef(ref));
if (!entry || entry.kind !== "dom") return null;
if (!entry) return null;
if (opts.tabId !== undefined && entry.tabId !== opts.tabId) return null;
return entry.backendNodeId;
}
@@ -58,11 +58,9 @@ export class RefStore {
* Used after every fresh `tool.snapshot`.
*/
replace(entries: Iterable<readonly [string, RefInput]>): void {
const generation = this.generation + 1;
const next = new Map<string, RefEntry>();
for (const [ref, input] of entries) next.set(normaliseRef(ref), this.entry(input, generation));
this.map = next;
this.generation = generation;
this.map.clear();
this.generation += 1;
for (const [ref, input] of entries) this.map.set(normaliseRef(ref), this.entry(input));
}
set(
@@ -75,7 +73,6 @@ export class RefStore {
} = {},
): void {
this.map.set(normaliseRef(ref), {
kind: "dom",
backendNodeId: id,
tabId: opts.tabId ?? null,
...(opts.frameId ? { frameId: opts.frameId } : {}),
@@ -92,37 +89,20 @@ export class RefStore {
return this.map.entries();
}
private entry(input: RefInput, generation: number): RefEntry {
if (typeof input !== "number" && "kind" in input) {
const { document } = input.candidate;
if (
!document?.attachmentId ||
!document.frameId ||
!Number.isSafeInteger(document.target?.tabId) ||
!Number.isSafeInteger(document.documentElementBackendNodeId) ||
document.documentElementBackendNodeId <= 0 ||
!Number.isSafeInteger(input.candidate.backendNodeId) ||
input.candidate.backendNodeId <= 0
)
throw new TypeError("visual ref requires a verified DOM identity and anchor");
// Preserve the read-only evidence and shared clipping chain; do not clone ancestors per ref.
return { kind: "visual-region", candidate: input.candidate, generation };
}
private entry(input: RefInput): RefEntry {
if (typeof input === "number") {
return {
kind: "dom",
backendNodeId: input,
tabId: null,
generation,
generation: this.generation,
};
}
return {
kind: "dom",
backendNodeId: input.backendNodeId,
tabId: input.tabId,
...(input.frameId ? { frameId: input.frameId } : {}),
...(input.cdpSessionId ? { cdpSessionId: input.cdpSessionId } : {}),
generation,
generation: this.generation,
};
}
}
@@ -599,79 +599,6 @@ describe("ToolDispatcher", () => {
expect(onSessionsChanged).toHaveBeenCalledTimes(2);
});
it.each([
"focus",
"blur",
"scroll_to",
] as const)("routes %s with hover cleanup and cooperative cancellation", async (action) => {
const tab = { id: 7, windowId: 4242, active: true };
vi.stubGlobal("chrome", {
tabs: {
get: vi.fn(async () => tab),
query: vi.fn(async () => [tab]),
sendMessage: vi.fn(async () => undefined),
},
});
const sessions = new SessionManager({
agentWindow: {
create: async () => 4242,
remove: async () => {},
ensureActiveTab: async () => 7,
},
});
const ctx = await sessions.start("aa11");
ctx.refStore.set("e1", 12, { tabId: 7 });
const { transport, sent, deliver } = fakeTransport();
const onBrowserControlResumed = vi.fn();
let resolveNode: ((value: object) => void) | undefined;
const cdp = {
send: vi.fn(async (_tabId: number, method: string) => {
if (method === "DOM.resolveNode")
return new Promise((resolve) => {
resolveNode = resolve;
});
if (method === "Runtime.callFunctionOn") return { result: { value: true } };
return {};
}),
} as unknown as TestDispatcherCdp;
const dispatcher = new ToolDispatcher({ transport, sessions, cdp, onBrowserControlResumed });
const hover = dispatcher as unknown as {
rememberHover: (sessionId: string, result: object) => void;
setHoverBypass: (sessionId: string, tabId: number, enabled: boolean) => Promise<void>;
};
hover.rememberHover("aa11", { tab_id: 7, x: 10, y: 20 });
await hover.setHoverBypass("aa11", 7, true);
dispatcher.start();
deliver(makeRequest(`tool.${action}`, { session_id: "aa11", ref: "e1" }));
await vi.waitFor(() => expect(resolveNode).toBeDefined());
expect(onBrowserControlResumed).toHaveBeenCalledWith("aa11");
deliver({ id: "cancel-focus", method: "cancel", params: { rpc_id: "r-1" } });
resolveNode!({ object: { objectId: "focus-target" } });
await vi.waitFor(() => expect(sent).toHaveLength(2));
expect(sent).toContainEqual({ id: "cancel-focus", result: { cancelled: true } });
expect(sent).toContainEqual({
id: "r-1",
error: expect.objectContaining({ code: "cancelled" }),
});
expect(cdp.send).not.toHaveBeenCalledWith(7, "DOM.focus", expect.anything());
expect(cdp.send).not.toHaveBeenCalledWith(7, "Runtime.callFunctionOn", expect.anything());
if (action === "scroll_to") {
expect(cdp.send).toHaveBeenCalledWith(7, "Runtime.releaseObjectGroup", {
objectGroup: expect.any(String),
});
} else {
expect(cdp.send).toHaveBeenCalledWith(7, "Runtime.releaseObject", {
objectId: "focus-target",
});
}
expect(chrome.tabs.sendMessage).toHaveBeenCalledWith(
7,
expect.objectContaining({ enabled: false }),
);
expect(dispatcher.inflightAbortControllers.size).toBe(0);
dispatcher.stop();
});
it("invokes onBrowserControlResumed for browser-control tools but not passive reads", async () => {
vi.stubGlobal("chrome", {
tabs: {
@@ -1,166 +0,0 @@
// @vitest-environment node
// Opt in with BSK_FOCUS_CHROME=/path/to/chrome; each test owns its browser/profile.
import { describe, expect, it } from "vitest";
import { SessionManager } from "@/session-manager/manager";
import { handleBlur, handleFocus } from "../interaction";
import type { CdpRunner } from "../shared";
type Send = <T = Record<string, unknown>>(
method: string,
params?: object,
sessionId?: string,
) => Promise<T>;
async function withFocusBrowser(
run: (h: {
evaluate: (expression: string) => Promise<unknown>;
ref: () => Promise<void>;
focus: (signal?: AbortSignal) => ReturnType<typeof handleFocus>;
blur: (signal?: AbortSignal) => ReturnType<typeof handleBlur>;
calls: string[];
afterCommand: { current: (method: string) => void };
}) => Promise<void>,
) {
// Reuse the existing isolated Chrome launcher; no new browser dependency.
const { withChrome } = await import(
new URL(
"../../../../../evals/browser/cases/regression/snapshot-coordinates/chrome.mjs",
import.meta.url,
).href
);
await withChrome(
{ executable: process.env.BSK_FOCUS_CHROME, deviceScale: 1, zoom: 1 },
async (send: Send) => {
const { targetId } = await send<{ targetId: string }>("Target.createTarget", {
url: "about:blank",
});
const { sessionId } = await send<{ sessionId: string }>("Target.attachToTarget", {
targetId,
flatten: true,
});
await send("Page.bringToFront", {}, sessionId);
const evaluate = async (expression: string) => {
const reply = await send<{ result: { value: unknown }; exceptionDetails?: unknown }>(
"Runtime.evaluate",
{ expression, returnByValue: true },
sessionId,
);
expect(reply.exceptionDetails).toBeUndefined();
return reply.result.value;
};
await evaluate(
`document.body.innerHTML = '<input id="target"><input id="other">'; window.probe = document.querySelector('#target');`,
);
const manager = new SessionManager({
agentWindow: {
create: async () => 100,
remove: async () => {},
ensureActiveTab: async () => 4,
},
});
const ctx = await manager.start("aa11");
const calls: string[] = [];
const afterCommand = { current: (_method: string) => {} };
const cdp: CdpRunner = {
send: async (_tabId, method, params) => {
calls.push(method);
const result = await send(method, params, sessionId);
afterCommand.current(method);
return result as never;
},
};
const deps = {
cdp,
tabsApi: {
get: async (id: number) => ({ id, windowId: 100, active: true }) as chrome.tabs.Tab,
query: async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab],
},
};
await run({
evaluate,
calls,
afterCommand,
ref: async () => {
const { result } = await send<{ result: { objectId: string } }>(
"Runtime.evaluate",
{ expression: "window.probe" },
sessionId,
);
const { node } = await send<{ node: { backendNodeId: number } }>(
"DOM.describeNode",
{ objectId: result.objectId },
sessionId,
);
await send("Runtime.releaseObject", { objectId: result.objectId }, sessionId);
ctx.refStore.set("e1", node.backendNodeId, { tabId: 4 });
},
focus: (signal) =>
handleFocus(manager, { session_id: "aa11", ref: "e1" }, { ...deps, signal }),
blur: (signal) =>
handleBlur(manager, { session_id: "aa11", ref: "e1" }, { ...deps, signal }),
});
},
);
}
describe.skipIf(!process.env.BSK_FOCUS_CHROME)("real browser focus and blur", () => {
it.each(["light", "open", "closed"])("verifies focus and blur in %s DOM", async (mode) => {
await withFocusBrowser(async (h) => {
if (mode !== "light")
await h.evaluate(`(() => {
const host = document.createElement('div'); document.body.append(host);
const nested = document.createElement('div'); host.attachShadow({mode:'${mode}'}).append(nested);
const input = document.createElement('input'); nested.attachShadow({mode:'${mode}'}).append(input);
window.probe = input;
})()`);
await h.ref();
expect(await h.focus()).toMatchObject({ focused: true });
expect(await h.evaluate("window.probe.matches(':focus')")).toBe(true);
expect(await h.blur()).toMatchObject({ was_focused: true, focused: false });
expect(await h.evaluate("window.probe.matches(':focus')")).toBe(false);
expect(h.calls.filter((method) => method === "Runtime.releaseObject")).toHaveLength(2);
});
});
it("detects focus changes from event-handler microtasks", async () => {
await withFocusBrowser(async (h) => {
await h.ref();
await h.evaluate(
"window.probe.addEventListener('focus', () => queueMicrotask(() => document.querySelector('#other').focus()), {once:true})",
);
expect(await h.focus()).toMatchObject({ code: "cdp_failed" });
expect(await h.focus()).toMatchObject({ focused: true });
await h.evaluate(
"window.probe.addEventListener('blur', () => queueMicrotask(() => window.probe.focus()), {once:true})",
);
expect(await h.blur()).toMatchObject({ code: "cdp_failed" });
expect(await h.evaluate("window.probe.matches(':focus')")).toBe(true);
});
});
it("cancels before blur while still releasing the resolved object", async () => {
await withFocusBrowser(async (h) => {
await h.ref();
await h.focus();
const abort = new AbortController();
h.afterCommand.current = (method) => {
if (method === "DOM.resolveNode") abort.abort();
};
expect(await h.blur(abort.signal)).toMatchObject({ code: "cancelled" });
expect(await h.evaluate("window.probe.matches(':focus')")).toBe(true);
expect(h.calls.at(-1)).toBe("Runtime.releaseObject");
});
});
it("reports a page exception with its cause", async () => {
await withFocusBrowser(async (h) => {
await h.ref();
await h.evaluate("window.probe.blur = () => { throw new Error('page blur failed'); }");
expect(await h.blur()).toMatchObject({
code: "cdp_failed",
message: expect.stringContaining("page blur failed"),
});
expect(h.calls.at(-1)).toBe("Runtime.releaseObject");
});
});
});
@@ -1,293 +0,0 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import type { CdpTarget } from "@/browser-driver/frame-graph";
import { SessionManager } from "@/session-manager/manager";
import { handleBlur, handleFocus } from "../interaction";
import type { CdpRunner } from "../shared";
interface ScriptParams {
functionDeclaration: string;
}
interface Call {
target: CdpTarget;
method: string;
}
async function setup(element?: HTMLElement, childSession = false) {
if (!element) {
document.body.innerHTML = '<input id="target"><input id="other">';
element = document.querySelector<HTMLInputElement>("#target")!;
}
const targetElement = element;
const manager = new SessionManager({
agentWindow: {
create: async () => 100,
remove: async () => {},
ensureActiveTab: async () => 4,
},
});
const ctx = await manager.start("aa11");
ctx.refStore.set("e1", 12, {
tabId: 4,
...(childSession ? { frameId: "child", cdpSessionId: "child-session" } : {}),
});
const calls: Call[] = [];
const afterCommand = vi.fn<(call: Call) => void>();
const script = vi.fn(async (params: ScriptParams) => {
try {
const fn = new Function(`return (${params.functionDeclaration})`)();
return { result: { value: await fn.call(targetElement) } };
} catch (error) {
return { exceptionDetails: { text: "Uncaught", exception: { description: String(error) } } };
}
});
const focus = vi.fn(() => targetElement.focus());
const release = vi.fn(async () => ({}));
const send = async (target: CdpTarget, method: string, params?: object) => {
const call = { target, method };
calls.push(call);
try {
switch (method) {
case "DOM.getDocument":
return { root: { nodeId: 1 } };
case "DOM.querySelector":
return { nodeId: 2 };
case "DOM.describeNode":
return { node: { backendNodeId: 12 } };
case "DOM.scrollIntoViewIfNeeded":
return {};
case "DOM.resolveNode":
return { object: { objectId: "focus-target" } };
case "DOM.focus":
focus();
return {};
case "Runtime.callFunctionOn":
return await script(params as ScriptParams);
case "Runtime.releaseObject":
return await release();
default:
throw new Error(`unexpected ${method}`);
}
} finally {
afterCommand(call);
}
};
const cdp: CdpRunner = {
send: (tabId, method, params) => send({ tabId }, method, params) as never,
sendToTarget: send as CdpRunner["sendToTarget"],
getFrameGraph: vi.fn(async () => ({
rootFrameId: "main",
frames: [
{ frameId: "main", target: { tabId: 4 } },
...(childSession
? [
{
frameId: "child",
parentFrameId: "main",
ownerBackendNodeId: 99,
target: { tabId: 4, sessionId: "child-session" },
},
]
: []),
],
})),
};
const tabsApi = {
get: vi.fn(async (id: number) => ({ id, windowId: 100, active: true }) as chrome.tabs.Tab),
query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
};
return {
element: targetElement,
calls,
afterCommand,
script,
focus,
release,
cdp,
tabsApi,
run: (
action: "focus" | "blur",
signal?: AbortSignal,
params: { ref?: string; selector?: string; tab_id?: number } = { ref: "e1" },
) =>
(action === "focus" ? handleFocus : handleBlur)(
manager,
{ session_id: "aa11", ...params },
{ cdp, tabsApi, signal },
),
};
}
afterEach(() => {
vi.restoreAllMocks();
document.body.innerHTML = "";
});
describe("focus and blur state verification", () => {
it.each(["open", "closed"] as const)("handles nested %s shadow roots", async (mode) => {
const host = document.createElement("div");
document.body.append(host);
const outer = host.attachShadow({ mode });
const innerHost = document.createElement("div");
outer.append(innerHost);
const inner = innerHost.attachShadow({ mode });
const element = document.createElement("input");
inner.append(element);
const h = await setup(element);
expect(await h.run("focus")).toMatchObject({ focused: true });
expect(inner.activeElement).toBe(element);
expect(await h.run("blur")).toMatchObject({ was_focused: true, focused: false });
expect(inner.activeElement).toBeNull();
expect(h.release).toHaveBeenCalledTimes(2);
});
it("keeps DOM focus usable when the document is in the background", async () => {
const h = await setup();
vi.spyOn(document, "hasFocus").mockReturnValue(false);
expect(await h.run("focus")).toMatchObject({ focused: true });
expect(await h.run("blur")).toMatchObject({ was_focused: true, focused: false });
});
it("blurring an unfocused target preserves the other element's focus", async () => {
const h = await setup();
const other = document.querySelector<HTMLInputElement>("#other")!;
other.focus();
expect(await h.run("blur")).toMatchObject({ was_focused: false, focused: false });
expect(document.activeElement).toBe(other);
});
it("detects focus redirected by a focus handler's microtask", async () => {
const h = await setup();
h.element.addEventListener("focus", () =>
queueMicrotask(() => document.querySelector<HTMLInputElement>("#other")!.focus()),
);
expect(await h.run("focus")).toMatchObject({
code: "cdp_failed",
message: "target element did not become focused",
});
expect(h.release).toHaveBeenCalledOnce();
});
it("does not report blur success when a microtask restores focus", async () => {
const h = await setup();
h.element.focus();
h.element.addEventListener("blur", () => queueMicrotask(() => h.element.focus()));
expect(await h.run("blur")).toMatchObject({
code: "cdp_failed",
message: "target element remained focused after blur()",
});
expect(document.activeElement).toBe(h.element);
expect(h.release).toHaveBeenCalledOnce();
});
it.each(["focus", "blur"] as const)("rejects a detached %s target", async (action) => {
const h = await setup();
h.element.remove();
expect(await h.run(action)).toMatchObject({ code: "cdp_failed" });
expect(h.release).toHaveBeenCalledOnce();
});
it("preserves a script exception's cause", async () => {
const h = await setup();
vi.spyOn(h.element, "blur").mockImplementation(() => {
throw new Error("page blur failed");
});
expect(await h.run("blur")).toMatchObject({
code: "cdp_failed",
message: expect.stringContaining("page blur failed"),
});
expect(h.release).toHaveBeenCalledOnce();
});
it.each(["focus", "blur"] as const)("rejects malformed %s script results", async (action) => {
const h = await setup();
h.script.mockResolvedValueOnce({ result: { value: null } });
expect(await h.run(action)).toMatchObject({
code: "cdp_failed",
message: expect.stringContaining("unexpected result"),
});
expect(h.release).toHaveBeenCalledOnce();
});
it.each([
"focus",
"blur",
] as const)("keeps %s, verification and cleanup in the OOPIF session", async (action) => {
const h = await setup(undefined, true);
if (action === "blur") h.element.focus();
expect(await h.run(action)).toMatchObject({ focused: action === "focus" });
const targetCalls = h.calls.filter((call) => call.method !== "DOM.scrollIntoViewIfNeeded");
expect(targetCalls.every((call) => call.target.sessionId === "child-session")).toBe(true);
expect(targetCalls.at(-1)?.method).toBe("Runtime.releaseObject");
});
it.each(["focus", "blur"] as const)("rejects %s on a user tab before CDP", async (action) => {
const h = await setup();
h.tabsApi.get.mockResolvedValue({ id: 9, windowId: 200, active: true } as chrome.tabs.Tab);
expect(await h.run(action, undefined, { ref: "e1", tab_id: 9 })).toMatchObject({
code: "permission_denied",
});
expect(h.calls).toEqual([]);
});
it("does not replace a successful result when navigation disposed the remote object", async () => {
const h = await setup();
h.release.mockRejectedValue(new Error("Cannot find context"));
expect(await h.run("focus")).toMatchObject({ focused: true });
});
});
describe("focus and blur cancellation", () => {
const boundaries = [
"DOM.getDocument",
"DOM.querySelector",
"DOM.describeNode",
"DOM.resolveNode",
"Runtime.callFunctionOn",
];
for (const action of ["focus", "blur"] as const) {
it.each(
action === "focus" ? [...boundaries, "DOM.scrollIntoViewIfNeeded", "DOM.focus"] : boundaries,
)(`${action} stops after cancellation during %s`, async (method) => {
const h = await setup();
const abort = new AbortController();
let cancelledAfter = 0;
h.afterCommand.mockImplementation((call) => {
if (call.method === method && !abort.signal.aborted) {
cancelledAfter = h.calls.length;
abort.abort();
}
});
expect(await h.run(action, abort.signal, { selector: "#target" })).toMatchObject({
code: "cancelled",
});
expect(abort.signal.aborted).toBe(true);
expect(
h.calls.slice(cancelledAfter).every((call) => call.method === "Runtime.releaseObject"),
).toBe(true);
if (h.calls.some((call) => call.method === "DOM.resolveNode"))
expect(h.release).toHaveBeenCalledOnce();
});
it(`${action} stops if cancellation arrives while resolving the tab`, async () => {
const h = await setup();
const abort = new AbortController();
h.tabsApi.query.mockImplementation(async () => {
abort.abort();
return [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab];
});
expect(await h.run(action, abort.signal)).toMatchObject({ code: "cancelled" });
expect(h.calls).toEqual([]);
});
}
it("does not scroll frame owners after cancellation in an OOPIF", async () => {
const h = await setup(undefined, true);
const abort = new AbortController();
h.afterCommand.mockImplementation((call) => {
if (call.method === "DOM.scrollIntoViewIfNeeded") abort.abort();
});
expect(await h.run("focus", abort.signal)).toMatchObject({ code: "cancelled" });
expect(h.calls.filter((call) => call.method === "DOM.scrollIntoViewIfNeeded")).toHaveLength(1);
expect(h.focus).not.toHaveBeenCalled();
});
});
@@ -405,7 +405,6 @@ describe("handleRequestHelp", () => {
agentWindowId: 99,
refStore: {
resolveEntry: () => ({
kind: "dom",
backendNodeId: 42,
tabId: 5,
frameId: "child",
@@ -2,10 +2,8 @@ import { describe, expect, it, vi } from "vitest";
import { SessionManager } from "@/session-manager/manager";
import type { CdpRunner } from "@/tools/shared";
import {
handleBlur,
handleClick,
handleFill,
handleFocus,
handleHover,
handlePress,
handleSelect,
@@ -544,161 +542,6 @@ describe("handleHover", () => {
});
});
describe("handleFocus and handleBlur", () => {
it("focuses a ref and verifies the deep active element", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
ctx.refStore.set("e3", 1234, { tabId: 4 });
const fake = makeFakeCdp({
"DOM.scrollIntoViewIfNeeded": () => ({}),
"DOM.focus": () => ({}),
"DOM.resolveNode": () => ({ object: { objectId: "focus-target" } }),
"Runtime.callFunctionOn": () => ({ result: { value: true } }),
});
const res = await handleFocus(
sm,
{ session_id: "aa11", ref: "@e3" },
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
);
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
expect(res).toMatchObject({ tab_id: 4, used_ref: "e3", focused: true });
expect(fake.sent.map((call) => call.method)).toEqual([
"DOM.scrollIntoViewIfNeeded",
"DOM.resolveNode",
"DOM.focus",
"Runtime.callFunctionOn",
"Runtime.releaseObject",
]);
});
it("focuses an OOPIF ref in its CDP session", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
ctx.refStore.set("e3", 1234, {
tabId: 4,
frameId: "child-frame",
cdpSessionId: "child-session",
});
const fake = makeFakeCdp({
"DOM.scrollIntoViewIfNeeded": () => ({}),
});
fake.cdp.getFrameGraph = vi.fn(async () => ({
rootFrameId: "main",
frames: [
{ frameId: "main", target: { tabId: 4 } },
{
frameId: "child-frame",
parentFrameId: "main",
ownerBackendNodeId: 99,
target: { tabId: 4, sessionId: "child-session" },
},
],
}));
const targetCalls: Array<{ sessionId?: string; method: string }> = [];
fake.cdp.sendToTarget = vi.fn(async (target, method) => {
targetCalls.push({ sessionId: target.sessionId, method });
if (method === "DOM.scrollIntoViewIfNeeded") return {};
if (method === "DOM.focus" || method === "Runtime.releaseObject") return {};
if (method === "DOM.resolveNode") return { object: { objectId: "focus-target" } };
if (method === "Runtime.callFunctionOn") return { result: { value: true } };
throw new Error(`unexpected child CDP call ${method}`);
}) as CdpRunner["sendToTarget"];
const res = await handleFocus(
sm,
{ session_id: "aa11", ref: "@e3" },
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
);
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
expect(res.focused).toBe(true);
expect(targetCalls).toEqual([
{ sessionId: "child-session", method: "DOM.scrollIntoViewIfNeeded" },
{ sessionId: "child-session", method: "DOM.resolveNode" },
{ sessionId: "child-session", method: "DOM.focus" },
{ sessionId: "child-session", method: "Runtime.callFunctionOn" },
{ sessionId: "child-session", method: "Runtime.releaseObject" },
]);
});
it("does not focus after cancellation during scrolling", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
ctx.refStore.set("e3", 1234, { tabId: 4 });
const abort = new AbortController();
const fake = makeFakeCdp({
"DOM.scrollIntoViewIfNeeded": () => {
abort.abort();
return {};
},
});
const res = await handleFocus(
sm,
{ session_id: "aa11", ref: "@e3" },
{ cdp: fake.cdp, tabsApi: fake.tabsApi, signal: abort.signal },
);
expect(res).toMatchObject({ code: "cancelled" });
expect(fake.sent.some((call) => call.method === "DOM.focus")).toBe(false);
});
it("blurs a ref and returns its previous focus state", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
ctx.refStore.set("e3", 1234, { tabId: 4 });
const fake = makeFakeCdp({
"DOM.resolveNode": () => ({ object: { objectId: "focus-target" } }),
"Runtime.callFunctionOn": vi
.fn()
.mockResolvedValueOnce({ result: { value: { ok: true, was_focused: true } } })
.mockResolvedValueOnce({ result: { value: false } }),
});
const res = await handleBlur(
sm,
{ session_id: "aa11", ref: "@e3" },
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
);
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
expect(res).toMatchObject({
tab_id: 4,
used_ref: "e3",
was_focused: true,
focused: false,
});
expect(fake.sent.map((call) => call.method)).toEqual([
"DOM.resolveNode",
"Runtime.callFunctionOn",
"Runtime.callFunctionOn",
"Runtime.releaseObject",
]);
});
it("rejects a target that does not implement blur", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
ctx.refStore.set("e3", 1234, { tabId: 4 });
const fake = makeFakeCdp({
"DOM.resolveNode": () => ({ object: { objectId: "focus-target" } }),
"Runtime.callFunctionOn": () => ({
result: { value: { ok: false, was_focused: false, focused: false } },
}),
});
const res = await handleBlur(
sm,
{ session_id: "aa11", ref: "@e3" },
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
);
expect(res).toMatchObject({ code: "invalid_params", message: /does not support blur/ });
});
});
function successfulFillScript(params: unknown) {
const script = params as { arguments?: Array<{ value: unknown }>; functionDeclaration: string };
const args = script.arguments ?? [];
@@ -3813,9 +3813,7 @@ describe("handleSnapshot", () => {
if ("code" in result) throw new Error(`unexpected error: ${JSON.stringify(result)}`);
expect(result.text).toContain('@e1 button "Frame action"');
const frameRef = [...ctx.refStore.entries()].find(
([, entry]) => entry.kind === "dom" && entry.backendNodeId === 22,
);
const frameRef = [...ctx.refStore.entries()].find(([, entry]) => entry.backendNodeId === 22);
expect(frameRef?.[1]).toMatchObject({
tabId: 4,
frameId: "child",
@@ -1,348 +0,0 @@
// @vitest-environment node
// Opt in with BSK_SCROLL_CHROME=/path/to/chrome. Uses an isolated browser/profile.
import { describe, expect, it } from "vitest";
import type { CdpFrame, CdpFrameGraph, CdpTarget } from "@/browser-driver/frame-graph";
import { SessionManager } from "@/session-manager/manager";
import type { ViewportRect } from "../geometry";
import { handleScrollTo } from "../scroll";
import type { CdpRunner } from "../shared";
type Send = <T = Record<string, unknown>>(
method: string,
params?: object,
sessionId?: string,
) => Promise<T>;
type Tree = { frame: { id: string; parentId?: string; name?: string }; childFrames?: Tree[] };
async function withBrowser(
run: (h: Awaited<ReturnType<typeof harness>>) => Promise<void>,
zoom = 1,
) {
const { withChrome } = await import(
new URL(
"../../../../../evals/browser/cases/regression/snapshot-coordinates/chrome.mjs",
import.meta.url,
).href
);
await withChrome(
{ executable: process.env.BSK_SCROLL_CHROME, deviceScale: 1, zoom },
async (send: Send) => run(await harness(send)),
);
}
async function harness(send: Send) {
const { targetId } = await send<{ targetId: string }>("Target.createTarget", {
url: "about:blank",
});
const { sessionId: rootSession } = await send<{ sessionId: string }>("Target.attachToTarget", {
targetId,
flatten: true,
});
await send("Page.bringToFront", {}, rootSession);
const manager = new SessionManager({
agentWindow: {
create: async () => 100,
remove: async () => {},
ensureActiveTab: async () => 4,
},
});
const ctx = await manager.start("aa11");
const calls: { method: string; sessionId?: string }[] = [];
const after = { command: (_method: string) => {} };
const sessionFor = (target: CdpTarget) => target.sessionId ?? rootSession;
const forward = async <T>(target: CdpTarget, method: string, params?: object): Promise<T> => {
calls.push({ method, sessionId: target.sessionId });
const result = await send<T>(method, params, sessionFor(target));
after.command(method);
return result;
};
const cdp: CdpRunner = {
send: (_tab, method, params) => forward({ tabId: 4 }, method, params),
sendToTarget: forward,
};
const evaluate = async <T>(expression: string, frame?: CdpFrame): Promise<T> => {
const session = frame ? sessionFor(frame.target) : rootSession;
const world = frame
? await send<{ executionContextId: number }>(
"Page.createIsolatedWorld",
{
frameId: frame.frameId,
worldName: "scroll-oracle",
},
session,
)
: undefined;
const reply = await send<{ result: { value: T }; exceptionDetails?: unknown }>(
"Runtime.evaluate",
{
expression,
contextId: world?.executionContextId,
returnByValue: true,
awaitPromise: true,
},
session,
);
expect(reply.exceptionDetails).toBeUndefined();
return reply.result.value;
};
const rememberRef = (frame: CdpFrame, backendNodeId: number) =>
ctx.refStore.set("e1", backendNodeId, {
tabId: 4,
frameId: frame.frameId,
cdpSessionId: frame.target.sessionId,
});
const ref = async (frame: CdpFrame, selector = "#probe") => {
const { executionContextId } = await send<{ executionContextId: number }>(
"Page.createIsolatedWorld",
{
frameId: frame.frameId,
worldName: "scroll-oracle",
},
sessionFor(frame.target),
);
const { result } = await send<{ result: { objectId: string } }>(
"Runtime.evaluate",
{
expression: `document.querySelector(${JSON.stringify(selector)})`,
contextId: executionContextId,
},
sessionFor(frame.target),
);
const { node } = await send<{ node: { backendNodeId: number } }>(
"DOM.describeNode",
{ objectId: result.objectId },
sessionFor(frame.target),
);
await send("Runtime.releaseObject", { objectId: result.objectId }, sessionFor(frame.target));
rememberRef(frame, node.backendNodeId);
};
const scroll = (target: { selector: string } | { ref: string }, signal?: AbortSignal) =>
handleScrollTo(
manager,
{ session_id: "aa11", ...target },
{
cdp,
signal,
tabsApi: {
get: async (id) => ({ id, windowId: 100, active: true }) as chrome.tabs.Tab,
query: async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab],
},
},
);
const loadFrames = async () => {
const { targetInfos } = await send<{ targetInfos: { targetId: string; type: string }[] }>(
"Target.getTargets",
);
const sessions = [rootSession];
for (const target of targetInfos.filter((t) => t.type === "iframe")) {
const { sessionId } = await send<{ sessionId: string }>("Target.attachToTarget", {
targetId: target.targetId,
flatten: true,
});
sessions.push(sessionId);
}
const frames: (CdpFrame & { name: string })[] = [];
for (const session of sessions) {
const { frameTree } = await send<{ frameTree: Tree }>("Page.getFrameTree", {}, session);
const target = { tabId: 4, ...(session !== rootSession ? { sessionId: session } : {}) };
const visit = (tree: Tree, parentFrameId = tree.frame.parentId) => {
frames.push({ frameId: tree.frame.id, parentFrameId, target, name: tree.frame.name ?? "" });
for (const child of tree.childFrames ?? []) visit(child, tree.frame.id);
};
visit(frameTree);
}
for (const frame of frames) {
if (!frame.parentFrameId) continue;
const parent = frames.find((f) => f.frameId === frame.parentFrameId)!;
const owner = await send<{ backendNodeId: number }>(
"DOM.getFrameOwner",
{ frameId: frame.frameId },
sessionFor(parent.target),
);
frame.ownerBackendNodeId = owner.backendNodeId;
}
const graph: CdpFrameGraph = { rootFrameId: frames[0].frameId, frames };
cdp.getFrameGraph = async () => graph;
return frames;
};
return { send, rootSession, evaluate, ref, rememberRef, scroll, loadFrames, calls, after };
}
function intersection(a: ViewportRect, b: ViewportRect): ViewportRect | null {
const x = Math.max(a.x, b.x),
y = Math.max(a.y, b.y);
const right = Math.min(a.x + a.width, b.x + b.width),
bottom = Math.min(a.y + a.height, b.y + b.height);
return right > x && bottom > y ? { x, y, width: right - x, height: bottom - y } : null;
}
function expectRect(actual: unknown, expected: ViewportRect) {
expect(actual).not.toHaveProperty("code");
for (const key of ["x", "y", "width", "height"] as const)
expect(Math.abs((actual as ViewportRect)[key] - expected[key]), key).toBeLessThan(2);
}
describe.skipIf(!process.env.BSK_SCROLL_CHROME)("real browser scroll-to", () => {
it("scrolls ordinary elements and reports the clipped portion of nested scrollers", async () => {
await withBrowser(async (h) => {
for (const html of [
'<div style="height:2200px"></div><div id="target" style="width:100px;height:40px"></div>',
'<div style="height:200px"></div><div id="clip" style="height:80px;width:200px;overflow:auto"><div id="target" style="width:100px;height:400px"></div></div>',
'<div style="height:200px"></div><div id="clip" style="height:80px;width:200px;overflow:auto;transform:scale(1.25);transform-origin:0 0"><div id="target" style="width:100px;height:400px"></div></div>',
]) {
await h.evaluate(`document.body.innerHTML = ${JSON.stringify(html)}`);
const result = await h.scroll({ selector: "#target" });
const oracle = await h.evaluate<{ rect: ViewportRect; clip?: ViewportRect }>(`({
rect: document.querySelector('#target').getBoundingClientRect().toJSON(),
clip: document.querySelector('#clip')?.getBoundingClientRect().toJSON()
})`);
expectRect(result, oracle.clip ? intersection(oracle.rect, oracle.clip)! : oracle.rect);
}
});
});
it.each([
"visibility:hidden",
"opacity:0",
"display:none",
"content-visibility:hidden",
])("rejects %s targets", async (style) => {
await withBrowser(async (h) => {
await h.evaluate(
`document.body.innerHTML = '<div style="${style}"><button id="target">Target</button></div>'`,
);
expect(await h.scroll({ selector: "#target" })).toMatchObject({
code: "permission_denied",
data: { reason: "element_not_visible" },
});
});
});
it("rejects targets fully outside an overflow:clip ancestor", async () => {
await withBrowser(async (h) => {
await h.evaluate(
`document.body.innerHTML = '<div style="height:80px;overflow:clip"><div style="height:150px"></div><button id="target">Target</button></div>'`,
);
expect(await h.scroll({ selector: "#target" })).toMatchObject({ code: "permission_denied" });
});
});
it.each(["open", "closed"])("clips %s shadow-root refs", async (mode) => {
await withBrowser(async (h) => {
await h.evaluate(`(() => {
const host = document.createElement('div'); document.body.append(host);
const root = host.attachShadow({mode:'${mode}'});
root.innerHTML = '<div style="height:60px;overflow:auto"><button id="probe" style="height:200px;width:100px">Target</button></div>';
window.probe = root.querySelector('#probe');
})()`);
const frames = await h.loadFrames();
const { result } = await h.send<{ result: { objectId: string } }>(
"Runtime.evaluate",
{ expression: "window.probe" },
h.rootSession,
);
// Build the ref from the actual closed-root node, as an observation does.
const { node } = await h.send<{ node: { backendNodeId: number } }>(
"DOM.describeNode",
{ objectId: result.objectId },
h.rootSession,
);
await h.send("Runtime.releaseObject", { objectId: result.objectId }, h.rootSession);
h.rememberRef(frames[0], node.backendNodeId);
expect(await h.scroll({ ref: "e1" })).toMatchObject({ height: 60 });
});
});
it("cancels after scrolling and releases the measurement object without follow-up reads", async () => {
await withBrowser(async (h) => {
await h.evaluate("document.body.innerHTML = '<button id=target>Target</button>'");
const abort = new AbortController();
h.after.command = (method) => {
if (method === "DOM.resolveNode") abort.abort();
};
expect(await h.scroll({ selector: "#target" }, abort.signal)).toMatchObject({
code: "cancelled",
});
expect(h.calls.at(-1)!.method).toBe("Runtime.releaseObjectGroup");
expect(h.calls.some((call) => call.method === "Runtime.callFunctionOn")).toBe(false);
});
});
it.each([
{ fixture: "snapshot-coordinates", zoom: 1 },
{ fixture: "snapshot-coordinates", zoom: 1.25 },
{ fixture: "oopif-scrollbars", zoom: 1 },
{ fixture: "oopif-scrollbars", zoom: 0.8 },
])("scrolls nested frames: $fixture at zoom $zoom", async ({ fixture, zoom }) => {
const { createEvalServer } = await import(
new URL("../../../../../evals/browser/lib/server.mjs", import.meta.url).href
);
const server = createEvalServer();
const { baseUrl } = await server.start();
try {
await withBrowser(async (h) => {
await h.send("Page.navigate", { url: `${baseUrl}/${fixture}?run=scroll` }, h.rootSession);
await expect
.poll(
() =>
server
.snapshot("scroll")
.events.some(
(e: { type: string; data: { root?: boolean } }) =>
e.type === "geometry.ready" && e.data.root,
),
{ timeout: 10_000 },
)
.toBe(true);
const frames = await h.loadFrames();
expect(frames.length).toBe(fixture === "snapshot-coordinates" ? 5 : 3);
for (const frame of frames) {
await h.ref(frame);
const result = await h.scroll({ ref: "e1" });
// Independent DOM oracle: CSS border boxes plus this fixture's known
// axis-aligned iframe scales. Re-read after each scroll changes layout.
let rect = await h.evaluate<ViewportRect>(
"document.querySelector('#probe').getBoundingClientRect().toJSON()",
frame,
);
let current = frame;
while (true) {
const viewport = await h.evaluate<ViewportRect>(
"({ x:0, y:0, width:document.documentElement.clientWidth, height:document.documentElement.clientHeight })",
current,
);
rect = intersection(rect, viewport)!;
if (!current.parentFrameId) break;
const parent = frames.find((f) => f.frameId === current.parentFrameId)!;
const owner = await h.evaluate<{ x: number; y: number; scale: number }>(
`(() => {
const iframe = document.querySelector('iframe[name="${current.name}"]');
const r = iframe.getBoundingClientRect(); const s = getComputedStyle(iframe);
const scale = r.width / iframe.offsetWidth;
return { x:r.x + (iframe.clientLeft + parseFloat(s.paddingLeft))*scale,
y:r.y + (iframe.clientTop + parseFloat(s.paddingTop))*scale, scale };
})()`,
parent,
);
rect = {
x: owner.x + rect.x * owner.scale,
y: owner.y + rect.y * owner.scale,
width: rect.width * owner.scale,
height: rect.height * owner.scale,
};
current = parent;
}
expectRect(result, rect);
}
const child = frames.find((frame) => frame.parentFrameId === frames[0].frameId)!;
await h.ref(child);
await h.evaluate(
`document.querySelector('iframe[name="${child.name}"]').style.opacity = '0'`,
);
expect(await h.scroll({ ref: "e1" })).toMatchObject({ code: "permission_denied" });
}, zoom);
} finally {
await server.stop();
}
}, 30_000);
});
@@ -1,310 +0,0 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import type { CdpFrameGraph, CdpTarget } from "@/browser-driver/frame-graph";
import { SessionManager } from "@/session-manager/manager";
import type { JavaScriptDialogInfo, ScrollToParams } from "@/transport/types";
import type { ViewportRect } from "../geometry";
import { handleScrollTo } from "../scroll";
import type { CdpRunner } from "../shared";
interface Call {
target: CdpTarget;
method: string;
params?: Record<string, unknown>;
aborted: boolean;
}
async function fixture(frame: "top" | "same-target" | "oopif" = "top") {
const manager = new SessionManager({
agentWindow: {
create: async () => 100,
remove: async () => {},
ensureActiveTab: async () => 4,
},
});
const ctx = await manager.start("aa11");
ctx.refStore.set("e3", 1234, {
tabId: 4,
...(frame !== "top" ? { frameId: "child" } : {}),
...(frame === "oopif" ? { cdpSessionId: "child-session" } : {}),
});
const abort = new AbortController();
const graph: CdpFrameGraph = {
rootFrameId: "main",
frames: [
{ frameId: "main", target: { tabId: 4 } },
{
frameId: "child",
parentFrameId: "main",
ownerBackendNodeId: 99,
target: { tabId: 4, ...(frame === "oopif" ? { sessionId: "child-session" } : {}) },
},
],
};
const rectangles = new Map<number, ViewportRect | null>([
[1234, { x: 10, y: 20, width: 100, height: 40 }],
[99, { x: 204, y: 306, width: 400, height: 200 }],
]);
const calls: Call[] = [];
const hooks = {
after: (_call: Call) => {},
reply: (_call: Call): object | undefined => undefined,
};
const send = async <T>(target: CdpTarget, method: string, params?: object): Promise<T> => {
const call = {
target,
method,
params: params as Call["params"],
aborted: abort.signal.aborted,
};
calls.push(call);
const override = hooks.reply(call);
hooks.after(call);
if (override) return override as T;
const args = call.params ?? {};
const replies: Record<string, unknown> = {
"DOM.getDocument": { root: { nodeId: 1 } },
"DOM.querySelector": { nodeId: 2 },
"DOM.describeNode": { node: { backendNodeId: 1234 } },
"DOM.scrollIntoViewIfNeeded": {},
"DOM.resolveNode": { object: { objectId: String(args.backendNodeId) } },
"Runtime.callFunctionOn": {
result: {
value: String(args.functionDeclaration).includes("IntersectionObserver")
? rectangles.get(Number(args.objectId))
: { width: 200, height: 100 },
},
},
"Runtime.releaseObject": {},
"Runtime.releaseObjectGroup": {},
"Runtime.evaluate": { result: { value: { width: 200, height: 100 } } },
"Page.getLayoutMetrics": {
cssLayoutViewport: {
clientWidth: target.sessionId ? 185 : 1280,
clientHeight: target.sessionId ? 89 : 720,
},
},
"DOM.getBoxModel": { model: { content: [204, 306, 604, 306, 604, 506, 204, 506] } },
};
if (!(method in replies)) throw new Error(`unexpected CDP call ${method}`);
return replies[method] as T;
};
const cdp: CdpRunner = {
send: (tabId, method, params) => send({ tabId }, method, params),
sendToTarget: send,
getFrameGraph: vi.fn(async () => {
hooks.after({ target: { tabId: 4 }, method: "getFrameGraph", aborted: abort.signal.aborted });
return graph;
}),
trackSessionTab: vi.fn(),
};
const tabsApi = {
get: vi.fn(async (id: number) => ({ id, windowId: 100, active: true }) as chrome.tabs.Tab),
query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
};
const run = (params: Partial<ScrollToParams> = {}) =>
handleScrollTo(
manager,
{ session_id: "aa11", ref: "@e3", ...params },
{ cdp, tabsApi, signal: abort.signal },
);
return { ctx, cdp, graph, rectangles, calls, hooks, abort, tabsApi, run };
}
afterEach(() => vi.restoreAllMocks());
describe("handleScrollTo", () => {
it("scrolls a ref and returns renderer-clipped bounds", async () => {
const f = await fixture();
expect(await f.run()).toMatchObject({
tab_id: 4,
used_ref: "e3",
x: 10,
y: 20,
width: 100,
height: 40,
});
expect(f.calls[0]).toMatchObject({
method: "DOM.scrollIntoViewIfNeeded",
params: { backendNodeId: 1234 },
});
const group = f.calls.find((call) => call.method === "DOM.resolveNode")!.params!.objectGroup;
expect(f.calls.at(-1)).toMatchObject({
method: "Runtime.releaseObjectGroup",
params: { objectGroup: group },
});
});
it("resolves a selector and tracks the attached tab even when it is missing", async () => {
const f = await fixture();
expect(await f.run({ ref: undefined, selector: "#target" })).toMatchObject({
used_selector: "#target",
});
f.hooks.reply = ({ method }) => (method === "DOM.querySelector" ? { nodeId: 0 } : undefined);
expect(await f.run({ ref: undefined, selector: "#missing" })).toMatchObject({
code: "not_found",
data: { reason: "selector_not_found" },
});
expect(f.cdp.trackSessionTab).toHaveBeenCalledWith("aa11", 4);
});
it.each([
"same-target",
"oopif",
] as const)("projects %s refs and scrolls their owners", async (frame) => {
const f = await fixture(frame);
expect(await f.run()).toMatchObject({ x: 224, y: 346, width: 200, height: 80 });
expect(
f.calls
.filter((call) => call.method === "DOM.scrollIntoViewIfNeeded")
.map((call) => call.params!.backendNodeId),
).toEqual([99, 1234]);
expect(f.cdp.getFrameGraph).toHaveBeenCalledTimes(1);
if (frame === "oopif")
expect(f.calls.find((call) => call.params?.backendNodeId === 1234)!.target.sessionId).toBe(
"child-session",
);
});
it("clips OOPIF bounds to occupied scrollbars without changing the frame scale", async () => {
const f = await fixture("oopif");
f.rectangles.set(1234, { x: 180, y: 80, width: 5, height: 9 });
expect(await f.run()).toMatchObject({ x: 564, y: 466, width: 10, height: 18 });
});
it("clips against the iframe owner's ordinary DOM ancestors", async () => {
const f = await fixture("oopif");
f.rectangles.set(99, { x: 204, y: 306, width: 400, height: 80 });
expect(await f.run()).toMatchObject({ x: 224, y: 346, width: 200, height: 40 });
});
it.each([1234, 99])("rejects a hidden or fully clipped node %i", async (node) => {
const f = await fixture("oopif");
f.rectangles.set(node, null);
expect(await f.run()).toMatchObject({
code: "permission_denied",
data: { reason: "element_not_visible" },
});
expect(f.calls.at(-1)!.method).toBe("Runtime.releaseObjectGroup");
});
it.each([
{ ref: undefined },
{ selector: "#target" },
{ timeout_ms: 0 },
])("rejects invalid params %j", async (params) => {
const f = await fixture();
expect(await f.run(params)).toMatchObject({ code: "invalid_params" });
expect(f.calls).toEqual([]);
});
it.each(["stale", "other-tab"])("rejects %s refs before scrolling", async (mode) => {
const f = await fixture();
if (mode === "stale") f.ctx.refStore.clear();
else f.ctx.refStore.set("e3", 1234, { tabId: 8 });
expect(await f.run()).toMatchObject({ code: "not_found", data: { reason: "ref_not_found" } });
expect(f.calls).toEqual([]);
});
it("enforces the Agent Window boundary", async () => {
const f = await fixture();
f.tabsApi.query.mockResolvedValue([{ id: 4, windowId: 200, active: true } as chrome.tabs.Tab]);
f.tabsApi.get.mockResolvedValue({ id: 4, windowId: 200, active: true } as chrome.tabs.Tab);
expect(await f.run({ tab_id: 4 })).toMatchObject({ code: "permission_denied" });
expect(f.calls).toEqual([]);
});
it.each([
"getFrameGraph",
"DOM.scrollIntoViewIfNeeded",
"DOM.resolveNode",
"Runtime.callFunctionOn",
"Page.getLayoutMetrics",
"Runtime.evaluate",
"DOM.getBoxModel",
])("honors cancellation during %s and only cleans up afterwards", async (method) => {
const f = await fixture("oopif");
f.hooks.after = (call) => {
if (call.method === method) f.abort.abort();
};
expect(await f.run()).toMatchObject({ code: "cancelled" });
expect(
f.calls.filter((call) => call.aborted && call.method !== "Runtime.releaseObjectGroup"),
).toEqual([]);
});
it.each([
"DOM.getDocument",
"DOM.querySelector",
"DOM.describeNode",
])("cancels selector resolution during %s", async (method) => {
const f = await fixture();
f.hooks.after = (call) => {
if (call.method === method) f.abort.abort();
};
expect(await f.run({ ref: undefined, selector: "#target" })).toMatchObject({
code: "cancelled",
});
expect(f.calls.filter((call) => call.aborted)).toEqual([]);
});
it("does not start after early cancellation", async () => {
const f = await fixture();
f.abort.abort();
expect(await f.run()).toMatchObject({ code: "cancelled" });
expect(f.calls).toEqual([]);
expect(f.tabsApi.query).not.toHaveBeenCalled();
});
it("does not issue fallback scrolling after cancellation", async () => {
const f = await fixture();
f.hooks.reply = ({ method }) => {
if (method === "DOM.scrollIntoViewIfNeeded") {
f.abort.abort();
throw new Error("scroll failed");
}
return undefined;
};
expect(await f.run()).toMatchObject({ code: "cancelled" });
expect(f.calls.map((call) => call.method)).toEqual(["DOM.scrollIntoViewIfNeeded"]);
});
it("stops target scrolling after a deadline expires while scrolling its owner", async () => {
const f = await fixture("oopif");
const now = vi.spyOn(Date, "now").mockReturnValue(100);
f.hooks.after = ({ method }) => {
if (method === "DOM.scrollIntoViewIfNeeded") now.mockReturnValue(201);
};
expect(await f.run({ timeout_ms: 100 })).toMatchObject({ code: "timeout" });
expect(f.calls.filter((call) => call.method === "DOM.scrollIntoViewIfNeeded")).toHaveLength(1);
});
it("reports measurement exceptions and releases objects even if cleanup fails", async () => {
const f = await fixture();
f.hooks.reply = ({ method }) => {
if (method === "Runtime.callFunctionOn")
return { exceptionDetails: { text: "measurement failed" } };
if (method === "Runtime.releaseObjectGroup") throw new Error("target closed");
return undefined;
};
expect(await f.run()).toMatchObject({ code: "cdp_failed", message: "measurement failed" });
expect(f.calls.at(-1)!.method).toBe("Runtime.releaseObjectGroup");
});
it("attaches dialogs observed during scrolling", async () => {
const f = await fixture();
f.cdp.dialogCursor = () => 5;
const dialogs: JavaScriptDialogInfo[] = [
{
sequence: 6,
tab_id: 4,
type: "alert",
message: "scrolled",
handled: "accepted",
url: "https://example.test",
},
];
f.cdp.dialogsSince = vi.fn(() => dialogs);
expect(await f.run()).toHaveProperty("dialogs", dialogs);
expect(f.cdp.dialogsSince).toHaveBeenCalledWith(4, 5);
});
});
@@ -1,199 +0,0 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import { SessionManager } from "@/session-manager/manager";
import { RefStore, type VisualRefInput } from "@/session-manager/ref-store";
import { handleDownload } from "../download";
import { handleRequestHelp, resetHelpLifecycleForTests } from "../human-loop";
import { handleClick, handleFill, handleHover, handlePress, handleSelect } from "../interaction";
import { handleGetHtml } from "../observation";
import type { CdpRunner } from "../shared";
import { lookupRefTarget, lookupSnapshotRef, resolveSnapshotRef } from "../snapshot-ref";
import { handleUpload } from "../upload";
function visual(): VisualRefInput {
return {
kind: "visual-region",
candidate: {
document: {
attachmentId: "a",
target: { tabId: 4, sessionId: "child" },
frameId: "frame",
documentElementBackendNodeId: 1,
},
backendNodeId: 42,
parentBackendNodeId: 1,
region: {
status: "available",
borderBox: { x: 0, y: 0, width: 100, height: 50 },
crop: { x: 0, y: 0, width: 100, height: 50 },
},
},
};
}
async function setup() {
const manager = new SessionManager({
agentWindow: {
create: async () => 100,
remove: async () => {},
ensureActiveTab: async () => 4,
},
});
const ctx = await manager.start("test");
ctx.refStore.replace([
["e1", visual()],
["e2", { backendNodeId: 43, tabId: 4 }],
]);
const send = vi.fn(async () => {
throw new Error("unexpected target operation");
});
const cdp = { send, sendToTarget: send, trackSessionTab: vi.fn() } as unknown as CdpRunner;
const tab = { id: 4, windowId: 100, active: true, url: "https://example.com" } as chrome.tabs.Tab;
const tabsApi = { get: vi.fn(async () => tab), query: vi.fn(async () => [tab]) };
return { manager, ctx, send, cdp, tabsApi };
}
describe("typed visual refs", () => {
afterEach(() => resetHelpLifecycleForTests());
it("keeps candidate evidence and cannot resolve a visual anchor as a bare DOM node", () => {
const store = new RefStore();
const input = visual();
store.replace([
["@e1", input],
["e2", { backendNodeId: 43, tabId: 4 }],
]);
const entry = store.resolveEntry("e1");
expect(entry?.kind).toBe("visual-region");
if (entry?.kind !== "visual-region") throw new Error("missing visual ref");
expect(entry.candidate).toBe(input.candidate);
expect(entry.generation).toBe(1);
expect(store.resolve("@e1")).toBeNull();
expect(store.resolve("e2")).toBe(43);
// The latest observation replaces both target kinds; eN strings carry no generation.
store.replace([["e1", { backendNodeId: 99, tabId: 4 }]]);
expect(store.resolve("e1")).toBe(99);
expect(store.resolveEntry("e1")).toMatchObject({ kind: "dom", generation: 2 });
expect(store.resolveEntry("e2")).toBeNull();
});
it("rejects missing visual identity without partially publishing a replacement", () => {
const store = new RefStore();
store.set("e1", 10, { tabId: 4 });
const invalid = visual();
invalid.candidate.document.attachmentId = "";
expect(() =>
store.replace([
["e2", 20],
["e3", invalid],
]),
).toThrow(TypeError);
expect(store.resolveEntry("e1")).toMatchObject({ generation: 0, backendNodeId: 10 });
expect(store.size()).toBe(1);
store.replace([["e1", visual()]]);
expect(store.resolveEntry("e1")?.generation).toBe(1);
});
it("isolates tabs/sessions and distinguishes unsupported targets from missing refs", async () => {
const { ctx } = await setup();
expect(lookupRefTarget(ctx, "@e1", 4)?.kind).toBe("visual-region");
expect(lookupRefTarget(ctx, "e1", 5)).toBeNull();
expect(lookupSnapshotRef(ctx, "e1", 4)).toBeNull();
expect(resolveSnapshotRef(ctx, "e1", 4)).toMatchObject({
code: "unsupported",
data: { reason: "ref_kind_unsupported" },
});
expect(resolveSnapshotRef(ctx, "e1", 5)).toMatchObject({
code: "not_found",
data: { reason: "ref_not_found" },
});
expect(lookupRefTarget({ ...ctx, refStore: new RefStore() }, "e1", 4)).toBeNull();
expect(lookupSnapshotRef(ctx, "e2", 4)).toMatchObject({ backendNodeId: 43 });
});
it.each([
"click",
"fill",
"hover",
"press",
"select",
"get_html",
])("rejects visual refs in %s before target effects", async (tool) => {
const { manager, send, cdp, tabsApi } = await setup();
const params = { session_id: "test", tab_id: 4, ref: "@e1" };
const deps = { cdp, tabsApi };
let result: unknown;
switch (tool) {
case "click":
result = await handleClick(manager, params, deps);
break;
case "fill":
result = await handleFill(manager, { ...params, value: "hello" }, deps);
break;
case "hover":
result = await handleHover(manager, params, deps);
break;
case "press":
result = await handlePress(manager, { ...params, key: "Enter" }, deps);
break;
case "select":
result = await handleSelect(manager, { ...params, values: ["one"] }, deps);
break;
case "get_html":
result = await handleGetHtml(manager, params, deps);
break;
}
expect(result).toMatchObject({ code: "unsupported", data: { reason: "ref_kind_unsupported" } });
expect(send).not.toHaveBeenCalled();
});
it.each([
"input",
"drop",
"download",
] as const)("rejects visual refs before %s file-transfer setup", async (mode) => {
const { manager, send, cdp, tabsApi } = await setup();
const params = { session_id: "test", tab_id: 4, ref: "e1" };
const result =
mode === "download"
? await handleDownload(
manager,
{ ...params, browser_relative_dir: "test" },
{ cdp, tabsApi },
)
: await handleUpload(
manager,
{
...params,
mode,
files: [{ transfer_id: "test", name: "test.txt", staged_path: "/unused/test.txt" }],
},
{ cdp, tabsApi },
);
expect(result).toMatchObject({ code: "unsupported", data: { reason: "ref_kind_unsupported" } });
expect(send).not.toHaveBeenCalled();
});
it("does not scroll or highlight a visual anchor in human help", async () => {
const { manager, cdp, tabsApi, send } = await setup();
const sendToTab = vi.fn(async (_tab: number, _message: unknown) => ({
type: "bsk-help-response",
outcome: "continued",
}));
const result = await handleRequestHelp(
manager,
{ session_id: "test", tab_id: 4, prompt: "help", targets: [{ ref: "@e1" }] },
{
cdp,
tabsApi,
sendToTab,
windows: { update: vi.fn(async () => ({}) as never) },
activateTab: vi.fn(async () => {}),
notifications: null,
autoAttachLifecycle: false,
},
);
expect(result).toMatchObject({ resolved_targets: [{ ref: "@e1", matched: false }] });
expect(send).not.toHaveBeenCalled();
expect(sendToTab.mock.calls[0][1]).toMatchObject({ rects: [], selectors: [] });
});
});
@@ -1,382 +0,0 @@
import { describe, expect, it, vi } from "vitest";
import type { CdpTarget } from "@/browser-driver/frame-graph";
import { SessionManager } from "@/session-manager/manager";
import { handleScreenshot } from "../observation";
import type { CdpRunner } from "../shared";
import { captureVisualScreenshot, visualScreenshotScale } from "../visual-screenshot";
import type { VisualCandidate, VisualFramePath } from "../vom/visual-discovery";
const styles = {
position: "static",
visibility: "visible",
opacity: "1",
display: "block",
"overflow-x": "visible",
"overflow-y": "visible",
transform: "none",
zoom: "1",
"clip-path": "none",
"mask-image": "none",
rotate: "none",
scale: "none",
perspective: "none",
clip: "auto",
contain: "none",
"overflow-clip-margin": "0px",
};
const rect = (x = 10, y = 20, width = 100, height = 40) => ({ x, y, width, height });
const root = {
attachmentId: "a",
target: { tabId: 4 },
frameId: "top",
documentElementBackendNodeId: 1,
};
function row(id: number, tag: string, box = rect()) {
return {
node: { backend: id },
tag,
box,
client: box,
contentSize: { width: box.width, height: box.height },
styles: { ...styles },
};
}
function encode(value: unknown): unknown {
if (value === null) return { type: "null" };
if (Array.isArray(value)) return { type: "array", value: value.map(encode) };
if (typeof value === "object") {
const record = value as Record<string, unknown>;
if ("backend" in record) return { type: "node", value: { backendNodeId: record.backend } };
return { type: "object", value: Object.entries(record).map(([k, v]) => [k, encode(v)]) };
}
return { type: typeof value, value };
}
function png(width: number, height: number) {
const bytes = new Uint8Array(33);
bytes.set([137, 80, 78, 71, 13, 10, 26, 10, 0, 0, 0, 13, 73, 72, 68, 82]);
const view = new DataView(bytes.buffer);
view.setUint32(16, width);
view.setUint32(20, height);
return btoa(String.fromCharCode(...bytes));
}
function fixture(child = false, oopif = false) {
const parent: VisualFramePath = { document: root };
const frame: VisualFramePath = child
? {
document: {
...root,
frameId: "child",
documentElementBackendNodeId: 11,
target: oopif ? { tabId: 4, sessionId: "oopif" } : { tabId: 4 },
},
parent: { frame: parent, ownerBackendNodeId: 2 },
}
: parent;
const crop = child ? rect(120, 240, 200, 80) : rect();
const candidate: VisualCandidate = {
document: frame.document,
backendNodeId: child ? 12 : 3,
parentBackendNodeId: frame.document.documentElementBackendNodeId,
framePath: frame,
region: { status: "available", borderBox: crop, crop },
};
const topRows = child
? [row(2, "iframe", rect(100, 200, 400, 200)), row(1, "html", rect(0, 0, 1200, 800))]
: [row(3, "canvas"), row(1, "html", rect(0, 0, 1200, 800))];
if (child) topRows[0].contentSize = { width: 200, height: 100 };
const childRows = [row(12, "canvas"), row(11, "html", rect(0, 0, 200, 100))];
let calls = 0;
const control = {
wrongOwner: false,
rootChanged: false,
failRead: false,
abortRead: false,
dpr: 1,
shots: [png(crop.width, crop.height)],
};
const controller = new AbortController();
const send = vi.fn(
async (target: CdpTarget, method: string, params: Record<string, unknown> = {}) => {
if (method === "DOM.getFrameOwner") return { backendNodeId: control.wrongOwner ? 999 : 2 };
if (method === "Page.createIsolatedWorld")
return { executionContextId: params.frameId === "child" ? 11 : 1 };
if (method === "DOM.resolveNode")
return { object: { objectId: String(params.backendNodeId) } };
if (method === "Runtime.callFunctionOn") {
const isChild = params.objectId === "12";
if (!(params.functionDeclaration as string).includes("styleNames"))
return {
result: {
deepSerializedValue: {
type: "node",
value: { backendNodeId: control.rootChanged ? 999 : isChild ? 11 : 1 },
},
},
};
if (control.failRead) throw new Error("read failed");
if (control.abortRead) controller.abort();
return {
result: {
deepSerializedValue: encode({
top: !isChild,
dpr: control.dpr,
rows: isChild ? childRows : topRows,
}),
},
};
}
if (method === "Runtime.releaseObjectGroup") return {};
// OOPIF projection uses the full viewport for scale, independently of
// the scrollbar-excluding layout viewport used for clipping.
if (method === "Runtime.evaluate" && target.sessionId === "oopif") {
expect(params).toMatchObject({
expression: "({ width: window.innerWidth, height: window.innerHeight })",
returnByValue: true,
});
return { result: { value: { width: 200, height: 100 } } };
}
if (method === "Page.getLayoutMetrics")
return {
cssLayoutViewport: { clientWidth: 1200, clientHeight: 800, pageX: 0, pageY: 0 },
cssVisualViewport: { scale: 1, zoom: 1 },
};
if (method === "DOM.getBoxModel")
return { model: { content: [100, 200, 500, 200, 500, 400, 100, 400] } };
if (method === "Page.captureScreenshot")
return { data: control.shots[Math.min(calls++, control.shots.length - 1)] };
throw new Error(`unexpected ${method} ${target.sessionId}`);
},
);
const cdp = {
send: ((tabId, method, params) =>
send({ tabId }, method, params as Record<string, unknown>)) as CdpRunner["send"],
sendToTarget: send as unknown as CdpRunner["sendToTarget"],
getAttachmentId: () => "a",
getFrameGraph: vi.fn(async () => {
throw new Error("whole page discovery forbidden");
}),
};
return { candidate, cdp, send, control, controller, topRows, childRows };
}
describe("visual screenshot", () => {
it.each([
[false, false],
[true, false],
[true, true],
])("screenshots the local target child=%s oopif=%s without whole-page discovery", async (child, oopif) => {
const f = fixture(child, oopif);
const result = await captureVisualScreenshot(f.cdp, f.candidate);
expect(result).toMatchObject({ width: child ? 200 : 100, height: child ? 80 : 40 });
expect(f.cdp.getFrameGraph).not.toHaveBeenCalled();
const names = f.send.mock.calls.map((c) => c[1]);
expect(names.filter((n) => n === "DOM.getFrameOwner")).toHaveLength(child ? 1 : 0);
expect(names.filter((n) => n === "DOM.resolveNode")).toHaveLength(child ? 2 : 1);
expect(names.some((n) => /scroll|DOMSnapshot|getFrameTree|Accessibility/.test(n))).toBe(false);
expect(f.send.mock.calls.find((c) => c[1] === "Page.captureScreenshot")?.[2]).toMatchObject({
clip: { ...f.candidate.region.crop, scale: 1 },
});
});
it.each([
false,
true,
])("projects a nested child through a parent target oopif=%s", async (oopif) => {
const f = fixture(true, oopif);
const parent = f.candidate.framePath!;
const document = {
...parent.document,
frameId: "grandchild",
documentElementBackendNodeId: 21,
};
const crop = rect(130, 250, 40, 20);
const candidate: VisualCandidate = {
...f.candidate,
document,
backendNodeId: 22,
framePath: { document, parent: { frame: parent, ownerBackendNodeId: 12 } },
region: { status: "available", borderBox: crop, crop },
};
f.childRows[0].tag = "iframe";
f.control.shots = [png(40, 20)];
const original = f.send.getMockImplementation()!;
f.send.mockImplementation(async (target, method, params = {}) => {
if (method === "DOM.getFrameOwner" && params.frameId === "grandchild")
return { backendNodeId: 12 };
if (method === "Runtime.callFunctionOn" && params.objectId === "22") {
if (!(params.functionDeclaration as string).includes("styleNames"))
return {
result: { deepSerializedValue: { type: "node", value: { backendNodeId: 21 } } },
};
return {
result: {
deepSerializedValue: encode({
top: false,
dpr: 1,
rows: [row(22, "canvas", rect(5, 5, 20, 10)), row(21, "html", rect(0, 0, 100, 40))],
}),
},
};
}
if (method === "DOM.getBoxModel" && params.backendNodeId === 12)
return {
model: {
content: oopif
? [10, 20, 110, 20, 110, 60, 10, 60]
: [120, 240, 320, 240, 320, 320, 120, 320],
},
};
if (method === "Page.getLayoutMetrics" && target.sessionId)
return {
cssLayoutViewport: { clientWidth: 200, clientHeight: 100, pageX: 0, pageY: 0 },
cssVisualViewport: { scale: 1, zoom: 1 },
};
return original(target, method, params);
});
expect(await captureVisualScreenshot(f.cdp, candidate)).toMatchObject({
width: 40,
height: 20,
});
expect(f.send.mock.calls.filter((c) => c[1] === "DOM.getFrameOwner")).toHaveLength(2);
expect(f.send.mock.calls.find((c) => c[1] === "Page.captureScreenshot")?.[2]).toMatchObject({
clip: crop,
});
expect(f.cdp.getFrameGraph).not.toHaveBeenCalled();
});
it("runs the live reader on only connected local ancestry", async () => {
const f = fixture();
await captureVisualScreenshot(f.cdp, f.candidate);
const params = f.send.mock.calls.find(
(c) =>
c[1] === "Runtime.callFunctionOn" &&
String(c[2]?.functionDeclaration).includes("styleNames"),
)![2]!;
const read = new Function(`return (${params.functionDeclaration});`)() as (
this: Element,
names: string[],
) => { rows: { node: Element }[] } | null;
const canvas = document.createElement("canvas");
document.body.append(canvas);
try {
const result = read.call(canvas, ["display"]);
expect(result?.rows.map((row) => row.node)).toEqual([
canvas,
document.body,
document.documentElement,
]);
canvas.remove();
expect(read.call(canvas, ["display"])).toBeNull();
const other = document.implementation.createHTMLDocument();
const foreign = other.createElement("canvas");
other.body.append(foreign);
expect(foreign.ownerDocument).toBe(other);
expect(read.call(foreign, ["display"])).toBeNull();
} finally {
canvas.remove();
}
});
it("keeps requests constant as ordinary ancestry grows", async () => {
const small = fixture(),
large = fixture();
large.topRows.splice(
1,
0,
...Array.from({ length: 100 }, (_, i) => row(100 + i, "div", rect(0, 0, 1200, 800))),
);
await captureVisualScreenshot(small.cdp, small.candidate);
await captureVisualScreenshot(large.cdp, large.candidate);
expect(large.send.mock.calls.map((c) => c[1])).toEqual(small.send.mock.calls.map((c) => c[1]));
});
it("preserves missing-path candidates but cannot execute them", async () => {
const f = fixture();
const { framePath: _, ...candidate } = f.candidate;
expect(await captureVisualScreenshot(f.cdp, candidate)).toMatchObject({
data: { reason: "visual_target_changed" },
});
expect(f.send).not.toHaveBeenCalled();
});
it.each([0.249, 0.25, 0.251])("compares rectangle edges with tolerance %s", async (delta) => {
const f = fixture();
f.topRows[0].box.x += delta;
const result = await captureVisualScreenshot(f.cdp, f.candidate);
expect("code" in result).toBe(delta > 0.25);
});
it("rejects a new clipping ancestor even if the resulting crop is identical", async () => {
const f = fixture();
const clip = row(8, "div", rect(0, 0, 1200, 800));
clip.styles["overflow-x"] = "hidden";
f.topRows.splice(1, 0, clip);
expect(await captureVisualScreenshot(f.cdp, f.candidate)).toMatchObject({
data: { reason: "visual_target_changed" },
});
expect(f.send.mock.calls.some((c) => c[1] === "Page.captureScreenshot")).toBe(false);
});
it("rejects changed owner and changed DOM before capture", async () => {
for (const kind of ["wrongOwner", "rootChanged"] as const) {
const f = fixture(true);
f.control[kind] = true;
expect(await captureVisualScreenshot(f.cdp, f.candidate)).toMatchObject({
data: { reason: "visual_target_changed" },
});
expect(f.send.mock.calls.some((c) => c[1] === "Page.captureScreenshot")).toBe(false);
}
});
it("releases retained objects on read failure and cancellation", async () => {
for (const kind of ["failRead", "abortRead"] as const) {
const f = fixture();
f.control[kind] = true;
const result = await captureVisualScreenshot(f.cdp, f.candidate, f.controller.signal);
expect(result).toMatchObject({ code: kind === "abortRead" ? "cancelled" : "cdp_failed" });
expect(f.send.mock.calls.at(-1)?.[1]).toBe("Runtime.releaseObjectGroup");
}
});
it("plans pixels without enlarging images and retries at most once", async () => {
expect(visualScreenshotScale(4096, 2048, 2)).toBe(0.25);
expect(visualScreenshotScale(4000, 4000, 1)).toBe(0.5);
const f = fixture();
f.control.shots = [png(3000, 3000), png(1800, 1800)];
expect(await captureVisualScreenshot(f.cdp, f.candidate)).toMatchObject({
width: 1800,
height: 1800,
});
const shots = f.send.mock.calls.filter((c) => c[1] === "Page.captureScreenshot");
expect(shots).toHaveLength(2);
expect((shots[1][2]!.clip as { scale: number }).scale).toBeLessThan(1);
const fail = fixture();
fail.control.shots = [png(3000, 3000)];
expect(await captureVisualScreenshot(fail.cdp, fail.candidate)).toMatchObject({
data: { reason: "visual_pixel_budget_exceeded" },
});
expect(fail.send.mock.calls.filter((c) => c[1] === "Page.captureScreenshot")).toHaveLength(2);
});
it("rejects invalid PNG without guessing dimensions", async () => {
const f = fixture();
f.control.shots = ["invalid"];
expect(await captureVisualScreenshot(f.cdp, f.candidate)).toMatchObject({
data: { reason: "screenshot_capture_failed" },
});
});
it("dispatches a visual ref through the screenshot tool and restores overlays", async () => {
const f = fixture();
const manager = new SessionManager({
agentWindow: {
create: async () => 100,
remove: async () => {},
ensureActiveTab: async () => 4,
},
});
const ctx = await manager.start("test");
ctx.refStore.replace([["e1", { kind: "visual-region", candidate: f.candidate }]]);
const tab = { id: 4, windowId: 100, active: true } as chrome.tabs.Tab;
const tabsApi = { get: async () => tab, query: async () => [tab] };
const sendToTab = vi.fn(async () => ({}));
const result = await handleScreenshot(
manager,
{ session_id: "test", ref: "e1" },
{ cdp: f.cdp, tabsApi, captureApi: { ...tabsApi, captureVisibleTab: vi.fn() }, sendToTab },
);
expect(result).toMatchObject({ width: 100, height: 40, format: "png" });
expect(sendToTab).toHaveBeenCalledTimes(2);
});
});
+1 -49
View File
@@ -2,14 +2,12 @@ import { OVERLAY_AUTOMATION_BYPASS } from "@/lib/overlay-bridge";
import type { SessionManager } from "@/session-manager/manager";
import type { Transport } from "@/transport/transport";
import type {
BlurParams,
ClickParams,
ConsoleParams,
DownloadParams,
EmulateParams,
EvaluateParams,
FillParams,
FocusParams,
GetHtmlParams,
HoverParams,
HoverResult,
@@ -29,7 +27,6 @@ import type {
ResponseFrame,
RpcError,
ScreenshotParams,
ScrollToParams,
SelectParams,
SnapshotParams,
UploadParams,
@@ -42,15 +39,7 @@ import { type EmulateCdpRunner, handleEmulate } from "./emulate";
import { classifyCdpError } from "./errors";
import { handleEvaluate } from "./evaluate";
import { handleRequestHelp } from "./human-loop";
import {
handleBlur,
handleClick,
handleFill,
handleFocus,
handleHover,
handlePress,
handleSelect,
} from "./interaction";
import { handleClick, handleFill, handleHover, handlePress, handleSelect } from "./interaction";
import {
handleNavigate,
handleNavigateBack,
@@ -72,7 +61,6 @@ import {
handleRecordStop,
type RecordRuntimeDeps,
} from "./record";
import { handleScrollTo } from "./scroll";
import {
handleSessionStart,
handleSessionStop,
@@ -527,39 +515,6 @@ export class ToolDispatcher {
);
return this.rememberHover((req.params as HoverParams).session_id, result);
}
case "tool.scroll_to":
return this.withHoverReleaseForRequest(
req.params as ScrollToParams,
() =>
handleScrollTo(
this.sessions,
req.params as ScrollToParams,
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
),
signal,
);
case "tool.focus":
return this.withHoverReleaseForRequest(
req.params as FocusParams,
() =>
handleFocus(
this.sessions,
req.params as FocusParams,
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
),
signal,
);
case "tool.blur":
return this.withHoverReleaseForRequest(
req.params as BlurParams,
() =>
handleBlur(
this.sessions,
req.params as BlurParams,
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
),
signal,
);
case "tool.fill":
return this.withHoverReleaseForRequest(
req.params as FillParams,
@@ -830,9 +785,6 @@ function sessionIdForBrowserControlMethod(req: RequestFrame): string | null {
case "tool.reload":
case "tool.click":
case "tool.hover":
case "tool.scroll_to":
case "tool.focus":
case "tool.blur":
case "tool.fill":
case "tool.press":
case "tool.select":
@@ -20,13 +20,8 @@ import type { CoordinateOwner, CssViewport, SnapshotProjectionResult } from "./c
import { readSnapshotOwnerSizes } from "./snapshot-owner-sizes";
export interface LayoutMetrics {
cssVisualViewport?: {
zoom?: number;
scale?: number;
clientWidth?: number;
clientHeight?: number;
};
visualViewport?: { zoom?: number; scale?: number; clientWidth?: number; clientHeight?: number };
cssVisualViewport?: { zoom?: number; clientWidth?: number; clientHeight?: number };
visualViewport?: { zoom?: number; clientWidth?: number; clientHeight?: number };
cssLayoutViewport?: {
clientWidth?: number;
clientHeight?: number;
@@ -212,16 +207,11 @@ export class GeometryContext {
ownerBackendNodeId: number,
ancestorClips: Polygon[],
viewport: Size,
contentSize?: Size,
): Promise<SnapshotProjectionResult> {
const key = `${cdpTargetKey(source.target)}:${ownerBackendNodeId}`;
let promise = this.snapshotOwners.get(key);
if (!promise) {
promise = contentSize
? this.ownerContent(source.target, ownerBackendNodeId).then((quad) =>
quad ? { quad, size: contentSize } : null,
)
: this.snapshotOwner(source.target, ownerBackendNodeId);
promise = this.snapshotOwner(source.target, ownerBackendNodeId);
this.snapshotOwners.set(key, promise);
}
let owner: { quad: Quad; size: Size } | null;
+2 -206
View File
@@ -1,4 +1,5 @@
// DOM interaction tools — click, hover, focus/blur, fill, press, and select.
// DOM interaction tools — `tool.click`, `tool.fill`, `tool.press`, and
// `tool.select`.
//
// All interaction tools:
// 1. Resolve target tab (sandbox: must be inside Agent Window).
@@ -13,14 +14,10 @@ import { ChromiumCdp } from "@/browser-driver/chromium-cdp";
import type { CdpTarget } from "@/browser-driver/frame-graph";
import type { SessionContext, SessionManager } from "@/session-manager/manager";
import type {
BlurParams,
BlurResult,
ClickParams,
ClickResult,
FillParams,
FillResult,
FocusParams,
FocusResult,
HoverParams,
HoverResult,
KeyModifier,
@@ -230,207 +227,6 @@ export async function resolveBackendNode(
}
}
// Check the target's own root, then its hosts: closed shadow roots are not
// reachable via host.shadowRoot. DOM focus remains meaningful in background tabs.
const FOCUS_CHECK = `function() {
if (!this.isConnected) return false;
let element = this;
while (element) {
const root = element.getRootNode();
if (root.activeElement !== element) return false;
element = root.host;
}
return true;
}`;
const BLUR_TARGET = `function() {
if (!this.isConnected) throw new Error('blur target is detached');
const wasFocused = (${FOCUS_CHECK}).call(this);
if (typeof this.blur !== 'function') return { ok: false, was_focused: wasFocused };
this.blur();
return { ok: true, was_focused: wasFocused };
}`;
interface FocusScriptReply<T> {
result?: { value?: T };
exceptionDetails?: { text?: string; exception?: { description?: string } };
}
function checkFocusAbort(signal: AbortSignal | undefined): void {
if (signal?.aborted) throw new DOMException("interaction aborted", "AbortError");
}
// Only focus/blur use this view. Guard every CDP boundary, including commands
// inside selector resolution and frame scrolling, without changing other tools.
function focusCdp(cdp: CdpRunner, signal: AbortSignal | undefined): CdpRunner {
const send = <T = unknown>(target: CdpTarget, method: string, params?: object): Promise<T> => {
checkFocusAbort(signal);
return cdpRunnerForTarget(cdp, target).send<T>(target.tabId, method, params);
};
return {
send: (tabId, method, params) => send({ tabId }, method, params),
sendToTarget: send,
trackSessionTab: cdp.trackSessionTab?.bind(cdp),
getFrameGraph: cdp.getFrameGraph
? (tabId) => {
checkFocusAbort(signal);
return cdp.getFrameGraph!(tabId);
}
: undefined,
};
}
// ---------------------------------------------------------------------------
// tool.focus / tool.blur
// ---------------------------------------------------------------------------
export function handleFocus(
manager: SessionManager,
params: FocusParams,
deps: InteractionDeps = getDefaultDeps(),
): Promise<FocusResult | RpcError> {
return changeFocus(manager, params, deps, "focus");
}
export function handleBlur(
manager: SessionManager,
params: BlurParams,
deps: InteractionDeps = getDefaultDeps(),
): Promise<BlurResult | RpcError> {
return changeFocus(manager, params, deps, "blur");
}
function changeFocus(
manager: SessionManager,
params: FocusParams,
deps: InteractionDeps,
action: "focus",
): Promise<FocusResult | RpcError>;
function changeFocus(
manager: SessionManager,
params: BlurParams,
deps: InteractionDeps,
action: "blur",
): Promise<BlurResult | RpcError>;
async function changeFocus(
manager: SessionManager,
params: FocusParams | BlurParams,
deps: InteractionDeps,
action: "focus" | "blur",
): Promise<FocusResult | BlurResult | RpcError> {
const ctx = lookupSession(manager, params, action);
if (isRpcError(ctx)) return ctx;
const cdp = focusCdp(deps.cdp, deps.signal);
let objectId: string | undefined;
let cleanupCdp: CdpRunner | undefined;
let tabId: number | undefined;
try {
checkFocusAbort(deps.signal);
const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi);
checkFocusAbort(deps.signal);
if (isRpcError(target)) return target;
const denied = enforceAgentWindow(ctx, target, action);
if (denied) return denied;
tabId = target.tabId;
const dialogCursor = markDialogCursor(deps.cdp, tabId);
const node = await resolveBackendNode(cdp, ctx, target, params, action);
checkFocusAbort(deps.signal);
if (isRpcError(node)) return node;
const nodeCdp = cdpRunnerForTarget(cdp, node.cdpTarget);
cleanupCdp = cdpRunnerForTarget(deps.cdp, node.cdpTarget);
deps.cdp.trackSessionTab?.(ctx.sessionId, tabId);
if (action === "focus") {
const scrollErr = await scrollElementAndFramesIntoView(
cdp,
tabId,
node.cdpTarget,
node.backendNodeId,
node.frameId,
);
checkFocusAbort(deps.signal);
if (scrollErr) return scrollErr;
}
const resolved = await backendNodeToObject(nodeCdp, tabId, node.backendNodeId);
// Keep the handle before checking cancellation so finally can release it.
if (!isRpcError(resolved)) objectId = resolved;
checkFocusAbort(deps.signal);
if (isRpcError(resolved)) return resolved;
const runScript = async <T>(functionDeclaration: string): Promise<T | undefined> => {
const reply = await nodeCdp.send<FocusScriptReply<T>>(
target.tabId,
"Runtime.callFunctionOn",
{
objectId,
functionDeclaration,
returnByValue: true,
},
);
checkFocusAbort(deps.signal);
if (reply.exceptionDetails) {
const details = reply.exceptionDetails;
throw new Error(
`${action} script failed: ${details.exception?.description ?? details.text ?? "unknown exception"}`,
);
}
return reply.result?.value;
};
let wasFocused: boolean | undefined;
if (action === "focus") {
await nodeCdp.send(tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
checkFocusAbort(deps.signal);
} else {
const mutation = await runScript<{ ok: boolean; was_focused: boolean }>(BLUR_TARGET);
if (typeof mutation?.ok !== "boolean" || typeof mutation.was_focused !== "boolean") {
throw new Error("blur script returned an unexpected result");
}
if (!mutation.ok) {
return { code: "invalid_params", message: "target element does not support blur()" };
}
wasFocused = mutation.was_focused;
}
// Use a separate call so microtasks from focus/blur handlers finish before
// we report success. In particular, a blur handler can restore focus.
const focused = await runScript<boolean>(FOCUS_CHECK);
if (typeof focused !== "boolean") {
throw new Error(`${action} verification returned an unexpected result`);
}
if (focused !== (action === "focus")) {
throw new Error(
action === "focus"
? "target element did not become focused"
: "target element remained focused after blur()",
);
}
return attachDialogs(deps.cdp, tabId, dialogCursor, {
tab_id: tabId,
used_ref: node.usedRef,
used_selector: node.usedSelector,
focused,
...(action === "blur" ? { was_focused: wasFocused! } : {}),
});
} catch (err) {
return (
throwIfAborted(deps.signal) ?? {
code: "cdp_failed",
message: err instanceof Error ? err.message : String(err),
}
);
} finally {
if (objectId !== undefined && cleanupCdp && tabId !== undefined) {
// Cleanup must also run after cancellation. Navigation may have already
// disposed the object, which must not replace the operation's result.
try {
await cleanupCdp.send(tabId, "Runtime.releaseObject", { objectId });
} catch {
// The target or execution context may no longer exist.
}
}
}
}
export async function resolveActionTarget(
cdp: CdpRunner,
ctx: SessionContext,
+33 -17
View File
@@ -1,8 +1,3 @@
import { parsePngDimensions } from "./png";
import { captureVisualScreenshot } from "./visual-screenshot";
export { parsePngDimensions } from "./png";
import type { CapturedSceneInput } from "./vom/facts";
// Observation handlers — `tool.snapshot`, `tool.get_html`, `tool.screenshot`,
// and semantic `tool.observe` (design §7). Each handler resolves the target
@@ -53,7 +48,7 @@ import {
normaliseRef as sharedNormaliseRef,
type ToolEffect,
} from "./shared";
import { lookupRefTarget, resolveSnapshotRef } from "./snapshot-ref";
import { resolveSnapshotRef } from "./snapshot-ref";
import { type CapturedNode, type CapturedSurfaceProbe, probeHoverSurfaces } from "./vom/capture";
import { captureObservationFacts, semanticCapture } from "./vom/capture-coordinator";
import type { FrameDocument as CapturedFrameDocument } from "./vom/frame-document";
@@ -106,6 +101,38 @@ export function stripDataUrlPrefix(dataUrl: string): string {
return m ? dataUrl.slice(m[0].length) : dataUrl;
}
/**
* Parse a PNG's IHDR chunk and return `(width, height)`. Returns
* `null` on any malformed input so callers fall back to `0/0` instead
* of throwing.
*
* PNG layout: 8-byte signature, then a 4-byte length, 4-byte type
* ("IHDR"), then the chunk data — width is bytes 16-19 BE, height is
* 20-23 BE.
*/
export function parsePngDimensions(base64: string): { width: number; height: number } | null {
try {
// atob is available in MV3 service workers.
const head = base64.length > 64 ? base64.slice(0, 64) : base64;
const bin = atob(head);
if (bin.length < 24) return null;
if (bin.charCodeAt(0) !== 0x89 || bin.charCodeAt(1) !== 0x50 || bin.charCodeAt(2) !== 0x4e) {
return null;
}
const u32 = (off: number) =>
(bin.charCodeAt(off) << 24) |
(bin.charCodeAt(off + 1) << 16) |
(bin.charCodeAt(off + 2) << 8) |
bin.charCodeAt(off + 3);
const width = u32(16) >>> 0;
const height = u32(20) >>> 0;
if (width === 0 || height === 0) return null;
return { width, height };
} catch {
return null;
}
}
export interface ScreenshotDeps {
cdp?: SharedCdpRunner;
tabsApi: ChromeTabsApi;
@@ -281,17 +308,6 @@ export async function handleScreenshot(
if (!deps.cdp) {
return { code: "cdp_failed", message: "screenshot ref capture requires CDP" };
}
const entry = lookupRefTarget(ctx, ref, target.tabId);
if (entry?.kind === "visual-region") {
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
const captured = await withExtensionOverlayHidden(
target.tabId,
() => captureVisualScreenshot(deps.cdp!, entry.candidate, signal),
deps.sendToTab,
);
if (isRpcError(captured)) return captured;
return withShotDialogs({ ...captured, format: "png", tab_id: target.tabId });
}
const node = resolveSnapshotRef(ctx, ref, target.tabId);
if (isRpcError(node)) return node;
if (signal?.aborted) return cancelled("screenshot");
-31
View File
@@ -1,31 +0,0 @@
/**
* Parse a PNG's IHDR chunk and return `(width, height)`. Returns
* `null` on any malformed input so callers fall back to `0/0` instead
* of throwing.
*
* PNG layout: 8-byte signature, then a 4-byte length, 4-byte type
* ("IHDR"), then the chunk data — width is bytes 16-19 BE, height is
* 20-23 BE.
*/
export function parsePngDimensions(base64: string): { width: number; height: number } | null {
try {
// atob is available in MV3 service workers.
const head = base64.length > 64 ? base64.slice(0, 64) : base64;
const bin = atob(head);
if (bin.length < 33) return null;
if (bin.slice(0, 8) !== "\x89PNG\r\n\x1a\n" || bin.slice(8, 16) !== "\x00\x00\x00\x0dIHDR") {
return null;
}
const u32 = (off: number) =>
(bin.charCodeAt(off) << 24) |
(bin.charCodeAt(off + 1) << 16) |
(bin.charCodeAt(off + 2) << 8) |
bin.charCodeAt(off + 3);
const width = u32(16) >>> 0;
const height = u32(20) >>> 0;
if (width === 0 || height === 0) return null;
return { width, height };
} catch {
return null;
}
}
@@ -1,142 +0,0 @@
import { cdpTargetKey } from "@/browser-driver/frame-graph";
import type { NodeAddress } from "./frame-geometry";
import {
clipPolygon,
projectRegionToViewport,
type Region,
rectPolygon,
regionBounds,
type ViewportRect,
} from "./geometry";
import { GeometryContext } from "./geometry/frame-context";
import { type CdpRunner, sendToCdpTarget } from "./shared";
// Ask the renderer to apply containing-block clips (including shadow DOM),
// instead of treating layout quads as visible pixels. Each document is measured
// locally; the existing frame projections supply the top-viewport coordinates.
// IntersectionObserver reports a bounding rectangle, not an occlusion/hit test.
const VISIBLE_RECT = `function(timeoutMs) {
const element = this;
const visible = () => element.isConnected && element.checkVisibility({
checkOpacity: true, checkVisibilityCSS: true, contentVisibilityAuto: true
});
if (typeof element.checkVisibility !== 'function' || !visible()) return null;
return new Promise((resolve, reject) => {
const observer = new IntersectionObserver(([entry]) => {
clearTimeout(timer);
observer.disconnect();
const rect = entry.intersectionRect;
resolve(visible() && entry.isIntersecting && rect.width > 0 && rect.height > 0
? { x: rect.x, y: rect.y, width: rect.width, height: rect.height }
: null);
}, { root: element.ownerDocument });
const timer = setTimeout(() => {
observer.disconnect();
reject(new Error('scroll-to visibility measurement timed out'));
}, timeoutMs);
observer.observe(element);
});
}`;
async function visibleRect(
cdp: CdpRunner,
address: NodeAddress,
deadline: number,
): Promise<ViewportRect | null> {
const resolved = await sendToCdpTarget<{ object?: { objectId?: string } }>(
cdp,
address.target,
"DOM.resolveNode",
{ backendNodeId: address.backendNodeId },
);
if (!resolved.object?.objectId) throw new Error("scroll-to could not resolve the target element");
const reply = await sendToCdpTarget<{
result?: { value?: ViewportRect | null };
exceptionDetails?: { text?: string; exception?: { description?: string } };
}>(cdp, address.target, "Runtime.callFunctionOn", {
objectId: resolved.object.objectId,
functionDeclaration: VISIBLE_RECT,
// Bound this read even if a background document stops rendering. Its
// observer always disconnects; cancellation cannot leave a live observer.
arguments: [{ value: Math.max(1, Math.min(1_000, deadline - Date.now())) }],
awaitPromise: true,
returnByValue: true,
});
if (reply.exceptionDetails) {
const details = reply.exceptionDetails;
throw new Error(details.exception?.description ?? details.text ?? "visibility script failed");
}
const rect = reply.result?.value;
if (rect === null) return null;
if (
!rect ||
![rect.x, rect.y, rect.width, rect.height].every(Number.isFinite) ||
rect.width <= 0 ||
rect.height <= 0
) {
throw new Error("scroll-to visibility measurement returned invalid bounds");
}
return rect;
}
async function projectRect(
context: GeometryContext,
address: NodeAddress,
rect: ViewportRect,
): Promise<Region> {
let region: Region = [rectPolygon({ x: rect.x, y: rect.y, w: rect.width, h: rect.height })];
if (!address.frameId) return region;
const frame = await context.frame(address.frameId);
if (!frame || cdpTargetKey(frame.target) !== cdpTargetKey(address.target)) {
throw new Error("scroll-to frame no longer belongs to its target");
}
const parent = frame.parentFrameId ? await context.frame(frame.parentFrameId) : undefined;
if (parent && cdpTargetKey(parent.target) === cdpTargetKey(frame.target)) {
// DOM rectangles are document-local. Same-target iframe quads are already
// target-local, so cross this one boundary before the target projection.
const viewport = await context.viewport(frame.target);
if (frame.ownerBackendNodeId === undefined || !viewport) {
throw new Error("scroll-to could not resolve the frame viewport");
}
const local = await context.snapshotProjection(address, frame.ownerBackendNodeId, [], viewport);
if (local.status !== "available") throw new Error("scroll-to could not project the frame");
region = projectRegionToViewport(region, local.projection.geometry);
}
const projection = await context.targetProjection(frame.frameId);
if (!projection) throw new Error("scroll-to could not project the target viewport");
return projectRegionToViewport(region, projection);
}
/** Only scroll-to needs renderer-clipped bounds; other interaction geometry is unchanged. */
export async function scrollVisibleBounds(
cdp: CdpRunner,
tabId: number,
address: NodeAddress,
deadline: number,
): Promise<ViewportRect | null> {
const context = new GeometryContext(cdp, tabId);
const rect = await visibleRect(cdp, address, deadline);
if (!rect) return null;
let region = await projectRect(context, address, rect);
if (address.frameId) {
const ancestry = await context.ancestry(address.frameId);
if (!ancestry) throw new Error("scroll-to could not resolve the frame ancestry");
for (const child of ancestry) {
const parent = child.parentFrameId ? await context.frame(child.parentFrameId) : undefined;
if (!parent || child.ownerBackendNodeId === undefined) {
throw new Error("scroll-to could not resolve the frame owner");
}
const owner = {
target: parent.target,
frameId: parent.frameId,
backendNodeId: child.ownerBackendNodeId,
};
// A visible child document can still have a hidden or clipped iframe.
const ownerRect = await visibleRect(cdp, owner, deadline);
if (!ownerRect) return null;
const clips = await projectRect(context, owner, ownerRect);
region = region.flatMap((polygon) => clips.map((clip) => clipPolygon(polygon, clip)));
}
}
return regionBounds(region);
}
-137
View File
@@ -1,137 +0,0 @@
import { ChromiumCdp } from "@/browser-driver/chromium-cdp";
import { type CdpTarget, cdpTargetKey } from "@/browser-driver/frame-graph";
import type { SessionManager } from "@/session-manager/manager";
import type { RpcError, ScrollToParams, ScrollToResult } from "@/transport/types";
import { attachDialogs, markDialogCursor } from "./dialogs";
import { cdpError, rpcError } from "./errors";
import { scrollElementAndFramesIntoView } from "./frame-geometry";
import { resolveBackendNode } from "./interaction";
import { scrollVisibleBounds } from "./scroll-visibility";
import {
type CdpRunner,
type ChromeTabsApi,
chromeTabsApi,
enforceAgentWindow,
isRpcError,
lookupSession,
resolveTargetTab,
sendToCdpTarget,
} from "./shared";
export interface ScrollToDeps {
cdp: CdpRunner;
tabsApi: ChromeTabsApi;
signal?: AbortSignal;
}
let defaultDeps: { cdp: ChromiumCdp; tabsApi: ChromeTabsApi } | null = null;
function getDefaultDeps(): { cdp: ChromiumCdp; tabsApi: ChromeTabsApi } {
if (!defaultDeps) defaultDeps = { cdp: new ChromiumCdp(), tabsApi: chromeTabsApi };
return defaultDeps;
}
export async function handleScrollTo(
manager: SessionManager,
params: ScrollToParams,
deps: ScrollToDeps = getDefaultDeps(),
): Promise<ScrollToResult | RpcError> {
const ctxOrErr = lookupSession(manager, params, "scroll-to");
if (isRpcError(ctxOrErr)) return ctxOrErr;
const ctx = ctxOrErr;
const timeout = params.timeout_ms ?? 30_000;
if (!Number.isInteger(timeout) || timeout <= 0) {
return { code: "invalid_params", message: "timeout_ms must be a positive integer" };
}
const deadline = Date.now() + timeout;
const checkActive = () => {
if (deps.signal?.aborted) throw new DOMException("scroll-to aborted", "AbortError");
if (Date.now() >= deadline) throw new DOMException("scroll-to timed out", "TimeoutError");
};
const objectGroup = `bsk-scroll-${crypto.randomUUID()}`;
const objectTargets = new Map<string, CdpTarget>();
const send = async <T>(target: CdpTarget, method: string, args?: object): Promise<T> => {
checkActive();
// Also own objects allocated by the shared scroll fallback. Record the
// target before awaiting so cancellation cannot skip their cleanup.
if (method === "DOM.resolveNode") objectTargets.set(cdpTargetKey(target), target);
const result = await sendToCdpTarget<T>(
deps.cdp,
target,
method,
method === "DOM.resolveNode" ? { ...args, objectGroup } : args,
);
checkActive();
return result;
};
let graph: ReturnType<NonNullable<CdpRunner["getFrameGraph"]>> | undefined;
const cdp: CdpRunner = {
send: (tabId, method, args) => send({ tabId }, method, args),
sendToTarget: send,
trackSessionTab: deps.cdp.trackSessionTab?.bind(deps.cdp),
getFrameGraph: deps.cdp.getFrameGraph
? async (tabId) => {
checkActive();
graph ??= deps.cdp.getFrameGraph!(tabId);
const result = await graph;
checkActive();
return result;
}
: undefined,
};
try {
checkActive();
const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi);
checkActive();
if (isRpcError(target)) return target;
const denied = enforceAgentWindow(ctx, target, "scroll-to");
if (denied) return denied;
const dialogCursor = markDialogCursor(deps.cdp, target.tabId);
const node = await resolveBackendNode(cdp, ctx, target, params, "scroll-to");
checkActive();
if (isRpcError(node)) return node;
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
const scrollError = await scrollElementAndFramesIntoView(
cdp,
target.tabId,
node.cdpTarget,
node.backendNodeId,
node.frameId,
);
checkActive();
if (scrollError) return scrollError;
const bounds = await scrollVisibleBounds(
cdp,
target.tabId,
{
target: node.cdpTarget,
backendNodeId: node.backendNodeId,
frameId: node.frameId,
},
deadline,
);
checkActive();
if (!bounds)
return rpcError(
"permission_denied",
"element_not_visible",
"scroll-to target has no visible area",
);
return attachDialogs(deps.cdp, target.tabId, dialogCursor, {
tab_id: target.tabId,
used_ref: node.usedRef,
used_selector: node.usedSelector,
...bounds,
});
} catch (error) {
if (deps.signal?.aborted) return { code: "cancelled", message: "scroll-to aborted" };
if (Date.now() >= deadline) return { code: "timeout", message: "scroll-to timed out" };
return cdpError(error);
} finally {
// Cleanup bypasses the cancellation guard and cannot replace the result.
for (const target of objectTargets.values()) {
await sendToCdpTarget(deps.cdp, target, "Runtime.releaseObjectGroup", { objectGroup }).catch(
() => {},
);
}
}
}
+6 -21
View File
@@ -3,7 +3,7 @@
// `ref_not_found` errors for hard-failure tool paths.
import type { SessionContext } from "@/session-manager/manager";
import { normaliseRef, type RefEntry } from "@/session-manager/ref-store";
import { normaliseRef } from "@/session-manager/ref-store";
import type { RpcError } from "@/transport/types";
import { rpcError } from "./errors";
@@ -14,22 +14,14 @@ export interface SnapshotRefLookup {
cdpSessionId?: string;
}
/** Typed lookup never turns a visual anchor into a DOM operation target. */
export function lookupRefTarget(
ctx: SessionContext,
refKey: string,
tabId: number,
): RefEntry | null {
function refEntryForTab(ctx: SessionContext, refKey: string, tabId: number) {
const entry = ctx.refStore.resolveEntry(refKey);
if (!entry) return null;
const ownerTabId =
entry.kind === "visual-region" ? entry.candidate.document.target.tabId : entry.tabId;
return ownerTabId === tabId ? entry : null;
return entry?.tabId === tabId ? entry : null;
}
/**
* Soft lookup: returns `null` when the ref is unknown or bound to a
* different tab, or is a visual region. Used by paths that report `matched: false` instead of
* different tab. Used by paths that report `matched: false` instead of
* emitting an RPC error (e.g. `tool.request_help`).
*/
export function lookupSnapshotRef(
@@ -38,8 +30,8 @@ export function lookupSnapshotRef(
tabId: number,
): SnapshotRefLookup | null {
const refKey = normaliseRef(ref);
const entry = lookupRefTarget(ctx, refKey, tabId);
if (!entry || entry.kind !== "dom") return null;
const entry = refEntryForTab(ctx, refKey, tabId);
if (!entry) return null;
return {
backendNodeId: entry.backendNodeId,
refKey,
@@ -58,13 +50,6 @@ export function resolveSnapshotRef(
ref: string,
tabId: number,
): SnapshotRefLookup | RpcError {
const entry = lookupRefTarget(ctx, ref, tabId);
if (entry?.kind === "visual-region")
return rpcError(
"unsupported",
"ref_kind_unsupported",
`ref ${ref} is a visual region, not a DOM operation target`,
);
const looked = lookupSnapshotRef(ctx, ref, tabId);
if (looked === null) {
return rpcError(
@@ -1,382 +0,0 @@
import type { CdpFrame } from "@/browser-driver/frame-graph";
import type { RpcError } from "@/transport/types";
import { rpcError } from "./errors";
import {
type GeometryProjection,
projectRectToViewport,
type Size,
type ViewportRect,
} from "./geometry";
import { type CssViewport, screenshotPageRect } from "./geometry/coordinate-types";
import { cssViewport, GeometryContext } from "./geometry/frame-context";
import { parsePngDimensions } from "./png";
import { type CdpRunner, sendToCdpTarget } from "./shared";
import { isAbortError, throwIfAborted } from "./vom/capture-abort";
import { resolveVerifiedNode } from "./vom/document-identity";
import type { DocumentIdentity } from "./vom/facts";
import { VISUAL_STYLES } from "./vom/snapshot";
import type { VisualCandidate, VisualFramePath } from "./vom/visual-discovery";
import {
EMPTY_VISUAL_CONTEXT,
extendVisualContext,
projectVisualBox,
resolveVisualRegion,
type VisualClipSource,
type VisualContext,
visualProjectionIssue,
} from "./vom/visual-region";
interface LiveRow {
node: number;
tag: string;
box: ViewportRect;
client: ViewportRect;
contentSize: Size;
styles: Record<string, string>;
}
interface LiveFrame {
top: boolean;
dpr: number;
rows: LiveRow[]; // Anchor first, root last.
}
// Read only one current ancestry, including shadow hosts, in the verified isolated world.
const READ_ANCESTRY = `function(styleNames) {
if (!this.isConnected || this.ownerDocument !== document) return null;
const rows = [];
for (let node = this; node; node = node.parentElement || node.getRootNode().host) {
if (!(node instanceof Element)) return null;
const s = getComputedStyle(node), r = node.getBoundingClientRect();
rows.push({ node, tag: node.localName,
box: { x:r.x, y:r.y, width:r.width, height:r.height },
client: { x:r.x+node.clientLeft, y:r.y+node.clientTop, width:node.clientWidth, height:node.clientHeight },
contentSize: { width:node.clientWidth-parseFloat(s.paddingLeft)-parseFloat(s.paddingRight), height:node.clientHeight-parseFloat(s.paddingTop)-parseFloat(s.paddingBottom) },
styles: Object.fromEntries(styleNames.map(key => [key,s.getPropertyValue(key)])) });
}
return { top: window === window.top, dpr: devicePixelRatio, rows };
}`;
interface DeepValue {
type: string;
value?: unknown;
}
/** This read returns JSON primitives plus DOM nodes (backend IDs), never remote object handles. */
function decode(value: DeepValue): unknown {
if (value.type === "node") return (value.value as { backendNodeId?: number })?.backendNodeId;
if (value.type === "array") return (value.value as DeepValue[]).map(decode);
if (value.type === "object")
return Object.fromEntries(
(value.value as [string, DeepValue][]).map(([key, item]) => [key, decode(item)]),
);
if (value.type === "null") return null;
if (["string", "number", "boolean"].includes(value.type)) return value.value;
throw new Error("unexpected live ancestry serialization");
}
function stale(message: string): RpcError {
return rpcError(
"not_found",
"visual_target_changed",
`${message}; observe again before requesting a screenshot`,
);
}
function sameIdentity(a: DocumentIdentity, b: DocumentIdentity): boolean {
return (
a.attachmentId === b.attachmentId &&
a.frameId === b.frameId &&
a.target.tabId === b.target.tabId &&
a.target.sessionId === b.target.sessionId &&
a.documentElementBackendNodeId === b.documentElementBackendNodeId
);
}
function closeRect(a: ViewportRect, b: ViewportRect): boolean {
return [
a.x - b.x,
a.y - b.y,
a.x + a.width - b.x - b.width,
a.y + a.height - b.y - b.height,
].every((delta) => Number.isFinite(delta) && Math.abs(delta) <= 0.25);
}
function sameClips(a?: VisualClipSource, b?: VisualClipSource): boolean {
while (a && b) {
if (
!sameIdentity(a.document, b.document) ||
a.backendNodeId !== b.backendNodeId ||
a.x !== b.x ||
a.y !== b.y ||
a.overflowX !== b.overflowX ||
a.overflowY !== b.overflowY ||
!closeRect(a.box, b.box)
)
return false;
a = a.parent;
b = b.parent;
}
return !a && !b;
}
/** Scale affects raster size only, never the selected CSS region. */
export function visualScreenshotScale(width: number, height: number, pixelsPerCss: number): number {
if (![width, height, pixelsPerCss].every((n) => Number.isFinite(n) && n > 0))
throw new Error("invalid screenshot pixel dimensions");
return Math.min(
1,
2048 / (Math.max(width, height) * pixelsPerCss),
Math.sqrt(4_000_000 / (width * height * pixelsPerCss * pixelsPerCss)),
);
}
async function readFrame(
cdp: CdpRunner,
document: DocumentIdentity,
anchor: number,
signal?: AbortSignal,
): Promise<LiveFrame | RpcError> {
const verified = await resolveVerifiedNode(cdp, document, anchor, signal);
if (verified.status !== "current") return stale(`visual DOM identity ${verified.status}`);
try {
throwIfAborted(signal);
const reply = await sendToCdpTarget<{
result?: { deepSerializedValue?: DeepValue };
exceptionDetails?: unknown;
}>(cdp, document.target, "Runtime.callFunctionOn", {
objectId: verified.objectId,
objectGroup: verified.objectGroup,
functionDeclaration: READ_ANCESTRY,
arguments: [{ value: VISUAL_STYLES }],
serializationOptions: {
serialization: "deep",
additionalParameters: { maxNodeDepth: 0, includeShadowTree: "none" },
},
});
throwIfAborted(signal);
if (reply.exceptionDetails || !reply.result?.deepSerializedValue)
return stale("visual ancestry unavailable");
const result = decode(reply.result.deepSerializedValue) as LiveFrame | null;
if (
!result?.rows?.length ||
result.rows[0].node !== anchor ||
result.rows.at(-1)?.node !== document.documentElementBackendNodeId ||
!result.rows.every((row) => Number.isSafeInteger(row.node) && row.node > 0)
)
return stale("visual ancestry incomplete");
if (cdp.getAttachmentId?.(document.target.tabId) !== document.attachmentId)
return stale("visual attachment changed");
return result;
} finally {
await sendToCdpTarget(cdp, document.target, "Runtime.releaseObjectGroup", {
objectGroup: verified.objectGroup,
}).catch(() => {});
throwIfAborted(signal);
}
}
/** All live reads for one screenshot use this one measurement context. */
async function resolveVisualRegionNow(
cdp: CdpRunner,
candidate: VisualCandidate,
signal?: AbortSignal,
): Promise<{ crop: ViewportRect; viewport: CssViewport; dpr: number } | RpcError> {
throwIfAborted(signal);
if (!candidate.framePath || !sameIdentity(candidate.document, candidate.framePath.document))
return stale("visual frame path unavailable");
const path: { frame: VisualFramePath; anchor: number }[] = [];
const seen = new Set<string>();
let frame: VisualFramePath | undefined = candidate.framePath;
let anchor = candidate.backendNodeId;
while (frame) {
if (
seen.has(frame.document.frameId) ||
frame.document.target.tabId !== candidate.document.target.tabId ||
cdp.getAttachmentId?.(frame.document.target.tabId) !== frame.document.attachmentId
)
return stale("invalid visual frame path");
seen.add(frame.document.frameId);
path.push({ frame, anchor });
anchor = frame.parent?.ownerBackendNodeId ?? 0;
frame = frame.parent?.frame;
}
path.reverse();
const frames: CdpFrame[] = path.map(({ frame }) => ({
frameId: frame.document.frameId,
target: frame.document.target,
...(frame.parent
? {
parentFrameId: frame.parent.frame.document.frameId,
ownerBackendNodeId: frame.parent.ownerBackendNodeId,
}
: {}),
}));
// Validate each recorded edge instead of asking the driver to fill all page owners.
for (const { frame } of path) {
if (!frame.parent) continue;
throwIfAborted(signal);
const owner = await sendToCdpTarget<{ backendNodeId?: number }>(
cdp,
frame.parent.frame.document.target,
"DOM.getFrameOwner",
{ frameId: frame.document.frameId },
);
if (owner.backendNodeId !== frame.parent.ownerBackendNodeId)
return stale("visual frame owner changed");
}
const geometry = new GeometryContext(
cdp,
candidate.document.target.tabId,
{ rootFrameId: frames[0].frameId, frames },
signal,
);
let context: VisualContext = EMPTY_VISUAL_CONTEXT;
let parentRead: LiveFrame | undefined;
let finalRegion: VisualCandidate["region"] | undefined;
let topDpr = 0;
for (let i = 0; i < path.length; i++) {
const { frame, anchor } = path[i];
const live = await readFrame(cdp, frame.document, anchor, signal);
if ("code" in live) return live;
if (live.top !== (i === 0)) return stale("visual root frame changed");
if (i === 0) topDpr = live.dpr;
const metrics = await geometry.layoutMetrics(frame.document.target);
const viewport = cssViewport(metrics);
const projections: GeometryProjection[] = [];
if (frame.parent) {
const parent = frame.parent.frame.document;
const size = parentRead!.rows[0].contentSize;
if (![size.width, size.height].every((n) => Number.isFinite(n) && n > 0))
return stale("iframe content size unavailable");
const parentViewport = await geometry.viewport(parent.target);
if (!parentViewport) return stale("parent viewport unavailable");
const local = await geometry.snapshotProjection(
{ target: parent.target, frameId: frame.document.frameId },
frame.parent.ownerBackendNodeId,
[],
parentViewport,
size,
);
const outer = await geometry.targetProjection(parent.frameId);
if (local.status !== "available" || !outer)
return stale("visual frame projection unavailable");
projections.push(local.projection.geometry, outer);
} else projections.push({ sourceClips: [], edges: [], topViewport: viewport });
const issue = visualProjectionIssue({
projections,
coordinates: { layoutUnitsPerCssPixel: 1, scrollCss: { x: 0, y: 0 } },
pageScale: metrics.cssVisualViewport?.scale ?? metrics.visualViewport?.scale,
});
if (issue) return stale(issue);
for (let j = live.rows.length - 1; j >= 0; j--) {
const row = live.rows[j];
const isAnchor = j === 0;
const node = {
document: frame.document,
backendNodeId: row.node,
styles: row.styles,
clientBox: projectVisualBox(row.client, projections),
};
context = extendVisualContext(
context,
node,
!isAnchor && row.tag !== "iframe" && row.tag !== "frame",
);
}
if (i < path.length - 1) {
const owner = live.rows[0];
context = {
...context,
hidden:
context.hidden ||
owner.styles.visibility === "hidden" ||
owner.styles.visibility === "collapse",
transformed: false,
localClip: false,
unpositionedClip: false,
};
} else {
const row = live.rows[0];
if (row.tag !== "canvas") return stale("visual anchor is no longer Canvas");
let visible: ViewportRect | null = row.box;
for (const projection of projections)
visible = visible
? projectRectToViewport(
{ x: visible.x, y: visible.y, w: visible.width, h: visible.height },
projection,
)
: null;
const region = resolveVisualRegion({
borderBox: projectVisualBox(row.box, projections),
frameVisibleBox: visible,
context,
visibility: row.styles.visibility,
});
if (region.status !== "available")
return stale(
`visual region ${region.status}${region.status === "unavailable" ? `: ${region.reason}` : ""}`,
);
finalRegion = region;
}
parentRead = live;
}
if (
!finalRegion ||
!closeRect(candidate.region.borderBox, finalRegion.borderBox) ||
!closeRect(candidate.region.crop, finalRegion.crop) ||
!sameClips(candidate.region.clips, finalRegion.clips)
)
return stale("visual region changed");
const viewport = cssViewport(await geometry.layoutMetrics(frames[0].target));
return { crop: finalRegion.crop, viewport, dpr: topDpr };
}
/** One target only. No frame discovery, scrolling, AX, or snapshot recapture. */
export async function captureVisualScreenshot(
cdp: CdpRunner,
candidate: VisualCandidate,
signal?: AbortSignal,
): Promise<{ image_base64: string; width: number; height: number } | RpcError> {
try {
const region = await resolveVisualRegionNow(cdp, candidate, signal);
if ("code" in region) return region;
const clip = screenshotPageRect(region.crop, region.viewport);
if (!clip) return stale("invalid screenshot coordinates");
let scale = visualScreenshotScale(region.crop.width, region.crop.height, region.dpr);
for (let attempt = 0; attempt < 2; attempt++) {
throwIfAborted(signal);
if (
cdp.getAttachmentId?.(candidate.document.target.tabId) !== candidate.document.attachmentId
)
return stale("visual attachment changed");
const shot = await cdp.send<{ data?: string }>(
candidate.document.target.tabId,
"Page.captureScreenshot",
{ format: "png", captureBeyondViewport: false, clip: { ...clip.rect, scale } },
);
throwIfAborted(signal);
const dims = shot.data ? parsePngDimensions(shot.data) : null;
if (!dims)
return rpcError(
"cdp_failed",
"screenshot_capture_failed",
"visual screenshot returned invalid PNG dimensions",
);
const correction = visualScreenshotScale(dims.width, dims.height, 1);
if (correction === 1) return { image_base64: shot.data!, ...dims };
scale *= correction * 0.99;
}
return rpcError(
"cdp_failed",
"visual_pixel_budget_exceeded",
"visual screenshot exceeds the pixel budget",
);
} catch (error) {
if (isAbortError(error) || signal?.aborted)
return { code: "cancelled", message: "visual screenshot aborted" };
return rpcError(
"cdp_failed",
"screenshot_capture_failed",
error instanceof Error ? error.message : String(error),
);
}
}
@@ -5,15 +5,12 @@ import { OVERLAY_HOST_MARKER_ATTR } from "@/lib/overlay-bridge";
import { captureVomObservation } from "../../observation";
import type { CdpRunner } from "../../shared";
import { captureObservationFacts, semanticCapture } from "../capture-coordinator";
import { buildSemanticGraph, buildSemanticVomScene } from "../semantic-graph";
import { REQUESTED_STYLES, type SnapshotReply, VISUAL_STYLES } from "../snapshot";
import { deduplicateVisualCandidates } from "../visual-dedup";
import { discoverVisualCandidates } from "../visual-discovery";
import { buildSemanticGraph } from "../semantic-graph/build";
import { REQUESTED_STYLES, type SnapshotReply } from "../snapshot";
function fixture(
options: {
frames?: CdpFrame[];
canvas?: boolean;
after?: Record<string, { element?: number; missing?: boolean; unreadable?: boolean }>;
fail?: string;
missingIdentity?: boolean;
@@ -81,27 +78,23 @@ function fixture(
if (method === "Page.getLayoutMetrics")
result = {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000, scale: 1 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 },
};
if (method === "DOMSnapshot.captureSnapshot") {
snapshots++;
const requested = (params as { computedStyles: readonly string[] }).computedStyles;
expect([REQUESTED_STYLES, VISUAL_STYLES]).toContainEqual(requested);
expect((params as { computedStyles: unknown }).computedStyles).toEqual(REQUESTED_STYLES);
result = {
strings: [
"#document",
"html",
options.canvas ? "canvas" : "button",
"button",
OVERLAY_HOST_MARKER_ATTR,
"",
"visible",
"1",
"static",
"auto",
"block",
"none",
"0px",
],
documents: frames
.filter(
@@ -151,18 +144,10 @@ function fixture(
[0, 0, 1000, 800],
[10, 20, 100, 40],
],
clientRects: [
[0, 0, 1000, 800],
[0, 0, 100, 40],
styles: [
[7, 8, 8, 5, 6],
[7, 8, 8, 5, 6],
],
styles: Array.from({ length: 2 }, () =>
requested.map(
(name) =>
[7, 8, 8, 5, 6, 9, 5, 5, 10, 6, 10, 10, 10, 10, 10, 8, 10, 11][
VISUAL_STYLES.indexOf(name as (typeof VISUAL_STYLES)[number])
],
),
),
},
};
}),
@@ -208,7 +193,7 @@ describe("captureObservationFacts", () => {
if (method === "Page.getLayoutMetrics")
return {
visualViewport: { clientWidth: 2000 },
cssVisualViewport: { clientWidth: 1000, scale: 1 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 999, pageY: 999 },
} as never;
const reply = await original(target, method, params);
@@ -674,7 +659,7 @@ describe("OOPIF capture", () => {
if (method === "Page.getLayoutMetrics") {
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000, scale: 1 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 300, clientHeight: 200, pageX: 0, pageY: 0 },
};
}
@@ -700,7 +685,7 @@ describe("OOPIF capture", () => {
if (method === "Page.getLayoutMetrics") {
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000, scale: 1 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 },
};
}
@@ -782,7 +767,7 @@ describe("OOPIF capture", () => {
if (method === "Page.getLayoutMetrics")
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000, scale: 1 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 300, clientHeight: 200 },
};
if (method === "DOMSnapshot.captureSnapshot")
@@ -844,7 +829,7 @@ describe("OOPIF capture", () => {
if (method === "Page.getLayoutMetrics") {
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000, scale: 1 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 },
};
}
@@ -934,7 +919,7 @@ function siblingCaptureFixture(
if (method === "Page.getLayoutMetrics")
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000, scale: 1 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 },
};
if (method === "DOM.getBoxModel")
@@ -1142,7 +1127,7 @@ describe("snapshot document provenance", () => {
: method === "Page.getLayoutMetrics"
? {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000, scale: 1 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 200, clientHeight: 100 },
}
: {}) as T,
@@ -1378,101 +1363,3 @@ describe("AX frame scheduling", () => {
expect(active).toBe(0);
});
});
describe("visual facts integration", () => {
it("defaults to base facts across targets and shares semantic behavior with visual capture", async () => {
const base = fixture({ canvas: true });
const visual = fixture({ canvas: true });
const baseFacts = await captureObservationFacts(base.cdp, 4);
const visualFacts = await captureObservationFacts(visual.cdp, 4, undefined, undefined, {
includeVisualFacts: true,
});
expect(baseFacts.visualFactsCollected).toBe(false);
expect(visualFacts.visualFactsCollected).toBe(true);
expect(baseFacts.documents).toHaveLength(4);
expect(visualFacts.documents).toHaveLength(4);
for (const doc of baseFacts.documents) {
expect(doc.geometry).toBeUndefined();
expect(doc.index.ancestryComplete).toBeUndefined();
for (const node of doc.index.nodes.values()) {
expect(node.parentMissing).toBeUndefined();
expect(node.layout?.clientRect).toBeUndefined();
if (node.layout) expect(Object.keys(node.layout.styles)).toEqual([...REQUESTED_STYLES]);
}
}
// This fixture has no child frame projection measurements. Enabling visual
// collection must preserve that missing evidence, not invent geometry.
expect(
visualFacts.documents.find((doc) => doc.frame.frameId === "main")?.geometry,
).toBeDefined();
expect(visualFacts.documents.filter((doc) => doc.geometry)).toHaveLength(1);
for (const doc of visualFacts.documents) {
expect(doc.index.ancestryComplete).toBeDefined();
for (const node of doc.index.nodes.values())
if (node.layout) {
expect(Object.keys(node.layout.styles)).toEqual([...VISUAL_STYLES]);
expect(Object.keys(node.layout.styles).some((key) => key.includes("radius"))).toBe(false);
}
}
for (const [capture, expected] of [
[base, REQUESTED_STYLES],
[visual, VISUAL_STYLES],
] as const) {
const calls = capture.logs.filter((c) => c.method === "DOMSnapshot.captureSnapshot");
expect(calls).toHaveLength(2);
for (const call of calls) expect(call.params.computedStyles).toEqual(expected);
}
expect(base.logs.map((c) => [c.target, c.method])).toEqual(
visual.logs.map((c) => [c.target, c.method]),
);
const scene = (input: typeof baseFacts) =>
buildSemanticVomScene({
documents: semanticCapture(input).documents,
viewport: input.viewport,
rootFrameId: input.rootFrameId,
excludedBackendNodeIds: semanticCapture(input).captured.excludedBackendNodeIds,
});
expect(scene(baseFacts)).toEqual(scene(visualFacts));
const before = base.logs.length;
expect(await discoverVisualCandidates(baseFacts)).toMatchObject({
complete: false,
candidates: [],
issues: [{ reason: "visual-facts-not-collected" }],
});
expect(base.logs).toHaveLength(before);
});
it("passes actual snapshot styles, client units, projection and identity to pure discovery without additional CDP", async () => {
const { cdp, logs } = fixture({
canvas: true,
frames: [{ frameId: "main", target: { tabId: 4 } }],
});
const facts = await captureObservationFacts(cdp, 4, undefined, undefined, {
includeVisualFacts: true,
});
const before = logs.length;
const result = await discoverVisualCandidates(facts);
expect(logs).toHaveLength(before);
expect(logs.filter((call) => call.method === "DOMSnapshot.captureSnapshot")).toHaveLength(1);
expect(result.complete).toBe(true);
expect(result.candidates).toHaveLength(1);
expect(result.candidates[0]).toMatchObject({
document: { frameId: "main", documentElementBackendNodeId: 1 },
region: { crop: { x: 10, y: 20, width: 100, height: 40 } },
});
const deduplicated = await deduplicateVisualCandidates(result);
expect(deduplicated.candidates).toEqual(result.candidates);
expect(deduplicated.candidates[0]).toBe(result.candidates[0]);
expect(deduplicated).toMatchObject({
candidateCount: 1,
deduplicatedCount: 0,
dedupDegraded: false,
});
expect(logs).toHaveLength(before);
expect(facts.documents[0].index.nodes.get(2)?.layout?.clientRect).toEqual([0, 0, 100, 40]);
expect(facts.documents[0].geometry?.pageScale).toBe(1);
const baseline = fixture({ frames: [{ frameId: "main", target: { tabId: 4 } }] });
await captureObservationFacts(baseline.cdp, 4);
expect(logs.map((call) => call.method)).toEqual(baseline.logs.map((call) => call.method));
});
});
@@ -1,151 +0,0 @@
import { describe, expect, it, vi } from "vitest";
import type { CdpRunner } from "../../shared";
import { verifyDocumentIdentity, verifyVisualTargetIdentity } from "../document-identity";
import type { VisualCandidate } from "../visual-discovery";
const candidate: VisualCandidate = {
document: {
attachmentId: "attached",
target: { tabId: 4, sessionId: "child-session" },
frameId: "child-frame",
documentElementBackendNodeId: 10,
},
backendNodeId: 20,
parentBackendNodeId: 10,
region: {
status: "available",
borderBox: { x: 0, y: 0, width: 10, height: 10 },
crop: { x: 0, y: 0, width: 10, height: 10 },
},
};
function fixture(
options: {
connected?: boolean;
sameDocument?: boolean;
root?: number;
missing?: boolean;
fail?: string;
abortAt?: string;
detachAt?: string;
} = {},
) {
const controller = new AbortController();
let attachment = "attached";
const send = vi.fn(
async (_tab: number, _session: string, method: string, params: Record<string, unknown>) => {
if (options.abortAt === method) controller.abort();
if (options.detachAt === method) attachment = "new-attachment";
if (options.fail === method) throw new Error("CDP failed");
if (method === "Page.createIsolatedWorld") return { executionContextId: 7 };
if (method === "DOM.resolveNode")
return options.missing ? {} : { object: { objectId: "anchor" } };
if (method === "Runtime.evaluate" || method === "Runtime.callFunctionOn") {
let root: unknown = {};
if (method === "Runtime.callFunctionOn") {
const document = { documentElement: {} };
// Exercise the actual predicate, including connection and owner checks.
root = new Function("document", `return (${params.functionDeclaration}).call(this)`).call(
{
isConnected: options.connected ?? true,
ownerDocument: options.sameDocument === false ? {} : document,
},
document,
);
}
return {
result: {
deepSerializedValue:
root === null
? { type: "null" }
: { type: "node", value: { backendNodeId: options.root ?? 10 } },
},
};
}
if (method === "Runtime.releaseObjectGroup") return {};
throw new Error(`unexpected ${method}`);
},
);
const cdp = {
send: vi.fn(),
sendToTarget: (
target: { tabId: number; sessionId?: string },
method: string,
params: Record<string, unknown>,
) => send(target.tabId, target.sessionId!, method, params),
getAttachmentId: () => attachment,
} as unknown as CdpRunner;
return { cdp, send, controller };
}
describe("visual target identity", () => {
it("uses the exact frame and target with bounded local reads and releases objects", async () => {
const { cdp, send } = fixture();
expect(await verifyVisualTargetIdentity(cdp, candidate)).toBe("current");
expect(send.mock.calls.map((c) => c[2])).toEqual([
"Page.createIsolatedWorld",
"DOM.resolveNode",
"Runtime.callFunctionOn",
"Runtime.releaseObjectGroup",
]);
expect(
send.mock.calls.every(([tab, session]) => tab === 4 && session === "child-session"),
).toBe(true);
expect(send.mock.calls[0][3]).toMatchObject({ frameId: "child-frame" });
expect(send.mock.calls[1][3]).toMatchObject({ backendNodeId: 20, executionContextId: 7 });
expect(send.mock.calls[2][3]).toMatchObject({ objectId: "anchor" });
expect(send.mock.calls[3][3].objectGroup).toBe(send.mock.calls[1][3].objectGroup);
});
it.each([
{ connected: false },
{ sameDocument: false },
{ root: 11 },
])("rejects detached/adopted/replaced DOM: %j", async (options) => {
const { cdp } = fixture(options);
expect(await verifyVisualTargetIdentity(cdp, candidate)).toBe("changed");
});
it("does not add anchor reads to existing capture identity verification", async () => {
const { cdp, send } = fixture();
expect(await verifyDocumentIdentity(cdp, candidate.document)).toBe("current");
expect(send.mock.calls.map((c) => c[2])).toEqual([
"Page.createIsolatedWorld",
"Runtime.evaluate",
"Runtime.releaseObjectGroup",
]);
});
it.each([
"Page.createIsolatedWorld",
"DOM.resolveNode",
"Runtime.callFunctionOn",
])("fails closed and cleans up when %s fails", async (fail) => {
const { cdp, send } = fixture({ fail });
expect(await verifyVisualTargetIdentity(cdp, candidate)).toBe("unavailable");
expect(send.mock.calls.at(-1)?.[2]).toBe("Runtime.releaseObjectGroup");
});
it("rejects attachment changes before and during verification", async () => {
const first = fixture();
expect(
await verifyVisualTargetIdentity({ ...first.cdp, getAttachmentId: () => "other" }, candidate),
).toBe("changed");
expect(first.send).not.toHaveBeenCalled();
const second = fixture({ detachAt: "Runtime.callFunctionOn" });
expect(await verifyVisualTargetIdentity(second.cdp, candidate)).toBe("changed");
const missing = fixture({ missing: true });
expect(await verifyVisualTargetIdentity(missing.cdp, candidate)).toBe("unavailable");
});
it.each([
"DOM.resolveNode",
"Runtime.callFunctionOn",
])("propagates cancellation at %s after releasing the object group", async (abortAt) => {
const { cdp, send, controller } = fixture({ abortAt });
await expect(
verifyVisualTargetIdentity(cdp, candidate, controller.signal),
).rejects.toMatchObject({ name: "AbortError" });
expect(send.mock.calls.at(-1)?.[2]).toBe("Runtime.releaseObjectGroup");
});
});
@@ -2,7 +2,7 @@ import { describe, expect, it, vi } from "vitest";
import { OVERLAY_HOST_MARKER_ATTR } from "@/lib/overlay-bridge";
import { createCaptureCheckpoint } from "../capture-abort";
import { buildDocumentIndex, type DecodedNode } from "../facts";
import { decodeDocument, REQUESTED_STYLES, VISUAL_SNAPSHOT } from "../snapshot";
import { decodeDocument, REQUESTED_STYLES } from "../snapshot";
function node(id: number, parent: number | null): DecodedNode {
return {
@@ -125,44 +125,3 @@ describe("document facts", () => {
expect(reads).toBeLessThanOrEqual(356);
});
});
describe("visual ancestry facts", () => {
it("distinguishes document roots, malformed roots, missing parents and cycles", async () => {
const input = [
{ ...node(0, null), nodeType: 9 },
{ ...node(1, 0), nodeType: 1 },
{ ...node(2, 1), nodeType: 11 },
node(3, 2),
node(4, 99),
node(5, 6),
node(6, 5),
node(7, 5),
{ ...node(8, null), nodeType: 1 },
{ ...node(9, null), nodeType: 9, parentMissing: true },
];
const index = await buildDocumentIndex(input.reverse(), undefined, true);
for (const id of [0, 1, 2, 3]) expect(index.ancestryComplete!.get(id)).toBe(true);
for (const id of [4, 5, 6, 7, 8, 9]) expect(index.ancestryComplete!.get(id)).toBe(false);
});
it("retains client offsets in their source units and does not disguise missing parent indices", async () => {
const decoded = await decodeDocument(
{
nodes: { backendNodeId: [1, 2], nodeType: [1, 1], nodeName: [0, 0], parentIndex: [99] },
layout: {
nodeIndex: [0],
bounds: [[745, 455.625, 97, 59.5]],
clientRects: [[5, 5, 68, 38]],
},
},
["div"],
undefined,
VISUAL_SNAPSHOT,
);
expect(decoded.nodes[0].layout?.clientRect).toEqual([5, 5, 68, 38]);
expect(decoded.nodes[0].layout?.bounds).toEqual([745, 455.625, 97, 59.5]);
expect(decoded.nodes.every((node) => node.parentMissing)).toBe(true);
const index = await buildDocumentIndex(decoded.nodes, undefined, true);
expect([...index.ancestryComplete!.values()]).toEqual([false, false]);
});
});
@@ -1,147 +0,0 @@
import { describe, expect, it } from "vitest";
import { deduplicateVisualCandidates, MAX_VISUAL_DEDUP_KEYS } from "../visual-dedup";
import type { VisualCandidate, VisualDiscoveryResult } from "../visual-discovery";
function candidate(id: number, width = 100, parent = 1): VisualCandidate {
const crop = { x: 0, y: 0, width, height: 100 };
return {
document: {
attachmentId: "a",
target: { tabId: 1 },
frameId: "main",
documentElementBackendNodeId: 1,
},
backendNodeId: id,
parentBackendNodeId: parent,
region: { status: "available", borderBox: crop, crop },
};
}
function discovery(candidates: VisualCandidate[]): VisualDiscoveryResult {
return { candidates, complete: true, issues: [], captureIssues: [] };
}
describe("visual candidate deduplication", () => {
it("preserves small, unnamed and labelled candidates and their original evidence", async () => {
const small = { ...candidate(3, 1), label: ' A "label"\n😀' };
const large = candidate(2, 1000);
const input = discovery([small, large]);
Object.freeze(input.candidates);
Object.freeze(small);
Object.freeze(large);
for (let i = 0; i < 2; i++) {
const result = await deduplicateVisualCandidates(input);
expect(result.candidates).toEqual([small, large]);
expect(result.candidates[0]).toBe(small);
expect(result.candidates[1]).toBe(large);
expect(result).toMatchObject({
candidateCount: 2,
deduplicatedCount: 0,
dedupDegraded: false,
});
}
});
it("only deduplicates exact crops within the same full DOM identity and direct parent", async () => {
const base = candidate(20);
const variants = [
candidate(30, 100, 2),
{ ...candidate(40), parentBackendNodeId: null },
...["x", "y", "width", "height"].map((axis) => ({
...base,
region: {
...base.region,
crop: {
...base.region.crop,
[axis]: base.region.crop[axis as keyof typeof base.region.crop] + 0.001,
},
},
})),
...[
{ frameId: "child" },
{ target: { tabId: 2 } },
{ target: { tabId: 1, sessionId: "remote" } },
{ documentElementBackendNodeId: 9 },
{ attachmentId: "b" },
].map((identity) => ({ ...base, document: { ...base.document, ...identity } })),
];
const duplicate = { ...candidate(10), label: "Not a new target" };
const result = await deduplicateVisualCandidates(discovery([base, ...variants, duplicate]));
expect(result.candidates).toEqual([base, ...variants]);
expect(result.candidates[0]).toBe(base);
expect(result.deduplicatedCount).toBe(1);
});
it.each([
50_001, 100_000,
])("keeps all %s distinct candidates including the last tiny Canvas", async (count) => {
const input = Array.from({ length: count }, (_, i) => candidate(i + 1, 100, i + 1));
input[count - 1] = candidate(count, 1, count);
// A remembered key still deduplicates; an unremembered key passes through.
input.push({ ...input[0] }, { ...input[count - 1] });
const result = await deduplicateVisualCandidates(discovery(input));
expect(result.candidates).toEqual([...input.slice(0, count), input[count + 1]]);
expect(result.candidates[count - 1]).toBe(input[count - 1]);
expect(result).toMatchObject({
candidateCount: count + 2,
deduplicatedCount: 1,
dedupDegraded: true,
});
});
it("does not report degradation when every key fits", async () => {
const input = Array.from({ length: MAX_VISUAL_DEDUP_KEYS }, (_, i) => candidate(i, 1, i));
input.push({ ...input[0] });
const result = await deduplicateVisualCandidates(discovery(input));
expect(result.candidates).toHaveLength(MAX_VISUAL_DEDUP_KEYS);
expect(result.dedupDegraded).toBe(false);
expect(result.deduplicatedCount).toBe(1);
});
it("preserves upstream unknowns without counting issues as Canvas nodes", async () => {
const input: VisualDiscoveryResult = {
...discovery([candidate(2)]),
complete: false,
issues: [{ reason: "geometry-unavailable", target: { tabId: 1 }, frameId: "missing" }],
captureIssues: [{ stage: "ax", reason: "capture-unavailable", target: { tabId: 1 } }],
};
const result = await deduplicateVisualCandidates(input);
expect(result.complete).toBe(false);
expect(result.candidateCount).toBe(1);
expect(result.issues).toBe(input.issues);
expect(result.captureIssues).toBe(input.captureIssues);
const absent = await deduplicateVisualCandidates({
...input,
candidates: [],
issues: [{ reason: "visual-facts-not-collected" }],
});
expect(absent).toMatchObject({
complete: false,
candidateCount: 0,
issues: [{ reason: "visual-facts-not-collected" }],
});
expect(await deduplicateVisualCandidates(discovery([]))).toMatchObject({
complete: true,
candidates: [],
candidateCount: 0,
dedupDegraded: false,
});
});
it("drains cancellation through the shared checkpoint", async () => {
const controller = new AbortController();
controller.abort();
await expect(
deduplicateVisualCandidates(discovery([]), controller.signal),
).rejects.toMatchObject({ name: "AbortError" });
const during = new AbortController();
const input = discovery(Array.from({ length: 100_000 }, (_, i) => candidate(i, 100, i)));
const timer = setTimeout(() => during.abort(), 0);
try {
await expect(deduplicateVisualCandidates(input, during.signal)).rejects.toMatchObject({
name: "AbortError",
});
} finally {
clearTimeout(timer);
}
});
});
@@ -1,528 +0,0 @@
import { describe, expect, it } from "vitest";
import type { CdpFrame } from "@/browser-driver/frame-graph";
import { OVERLAY_HOST_MARKER_ATTR } from "@/lib/overlay-bridge";
import type { GeometryProjection } from "../../geometry";
import { buildDocumentIndex, type DocumentFacts, type ObservationFacts } from "../facts";
import type { FrameOwnedAxNode } from "../frame-document";
import { normalizeDocument } from "../normalize";
import { VISUAL_SNAPSHOT, VISUAL_STYLES } from "../snapshot";
import { discoverVisualCandidates } from "../visual-discovery";
const defaults: Record<string, string> = {
position: "static",
"pointer-events": "auto",
cursor: "auto",
visibility: "visible",
opacity: "1",
display: "block",
"overflow-x": "visible",
"overflow-y": "visible",
transform: "none",
zoom: "1",
"clip-path": "none",
"mask-image": "none",
rotate: "none",
scale: "none",
perspective: "none",
clip: "auto",
contain: "none",
"overflow-clip-margin": "0px",
};
const projection: GeometryProjection = {
sourceClips: [],
edges: [],
topViewport: { width: 1000, height: 800 },
};
interface Spec {
id: number;
parent?: number;
tag?: string;
styles?: Record<string, string>;
bounds?: number[] | null;
client?: number[];
attrs?: Record<string, string>;
}
async function document(
nodes: Spec[],
options: {
frame?: CdpFrame;
projection?: GeometryProjection;
pageScale?: number;
scrollY?: number;
layoutScale?: number;
} = {},
): Promise<DocumentFacts<FrameOwnedAxNode>> {
const frame = options.frame ?? { frameId: "main", target: { tabId: 1 } };
const strings: string[] = [];
const str = (value: string) => {
const index = strings.indexOf(value);
if (index >= 0) return index;
strings.push(value);
return strings.length - 1;
};
const input: Spec[] = [
{ id: 0, tag: "#document", bounds: null },
{ id: 1, parent: 0, tag: "html", bounds: [0, 0, 1000, 800] },
...nodes,
];
const layouts = input.filter((node) => node.bounds !== null);
const normalized = await normalizeDocument(
{
nodes: {
backendNodeId: input.map((n) => n.id),
nodeType: input.map((n) =>
n.tag === "#document" ? 9 : n.tag === "#document-fragment" ? 11 : 1,
),
parentIndex: input.map((n) =>
n.parent === undefined ? -1 : input.findIndex((p) => p.id === n.parent),
),
nodeName: input.map((n) => str(n.tag ?? "canvas")),
attributes: input.map((n) =>
Object.entries(n.attrs ?? {}).flatMap(([k, v]) => [str(k), str(v)]),
),
},
layout: {
nodeIndex: layouts.map((n) => input.indexOf(n)),
bounds: layouts.map((n) => n.bounds ?? [10, 20, 120, 40]),
clientRects: layouts.map((n) => n.client ?? [0, 0, 1000, 800]),
styles: layouts.map((n) =>
VISUAL_STYLES.map((key) => str({ ...defaults, ...n.styles }[key])),
),
},
},
strings,
{
frameId: frame.frameId,
ownerFrameBackendNodeId: frame.ownerBackendNodeId ?? null,
target: frame.target,
projection: {
status: "available",
projection: {
source: { target: frame.target, frameId: frame.frameId },
geometry: options.projection ?? projection,
},
},
coordinates: {
layoutUnitsPerCssPixel: options.layoutScale ?? 1,
scrollCss: { x: 0, y: options.scrollY ?? 0 },
},
pageScale: options.pageScale ?? 1,
},
undefined,
VISUAL_SNAPSHOT,
);
return {
frame,
identity: {
attachmentId: "attached",
target: frame.target,
frameId: frame.frameId,
documentElementBackendNodeId: 1,
},
index: normalized.index,
geometry: normalized.geometry,
domNodes: normalized.nodes,
axNodes: [],
};
}
function facts(
documents: DocumentFacts<FrameOwnedAxNode>[],
issues: ObservationFacts<FrameOwnedAxNode>["issues"] = [],
): ObservationFacts<FrameOwnedAxNode> {
return {
visualFactsCollected: true,
rootFrameId: "main",
documents,
viewport: { width: 1000, height: 800 },
issues,
startedAt: 1,
finishedAt: 2,
};
}
describe("Canvas discovery", () => {
it("shares frame address paths from existing Facts without changing candidate evidence", async () => {
const parent = await document([{ id: 2, parent: 1, tag: "iframe" }]);
const child = await document(
[
{ id: 3, parent: 1 },
{ id: 4, parent: 1 },
],
{
frame: {
frameId: "child",
parentFrameId: "main",
ownerBackendNodeId: 2,
target: { tabId: 1 },
},
},
);
const result = await discoverVisualCandidates(facts([child, parent]));
expect(result.candidates).toHaveLength(2);
const path = result.candidates[0].framePath;
expect(path).toBe(result.candidates[1].framePath);
expect(path?.document).toBe(child.identity);
expect(path?.parent?.ownerBackendNodeId).toBe(2);
expect(path?.parent?.frame.document).toBe(parent.identity);
expect(path?.parent?.frame.parent).toBeUndefined();
});
it("distinguishes visual facts not collected from complete empty discovery and honors cancellation", async () => {
const empty = facts([]);
expect(await discoverVisualCandidates(empty)).toMatchObject({
complete: true,
candidates: [],
issues: [],
});
expect(await discoverVisualCandidates({ ...empty, visualFactsCollected: false })).toMatchObject(
{ complete: false, candidates: [], issues: [{ reason: "visual-facts-not-collected" }] },
);
const controller = new AbortController();
controller.abort();
await expect(
discoverVisualCandidates({ ...empty, visualFactsCollected: false }, controller.signal),
).rejects.toMatchObject({ name: "AbortError" });
});
it("discovers Canvas and frame contents without corner radius facts", async () => {
const own = await document([{ id: 2, parent: 1, styles: {} }]);
expect(await discoverVisualCandidates(facts([own]))).toMatchObject({
candidates: [{ backendNodeId: 2 }],
complete: true,
issues: [],
});
const parent = await document([{ id: 2, parent: 1, tag: "iframe", styles: {} }]);
const child = await document([{ id: 3, parent: 1 }], {
frame: {
frameId: "child",
parentFrameId: "main",
ownerBackendNodeId: 2,
target: { tabId: 1 },
},
});
expect(await discoverVisualCandidates(facts([parent, child]))).toMatchObject({
candidates: [{ backendNodeId: 3 }],
complete: true,
issues: [],
});
});
it.each([
0.8, 1, 1.25, 2,
])("combines layout-unit bounds with CSS client clips at scale %s", async (scale) => {
const doc = await document(
[
{
id: 2,
parent: 1,
tag: "div",
bounds: [100, 200, 64, 34].map((n) => n * scale),
client: [2, 2, 60, 30],
styles: { "overflow-x": "hidden", "overflow-y": "hidden" },
},
{ id: 3, parent: 2, bounds: [102, 202, 120, 40].map((n) => n * scale) },
],
{ layoutScale: scale, scrollY: 75 },
);
const result = await discoverVisualCandidates(facts([doc]));
expect(result.issues).toEqual([]);
const expected = {
borderBox: { x: 102, y: 127, width: 120, height: 40 },
crop: { x: 102, y: 127, width: 60, height: 30 },
};
for (const box of ["borderBox", "crop"] as const)
for (const key of ["x", "y", "width", "height"] as const)
expect(result.candidates[0].region[box][key]).toBeCloseTo(expected[box][key], 8);
});
it("normalizes a zero-width ancestor origin before one-axis clipping", async () => {
const doc = await document(
[
{
id: 2,
parent: 1,
tag: "div",
bounds: [200, 400, 0, 60],
client: [0, 0, 0, 30],
styles: { "overflow-y": "hidden" },
},
{ id: 3, parent: 2, bounds: [200, 400, 240, 80] },
],
{ layoutScale: 2, scrollY: 75 },
);
const result = await discoverVisualCandidates(facts([doc]));
expect(result.issues).toEqual([]);
expect(result.candidates[0].region.crop).toEqual({ x: 100, y: 125, width: 120, height: 30 });
});
it("keeps unnamed, semantic, aria-hidden, inert, pointer-none and visibility override Canvas", async () => {
const doc = await document([
{ id: 2, parent: 1 },
{ id: 3, parent: 1, attrs: { "aria-label": "Chart" } },
{
id: 4,
parent: 1,
attrs: { "aria-hidden": "true", inert: "" },
styles: { "pointer-events": "none" },
},
{ id: 5, parent: 1, tag: "div", styles: { visibility: "hidden" } },
{ id: 6, parent: 5, styles: { visibility: "visible" } },
{ id: 7, parent: 1, attrs: { hidden: "" }, styles: { display: "block" } },
]);
doc.axNodes.push({ nodeId: "canvas", backendDOMNodeId: 2 } as FrameOwnedAxNode);
const result = await discoverVisualCandidates(facts([doc]));
expect(result.candidates.map((c) => c.backendNodeId)).toEqual([2, 3, 4, 6, 7]);
expect(result.candidates[1].label).toBe("Chart");
expect(result.complete).toBe(true);
});
it("excludes CSS transparent subtrees, hidden/zero/no-layout/fully clipped nodes and overlays", async () => {
const doc = await document([
{ id: 2, parent: 1, tag: "div", styles: { opacity: "0" } },
{ id: 3, parent: 2 },
{ id: 4, parent: 1, styles: { visibility: "hidden" } },
{ id: 5, parent: 1, bounds: [0, 0, 0, 40] },
{ id: 6, parent: 1, bounds: null },
{ id: 7, parent: 1, bounds: [1200, 0, 100, 50] },
{ id: 8, parent: 1, tag: "div", attrs: { [OVERLAY_HOST_MARKER_ATTR]: "" } },
{ id: 9, parent: 8, tag: "#document-fragment", bounds: null },
{ id: 10, parent: 9 },
]);
expect(await discoverVisualCandidates(facts([doc]))).toMatchObject({
candidates: [],
issues: [],
complete: true,
});
});
it.each([
["both", "hidden", "hidden", [0, 0, 60, 30], [60, 30]],
["x", "clip", "visible", [0, 0, 60, 20], [60, 40]],
["border", "hidden", "hidden", [5, 5, 68, 38], [64, 34]],
])("uses client box for %s clipping", async (_name, x, y, client, expected) => {
const doc = await document([
{
id: 2,
parent: 1,
tag: "div",
bounds: [0, 0, 78, 48],
client: client as number[],
styles: { "overflow-x": x as string, "overflow-y": y as string },
},
{
id: 3,
parent: 2,
bounds: x === "clip" || _name === "both" ? [0, 0, 120, 40] : [9, 9, 120, 40],
},
]);
const [candidate] = (await discoverVisualCandidates(facts([doc]))).candidates;
expect([candidate.region.crop.width, candidate.region.crop.height]).toEqual(expected);
expect(candidate.region.clips?.backendNodeId).toBe(2);
});
it("keeps Canvas border-box and clips partial viewport intersections", async () => {
const doc = await document([{ id: 2, parent: 1, bounds: [950, 20, 130, 50] }]);
const [candidate] = (await discoverVisualCandidates(facts([doc]))).candidates;
expect(candidate.region.borderBox.width).toBe(130);
expect(candidate.region.crop).toEqual({ x: 950, y: 20, width: 50, height: 50 });
});
it("propagates owner ancestry across targets, without depending on semantic parents", async () => {
const parent = await document([
{
id: 2,
parent: 1,
tag: "div",
bounds: [0, 0, 200, 100],
client: [0, 0, 200, 100],
styles: { "overflow-x": "hidden", "overflow-y": "hidden" },
},
{ id: 3, parent: 2, tag: "iframe", bounds: [100, 0, 200, 200] },
]);
parent.domNodes.splice(0); // Visual discovery never reads the semantic subset.
const child = await document([{ id: 2, parent: 1, bounds: [0, 0, 200, 200] }], {
frame: {
frameId: "child",
parentFrameId: "main",
ownerBackendNodeId: 3,
target: { tabId: 1, sessionId: "remote" },
},
projection: {
...projection,
edges: [
{
sourceViewport: { width: 200, height: 200 },
destinationQuad: [
{ x: 100, y: 0 },
{ x: 300, y: 0 },
{ x: 300, y: 200 },
{ x: 100, y: 200 },
],
},
],
},
});
const result = await discoverVisualCandidates(facts([child, parent]));
expect(result.complete).toBe(true);
expect(result.candidates[0]).toMatchObject({
document: { frameId: "child", target: { sessionId: "remote" } },
region: { crop: { x: 100, y: 0, width: 100, height: 100 } },
});
const owner = parent.index.nodes.get(3)!;
owner.layout!.styles = { ...owner.layout!.styles, visibility: "hidden" };
expect((await discoverVisualCandidates(facts([parent, child]))).candidates).toHaveLength(0);
});
it.each<Record<string, string>>([
{ transform: "matrix(0,1,-1,0,0,0)" },
{ "clip-path": "circle(50%)" },
{ "mask-image": "url(mask)" },
{ zoom: "1.25", "overflow-x": "hidden" },
])("reports unsupported ancestor geometry: %o", async (styles) => {
const doc = await document([
{ id: 2, parent: 1, tag: "div", styles },
{ id: 3, parent: 2 },
]);
const result = await discoverVisualCandidates(facts([doc]));
expect(result.candidates).toHaveLength(0);
expect(result.issues[0].reason).toBe("geometry-unsupported");
});
it("supports positive scale without overflow, rejects pinch and non-rectangular frame projection", async () => {
const nodes = [
{
id: 2,
parent: 1,
styles: { transform: "matrix(1.25,0,0,1.25,0,0)" },
bounds: [10, 20, 150, 50],
},
];
expect(
(await discoverVisualCandidates(facts([await document(nodes)]))).candidates,
).toHaveLength(1);
expect(
(await discoverVisualCandidates(facts([await document(nodes, { pageScale: 1.5 })]))).issues[0]
.reason,
).toBe("geometry-unsupported");
const rotated = {
...projection,
edges: [
{
sourceViewport: { width: 200, height: 200 },
destinationQuad: [
{ x: 10, y: 0 },
{ x: 200, y: 10 },
{ x: 190, y: 200 },
{ x: 0, y: 190 },
] as [
{ x: number; y: number },
{ x: number; y: number },
{ x: number; y: number },
{ x: number; y: number },
],
},
],
};
expect(
(await discoverVisualCandidates(facts([await document(nodes, { projection: rotated })])))
.issues[0].reason,
).toBe("geometry-unsupported");
});
it("keeps partial capture separate from valid geometry, rejects unverified identities", async () => {
const doc = await document([{ id: 2, parent: 1 }]);
const partial = facts(
[doc],
[{ target: doc.frame.target, frameId: "main", stage: "ax", reason: "capture-unavailable" }],
);
const result = await discoverVisualCandidates(partial);
expect(result.candidates).toHaveLength(1);
expect(result.captureIssues).toBe(partial.issues);
expect(result.complete).toBe(false);
const unverified = { ...doc, identity: undefined };
const failed = await discoverVisualCandidates(facts([unverified]));
expect(failed.candidates).toHaveLength(0);
expect(failed.issues[0].reason).toBe("identity-unverified");
});
it("does not trust orphan, cyclic, boxless or missing frame-owner ancestry", async () => {
const doc = await document([
{ id: 2, parent: 1, tag: "div" },
{ id: 3, parent: 2 },
]);
for (const parent of [99, 3]) {
doc.index.nodes.get(2)!.parentBackendNodeId = parent;
const index = await buildDocumentIndex([...doc.index.nodes.values()], undefined, true);
const result = await discoverVisualCandidates(facts([{ ...doc, index }]));
expect(result.candidates).toHaveLength(0);
expect(result.issues[0].reason).toBe("ancestry-incomplete");
}
const boxless = await document([
{ id: 2, parent: 1, tag: "div", bounds: null },
{ id: 3, parent: 2 },
]);
expect((await discoverVisualCandidates(facts([boxless]))).issues[0].reason).toBe(
"facts-unavailable",
);
const child = {
...doc,
frame: { ...doc.frame, frameId: "child", parentFrameId: "absent", ownerBackendNodeId: 2 },
};
expect((await discoverVisualCandidates(facts([child]))).issues[0].reason).toBe(
"ownership-unresolved",
);
});
it("is deterministic, does not deduplicate or mutate inputs", async () => {
const input = facts([
await document([
{ id: 2, parent: 1 },
{ id: 3, parent: 1 },
]),
]);
const freeze = (value: unknown): void => {
if (!value || typeof value !== "object" || Object.isFrozen(value)) return;
if (value instanceof Map) for (const item of value.values()) freeze(item);
for (const item of Object.values(value)) freeze(item);
Object.freeze(value);
};
freeze(input);
const first = await discoverVisualCandidates(input);
expect(await discoverVisualCandidates(input)).toEqual(first);
expect(first.candidates).toHaveLength(2);
});
it.each([
1000, 10000, 100000,
])("memoizes deep ancestry across %i nodes and checks cancellation", async (count) => {
const doc = await document([{ id: 2, parent: 1 }]);
const template = doc.index.nodes.get(2)!;
let reads = 0;
const input = [doc.index.nodes.get(0)!, doc.index.nodes.get(1)!];
for (let id = 2; id < count + 2; id++) {
const node = {
...template,
backendNodeId: id,
tag: id < count / 2 ? "div" : "canvas",
parentBackendNodeId: id < count / 2 ? id - 1 : count / 2 - 1,
};
Object.defineProperty(node, "parentBackendNodeId", {
get: () => {
reads++;
return id < count / 2 ? id - 1 : count / 2 - 1;
},
});
input.push(node);
}
const index = await buildDocumentIndex(input, undefined, true);
reads = 0;
const result = await discoverVisualCandidates(facts([{ ...doc, index }]));
expect(result.candidates).toHaveLength(count / 2 + 2);
expect(reads).toBeLessThan(count * 5);
const controller = new AbortController();
controller.abort();
await expect(discoverVisualCandidates(facts([doc]), controller.signal)).rejects.toMatchObject({
name: "AbortError",
});
});
});
@@ -1,250 +0,0 @@
import { describe, expect, it } from "vitest";
import {
EMPTY_VISUAL_CONTEXT,
extendVisualContext,
projectVisualBox,
resolveVisualRegion,
type VisualAncestor,
} from "../visual-region";
const styles = {
position: "static",
visibility: "visible",
opacity: "1",
display: "block",
transform: "none",
zoom: "1",
"overflow-x": "visible",
"overflow-y": "visible",
"clip-path": "none",
"mask-image": "none",
rotate: "none",
scale: "none",
perspective: "none",
clip: "auto",
contain: "none",
"overflow-clip-margin": "0px",
};
const ancestor: VisualAncestor = {
document: {
attachmentId: "a",
target: { tabId: 1 },
frameId: "main",
documentElementBackendNodeId: 1,
},
backendNodeId: 2,
styles,
clientBox: { x: 5, y: 5, width: 60, height: 30 },
};
const box = { x: 0, y: 0, width: 120, height: 40 };
describe("shared visual region rules", () => {
it.each([
"visible",
"hidden",
"auto",
"scroll",
"clip",
])("keeps %s screenshot bounds independent of rounded styling", (overflow) => {
for (const clipContents of [true, false]) {
const node = {
...ancestor,
styles: { ...styles, "overflow-x": overflow, "overflow-y": overflow },
};
const baseline = extendVisualContext(EMPTY_VISUAL_CONTEXT, node, clipContents);
for (const radius of ["0px", "8px", "50%", "4px 8px"]) {
const rounded = extendVisualContext(
EMPTY_VISUAL_CONTEXT,
{
...node,
styles: {
...node.styles,
"border-top-left-radius": radius,
"border-top-right-radius": radius,
"border-bottom-left-radius": radius,
"border-bottom-right-radius": radius,
},
},
clipContents,
);
expect(rounded).toEqual(baseline);
expect(
resolveVisualRegion({
borderBox: box,
frameVisibleBox: box,
visibility: "visible",
context: rounded,
}),
).toMatchObject({ status: "available" });
expect(
resolveVisualRegion({
borderBox: box,
frameVisibleBox: box,
visibility: "hidden",
context: rounded,
}),
).toEqual({ status: "empty" });
if (clipContents && overflow !== "visible")
expect(
resolveVisualRegion({
borderBox: box,
frameVisibleBox: { x: 80, y: 0, width: 20, height: 20 },
visibility: "visible",
context: rounded,
}),
).toEqual({ status: "empty" });
}
}
});
it("accepts independently supplied normalized facts and preserves clipping policy", () => {
const context = extendVisualContext(EMPTY_VISUAL_CONTEXT, {
...ancestor,
styles: { ...styles, "overflow-x": "hidden" },
});
expect(
resolveVisualRegion({ borderBox: box, frameVisibleBox: box, visibility: "visible", context }),
).toMatchObject({
status: "available",
crop: { x: 5, y: 0, width: 60, height: 40 },
clips: { overflowX: "hidden", overflowY: "visible" },
});
const differentPolicy = extendVisualContext(EMPTY_VISUAL_CONTEXT, {
...ancestor,
styles: { ...styles, "overflow-x": "scroll" },
});
expect(differentPolicy.clip).toEqual(context.clip);
expect(differentPolicy.clips).not.toEqual(context.clips);
});
it("distinguishes fully clipped, missing client geometry and invalid rectangles", () => {
const clipping = { ...ancestor, styles: { ...styles, "overflow-x": "hidden" } };
const context = extendVisualContext(EMPTY_VISUAL_CONTEXT, {
...clipping,
clientBox: { x: 0, y: 0, width: 0, height: 20 },
});
expect(
resolveVisualRegion({ borderBox: box, frameVisibleBox: box, visibility: "visible", context })
.status,
).toBe("empty");
const unavailable = extendVisualContext(EMPTY_VISUAL_CONTEXT, { ...clipping, clientBox: null });
expect(
resolveVisualRegion({
borderBox: box,
frameVisibleBox: box,
visibility: "visible",
context: unavailable,
}),
).toEqual({ status: "unavailable", reason: "geometry-unavailable" });
expect(
resolveVisualRegion({
borderBox: { ...box, x: NaN },
frameVisibleBox: box,
visibility: "visible",
context: EMPTY_VISUAL_CONTEXT,
}).status,
).toBe("unavailable");
expect(projectVisualBox({ x: 5, y: 5, width: 0, height: 20 }, [])).toEqual({
x: 5,
y: 5,
width: 0,
height: 20,
});
});
it.each<Record<string, string>>([
{ rotate: "45deg" },
{ scale: "-1 1" },
{ perspective: "100px" },
{ clip: "rect(0px, 20px, 20px, 0px)" },
{ contain: "paint" },
{ "overflow-x": "clip", "overflow-clip-margin": "10px" },
{ transform: "matrix3d(1,0,0,0,0,1,0,0,0,0,1,0.01,0,0,0,1)" },
])("refuses unsupported CSS evidence without changing DOM geometry: %o", (change) => {
const context = extendVisualContext(EMPTY_VISUAL_CONTEXT, {
...ancestor,
styles: { ...styles, ...change },
});
expect(context.issue).toBe("geometry-unsupported");
});
it("accepts axis-preserving matrix3d translation and independent positive scale", () => {
for (const change of [
{ transform: "matrix3d(1,0,0,0,0,1,0,0,0,0,1,0,10,20,0,1)" },
{ scale: "1.25" },
])
expect(
extendVisualContext(EMPTY_VISUAL_CONTEXT, { ...ancestor, styles: { ...styles, ...change } })
.issue,
).toBeUndefined();
});
it.each([
"matrix(1,0,0,1,0,0)",
"matrix(1,0,0,1,10,20)",
])("accepts unscaled overflow transform %s", (transform) => {
const context = extendVisualContext(EMPTY_VISUAL_CONTEXT, {
...ancestor,
styles: {
...styles,
transform,
position: "relative",
"overflow-x": "hidden",
"overflow-y": "hidden",
},
});
expect(context.issue).toBeUndefined();
const child = extendVisualContext(
context,
{
...ancestor,
styles: {
...styles,
position: "absolute",
"overflow-x": "clip",
"overflow-y": "clip",
"overflow-clip-margin": "content-box",
},
},
false,
);
expect(
resolveVisualRegion({
borderBox: box,
frameVisibleBox: box,
visibility: "visible",
context: child,
}),
).toMatchObject({ status: "available", crop: { x: 5, y: 5, width: 60, height: 30 } });
});
it("supports positioned overflow containers but refuses uncertain out-of-flow clipping", () => {
const overflow = { ...ancestor, styles: { ...styles, "overflow-x": "hidden" } };
const staticParent = extendVisualContext(EMPTY_VISUAL_CONTEXT, overflow);
expect(
extendVisualContext(
staticParent,
{ ...ancestor, styles: { ...styles, position: "absolute" } },
false,
).issue,
).toBe("geometry-unsupported");
const positionedParent = extendVisualContext(EMPTY_VISUAL_CONTEXT, {
...overflow,
styles: { ...overflow.styles, position: "relative" },
});
const child = extendVisualContext(
positionedParent,
{ ...ancestor, styles: { ...styles, position: "absolute" } },
false,
);
expect(child.issue).toBeUndefined();
expect(child.clip).toEqual(positionedParent.clip);
expect(
extendVisualContext(
positionedParent,
{ ...ancestor, styles: { ...styles, position: "fixed" } },
false,
).issue,
).toBe("geometry-unsupported");
});
});
@@ -22,12 +22,7 @@ import {
type FrameOwnedAxNode,
} from "./frame-document";
import { type NormalizedFrameDocument, normalizeSnapshot } from "./normalize";
import {
BASIC_SNAPSHOT,
describeSnapshotFrames,
type SnapshotReply,
VISUAL_SNAPSHOT,
} from "./snapshot";
import { describeSnapshotFrames, REQUESTED_STYLES, type SnapshotReply } from "./snapshot";
interface TargetBatch<T extends FrameOwnedAxNode> {
target: CdpTarget;
@@ -74,9 +69,7 @@ export async function captureObservationFacts<T extends FrameOwnedAxNode>(
tabId: number,
signal?: AbortSignal,
pageUrl?: string,
options: { includeVisualFacts?: boolean } = {},
): Promise<ObservationFacts<T>> {
const profile = options.includeVisualFacts ? VISUAL_SNAPSHOT : BASIC_SNAPSHOT;
const startedAt = Date.now();
throwCaptureAborted(signal);
let graph: CdpFrameGraph | undefined;
@@ -132,7 +125,7 @@ export async function captureObservationFacts<T extends FrameOwnedAxNode>(
throwCaptureAborted(signal);
batch.snapshotAttachmentId = cdp.getAttachmentId?.(tabId);
const snapshot = await scoped.send<SnapshotReply>(tabId, "DOMSnapshot.captureSnapshot", {
computedStyles: profile.computedStyles,
computedStyles: REQUESTED_STYLES,
includePaintOrder: true,
includeDOMRects: true,
});
@@ -153,7 +146,6 @@ export async function captureObservationFacts<T extends FrameOwnedAxNode>(
issues,
signal,
observed.rootFrameId,
profile,
);
const formsAvailable = await enrichFormControlStates(
scoped,
@@ -383,7 +375,6 @@ export async function captureObservationFacts<T extends FrameOwnedAxNode>(
facts.push({
frame,
identity: identities.get(document.frameId),
...(doc?.geometry ? { geometry: doc.geometry } : {}),
index: fallback?.size ? { ...index, excludedBackendNodeIds: fallback } : index,
domNodes,
axNodes,
@@ -392,15 +383,7 @@ export async function captureObservationFacts<T extends FrameOwnedAxNode>(
throwCaptureAborted(signal);
if (firstFailure && facts.every((doc) => !doc.domNodes.length && !doc.axNodes.length))
throw firstFailure;
return {
visualFactsCollected: profile.includeVisualFacts,
rootFrameId,
viewport,
documents: facts,
issues,
startedAt,
finishedAt: Date.now(),
};
return { rootFrameId, viewport, documents: facts, issues, startedAt, finishedAt: Date.now() };
}
/** Existing semantic consumers get a narrow view; no raw snapshot
@@ -2,7 +2,6 @@ import type { CdpRunner } from "../shared";
import { sendToCdpTarget } from "../shared";
import { isAbortError, throwIfAborted } from "./capture-abort";
import type { DocumentIdentity } from "./facts";
import type { VisualCandidate } from "./visual-discovery";
/** Compare the snapshot root with the current root in that exact frame.
* Deep serialization supplies the backend ID without a separate describeNode. */
@@ -11,110 +10,44 @@ export async function verifyDocumentIdentity(
identity: DocumentIdentity,
signal?: AbortSignal,
): Promise<"current" | "changed" | "unavailable"> {
return (await verifyIdentity(cdp, identity, signal)).status;
}
/** Verify the visual anchor in its original frame; geometry is checked by screenshot execution. */
export async function verifyVisualTargetIdentity(
cdp: CdpRunner,
candidate: VisualCandidate,
signal?: AbortSignal,
): Promise<"current" | "changed" | "unavailable"> {
return (await verifyIdentity(cdp, candidate.document, signal, candidate.backendNodeId)).status;
}
export type VerifiedNodeResult =
| { status: "changed" | "unavailable" }
| { status: "current"; objectId: string; objectGroup: string };
/** Caller must release objectGroup in finally after its local read. */
export async function resolveVerifiedNode(
cdp: CdpRunner,
identity: DocumentIdentity,
backendNodeId: number,
signal?: AbortSignal,
): Promise<VerifiedNodeResult> {
const result = await verifyIdentity(cdp, identity, signal, backendNodeId, true);
return result.status === "current" && result.objectId && result.objectGroup
? { status: "current", objectId: result.objectId, objectGroup: result.objectGroup }
: { status: result.status === "changed" ? "changed" : "unavailable" };
}
async function verifyIdentity(
cdp: CdpRunner,
identity: DocumentIdentity,
signal?: AbortSignal,
anchorBackendNodeId?: number,
retain = false,
): Promise<{
status: "current" | "changed" | "unavailable";
objectId?: string;
objectGroup?: string;
}> {
const attached = () => cdp.getAttachmentId?.(identity.target.tabId) === identity.attachmentId;
throwIfAborted(signal);
if (!attached()) return { status: "changed" };
if (!attached()) return "changed";
const objectGroup = `bsk-document-identity-${crypto.randomUUID()}`;
const send = <T>(method: string, params: object) => {
throwIfAborted(signal);
return sendToCdpTarget<T>(cdp, identity.target, method, params);
};
let objectId: string | undefined;
let retained = false;
try {
const world = await send<{ executionContextId: number }>("Page.createIsolatedWorld", {
frameId: identity.frameId,
worldName: "bsk-document-identity",
});
const serializationOptions = {
serialization: "deep",
additionalParameters: { maxNodeDepth: 0, includeShadowTree: "none" },
};
type RootReply = {
const reply = await send<{
result?: { deepSerializedValue?: { type: string; value?: { backendNodeId?: number } } };
};
let reply: RootReply;
if (anchorBackendNodeId === undefined) {
reply = await send<RootReply>("Runtime.evaluate", {
expression: "document.documentElement",
contextId: world.executionContextId,
objectGroup,
serializationOptions,
});
} else {
const anchor = await send<{ object?: { objectId?: string } }>("DOM.resolveNode", {
backendNodeId: anchorBackendNodeId,
executionContextId: world.executionContextId,
objectGroup,
});
if (!anchor.object?.objectId) return { status: "unavailable" };
objectId = anchor.object.objectId;
reply = await send<RootReply>("Runtime.callFunctionOn", {
objectId: anchor.object.objectId,
functionDeclaration:
"function() { return this.isConnected && this.ownerDocument === document ? document.documentElement : null; }",
objectGroup,
serializationOptions,
});
}
if (!attached()) return { status: "changed" };
}>("Runtime.evaluate", {
expression: "document.documentElement",
contextId: world.executionContextId,
objectGroup,
serializationOptions: {
serialization: "deep",
additionalParameters: { maxNodeDepth: 0, includeShadowTree: "none" },
},
});
if (!attached()) return "changed";
const root = reply.result?.deepSerializedValue;
if (root?.type === "null") return { status: "changed" };
if (root?.type === "null") return "changed";
const id = root?.type === "node" ? root.value?.backendNodeId : undefined;
if (id === undefined) return { status: "unavailable" };
if (id !== identity.documentElementBackendNodeId) return { status: "changed" };
throwIfAborted(signal);
retained = retain && !!objectId;
return { status: "current", ...(retained ? { objectId, objectGroup } : {}) };
if (id === undefined) return "unavailable";
return id === identity.documentElementBackendNodeId ? "current" : "changed";
} catch (error) {
throwIfAborted(signal);
if (isAbortError(error)) throw error;
return { status: attached() ? "unavailable" : "changed" };
return attached() ? "unavailable" : "changed";
} finally {
if (!retained)
await sendToCdpTarget(cdp, identity.target, "Runtime.releaseObjectGroup", {
objectGroup,
}).catch(() => {});
await sendToCdpTarget(cdp, identity.target, "Runtime.releaseObjectGroup", {
objectGroup,
}).catch(() => {});
throwIfAborted(signal);
}
}
+3 -30
View File
@@ -1,8 +1,7 @@
import type { Viewport } from "@browser-skill/vom";
import type { CdpFrame, CdpTarget } from "@/browser-driver/frame-graph";
import { isOverlayHostNode } from "@/lib/overlay-bridge";
import type { GeometryProjection } from "../geometry";
import type { FrameProjectionIssue, SnapshotCoordinates } from "../geometry/coordinate-types";
import type { FrameProjectionIssue } from "../geometry/coordinate-types";
import { createCaptureCheckpoint } from "./capture-abort";
import type { CapturedNode } from "./capture-types";
import type { FrameOwnedAxNode } from "./frame-document";
@@ -11,26 +10,21 @@ import type { FrameOwnedAxNode } from "./frame-document";
export interface SnapshotLayout {
readonly boundsSpace: "snapshot-document-layout";
bounds?: number[];
/** [clientLeft, clientTop, clientWidth, clientHeight], untransformed local CSS units. */
clientRect?: number[];
styles: Readonly<Record<string, string>>;
}
export interface DecodedNode extends Omit<CapturedNode, "rect" | "localRect" | "rendered"> {
nodeType?: number;
parentMissing?: boolean;
layout?: SnapshotLayout;
}
export interface NodeFacts extends CapturedNode {
nodeType?: number;
parentMissing?: boolean;
layout?: SnapshotLayout;
}
export interface DocumentIndex<T extends DecodedNode = NodeFacts> {
readonly nodes: ReadonlyMap<number, T>;
readonly ancestryComplete?: ReadonlyMap<number, boolean>;
readonly excludedBackendNodeIds: ReadonlySet<number>;
}
@@ -43,12 +37,10 @@ export interface DecodedDocument {
export async function buildDocumentIndex<T extends DecodedNode>(
input: readonly T[],
signal?: AbortSignal,
includeVisualFacts = false,
): Promise<DocumentIndex<T>> {
const checkpoint = createCaptureCheckpoint(signal);
const nodes = new Map<number, T>();
const overlayByNode = new Map<number, boolean>();
const ancestryComplete = includeVisualFacts ? new Map<number, boolean>() : undefined;
const excludedBackendNodeIds = new Set<number>();
for (let i = 0; i < input.length; i++) {
if (i % 256 === 0) {
@@ -69,7 +61,6 @@ export async function buildDocumentIndex<T extends DecodedNode>(
const visiting = new Set<number>();
let current: DecodedNode | undefined = node;
let overlay = false;
let complete = false;
while (current && !overlayByNode.has(current.backendNodeId)) {
if (work++ % 256 === 0) {
const pending = checkpoint();
@@ -85,16 +76,13 @@ export async function buildDocumentIndex<T extends DecodedNode>(
visiting.add(current.backendNodeId);
path.push(current);
if (current.parentBackendNodeId === null) {
if (ancestryComplete) complete = !current.parentMissing && current.nodeType === 9;
current = undefined;
break;
}
current = nodes.get(current.parentBackendNodeId);
}
if (current && overlayByNode.has(current.backendNodeId)) {
if (current && overlayByNode.has(current.backendNodeId))
overlay = overlayByNode.get(current.backendNodeId)!;
if (ancestryComplete) complete = ancestryComplete.get(current.backendNodeId) === true;
}
for (let i = path.length - 1; i >= 0; i--) {
if (work++ % 256 === 0) {
const pending = checkpoint();
@@ -103,14 +91,10 @@ export async function buildDocumentIndex<T extends DecodedNode>(
const item = path[i];
overlay = overlay || isOverlayHostNode(item.tag, Object.keys(item.attrs));
overlayByNode.set(item.backendNodeId, overlay);
if (ancestryComplete) {
complete = complete && !item.parentMissing;
ancestryComplete.set(item.backendNodeId, complete);
}
if (overlay) excludedBackendNodeIds.add(item.backendNodeId);
}
}
return { nodes, ...(ancestryComplete ? { ancestryComplete } : {}), excludedBackendNodeIds };
return { nodes, excludedBackendNodeIds };
}
export interface DocumentIdentity {
@@ -143,16 +127,7 @@ export interface CaptureIssue {
| "geometry-unavailable";
}
/** Shared per document; never retain the live GeometryContext in published facts. */
export interface DocumentGeometry {
readonly projections: readonly GeometryProjection[];
readonly coordinates: SnapshotCoordinates;
/** Visual viewport scale (pinch), distinct from browser UI zoom. */
readonly pageScale?: number;
}
export interface DocumentFacts<T extends FrameOwnedAxNode> {
readonly geometry?: DocumentGeometry;
readonly frame: CdpFrame;
readonly identity?: DocumentIdentity;
readonly index: DocumentIndex;
@@ -161,8 +136,6 @@ export interface DocumentFacts<T extends FrameOwnedAxNode> {
}
export interface ObservationFacts<T extends FrameOwnedAxNode> {
/** Whether visual collection was enabled; partial capture failures remain in issues. */
readonly visualFactsCollected: boolean;
readonly rootFrameId: string;
readonly viewport: Viewport;
readonly documents: readonly DocumentFacts<T>[];
+3 -34
View File
@@ -18,18 +18,10 @@ import {
isAbortError as isCaptureAbort,
throwIfAborted as throwCaptureAborted,
} from "./capture-abort";
import { buildDocumentIndex, type CaptureIssue, type DocumentIndex, type NodeFacts } from "./facts";
import {
buildDocumentIndex,
type CaptureIssue,
type DocumentGeometry,
type DocumentIndex,
type NodeFacts,
} from "./facts";
import {
BASIC_SNAPSHOT,
decodeDocument,
type SnapshotDocument,
type SnapshotProfile,
type SnapshotReply,
snapshotFrameId,
} from "./snapshot";
@@ -41,11 +33,9 @@ export interface FrameContext {
targetProjection?: GeometryProjection | null;
target: CdpTarget;
coordinates: SnapshotCoordinates | null;
pageScale?: number;
}
export interface NormalizedDocument {
geometry?: DocumentGeometry;
nodes: NodeFacts[];
index: DocumentIndex;
documentElementBackendNodeId?: number;
@@ -57,9 +47,8 @@ export async function normalizeDocument(
strings: string[],
context: FrameContext,
signal?: AbortSignal,
profile: SnapshotProfile = BASIC_SNAPSHOT,
): Promise<NormalizedDocument> {
const decoded = await decodeDocument(doc, strings, signal, profile);
const decoded = await decodeDocument(doc, strings, signal);
const checkpoint = createCaptureCheckpoint(signal);
const nodes: NodeFacts[] = [];
for (let i = 0; i < decoded.nodes.length; i++) {
@@ -104,27 +93,12 @@ export async function normalizeDocument(
(Number.parseFloat(opacity) || 0) > 0,
});
}
const index = await buildDocumentIndex(nodes, signal, profile.includeVisualFacts);
const index = await buildDocumentIndex(nodes, signal);
return {
nodes: nodes.filter(
(node) => !node.tag.startsWith("#") && !index.excludedBackendNodeIds.has(node.backendNodeId),
),
index,
...(profile.includeVisualFacts &&
context.coordinates &&
context.projection?.status === "available" &&
context.targetProjection !== null
? {
geometry: {
projections: [
context.projection.projection.geometry,
...(context.targetProjection ? [context.targetProjection] : []),
],
coordinates: context.coordinates,
pageScale: context.pageScale,
},
}
: {}),
documentElementBackendNodeId: nodes.find(
(node) =>
node.nodeType === 1 &&
@@ -148,7 +122,6 @@ export async function normalizeSnapshot(
issues: CaptureIssue[],
signal?: AbortSignal,
rootFrameId?: string,
profile: SnapshotProfile = BASIC_SNAPSHOT,
): Promise<NormalizedFrameDocument[]> {
const strings = snapshot.strings ?? [];
const raw = snapshot.documents ?? [];
@@ -340,15 +313,11 @@ export async function normalizeSnapshot(
frameId: frame.frameId,
ownerFrameBackendNodeId: frame.ownerBackendNodeId ?? null,
projection: state,
...(profile.includeVisualFacts
? { pageScale: metrics.cssVisualViewport?.scale ?? metrics.visualViewport?.scale }
: {}),
target,
...(target.sessionId ? { targetProjection } : {}),
coordinates,
},
signal,
profile,
);
result.push({ ...normalized, frame });
}
+4 -35
View File
@@ -9,30 +9,9 @@ export const REQUESTED_STYLES = [
"visibility",
"opacity",
] as const;
export const VISUAL_STYLES = [
...REQUESTED_STYLES,
"display",
"overflow-x",
"overflow-y",
"transform",
"zoom",
"clip-path",
"mask-image",
"rotate",
"scale",
"perspective",
"clip",
"contain",
"overflow-clip-margin",
] as const;
/** The same profile owns request columns, decoding and visual-only derived facts. */
export const BASIC_SNAPSHOT = {
includeVisualFacts: false,
computedStyles: REQUESTED_STYLES,
} as const;
export const VISUAL_SNAPSHOT = { includeVisualFacts: true, computedStyles: VISUAL_STYLES } as const;
export type SnapshotProfile = typeof BASIC_SNAPSHOT | typeof VISUAL_SNAPSHOT;
const STYLE_COL = Object.fromEntries(
REQUESTED_STYLES.map((name, index) => [name, index]),
) as Record<(typeof REQUESTED_STYLES)[number], number>;
/** Sparse array format Chrome uses for infrequently-set per-node fields. */
interface SparseArray {
index: number[];
@@ -70,7 +49,6 @@ export interface SnapshotDocument {
nodeIndex?: number[];
styles?: number[][];
bounds?: number[][];
clientRects?: number[][];
paintOrders?: number[];
};
}
@@ -125,7 +103,6 @@ export async function decodeDocument(
doc: SnapshotDocument,
strings: string[],
signal?: AbortSignal,
profile: SnapshotProfile = BASIC_SNAPSHOT,
): Promise<DecodedDocument> {
const checkpoint = createCaptureCheckpoint(signal);
const dn = doc.nodes;
@@ -193,17 +170,13 @@ export async function decodeDocument(
const li = layoutByNode.get(n);
const styleRow = li === undefined ? [] : (dl?.styles?.[li] ?? []);
const styles: Record<string, string> = {};
for (let i = 0; i < profile.computedStyles.length; i++)
styles[profile.computedStyles[i]] = str(strings, styleRow[i]);
for (const name of REQUESTED_STYLES) styles[name] = str(strings, styleRow[STYLE_COL[name]]);
const layout =
li === undefined
? undefined
: {
boundsSpace: "snapshot-document-layout" as const,
bounds: dl?.bounds?.[li],
...(profile.includeVisualFacts && dl?.clientRects?.[li]
? { clientRect: dl.clientRects[li] }
: {}),
styles,
};
@@ -227,10 +200,6 @@ export async function decodeDocument(
backendNodeId,
nodeType: dn.nodeType?.[n],
parentBackendNodeId,
...(profile.includeVisualFacts &&
(dn.parentIndex?.[n] === undefined || (parentIdx >= 0 && parentBackendNodeId === null))
? { parentMissing: true }
: {}),
tag,
attrs,
@@ -1,64 +0,0 @@
import { createCaptureCheckpoint } from "./capture-abort";
import type { VisualCandidate, VisualDiscoveryResult } from "./visual-discovery";
export const MAX_VISUAL_DEDUP_KEYS = 50_000;
export interface VisualDedupResult extends VisualDiscoveryResult {
/** Known valid candidates in this capture, not the total Canvas count on the page. */
readonly candidateCount: number;
/** Candidates actually removed by exact coverage deduplication. */
readonly deduplicatedCount: number;
readonly dedupDegraded: boolean;
}
/** Keep the first real anchor for each exact DOM/parent/crop key, in encounter order.
* Once the key set is full, unknown keys pass through: capacity never drops candidates. */
export async function deduplicateVisualCandidates(
discovery: VisualDiscoveryResult,
signal?: AbortSignal,
): Promise<VisualDedupResult> {
const checkpoint = createCaptureCheckpoint(signal);
const keys = new Set<string>();
const candidates: VisualCandidate[] = [];
let deduplicatedCount = 0;
let dedupDegraded = false;
for (let i = 0; i < discovery.candidates.length; i++) {
if (i % 256 === 0) {
const pending = checkpoint();
if (pending) await pending;
}
const candidate = discovery.candidates[i];
const {
document,
region: { crop },
} = candidate;
const key = JSON.stringify([
document.attachmentId,
document.target.tabId,
document.target.sessionId ?? null,
document.frameId,
document.documentElementBackendNodeId,
candidate.parentBackendNodeId,
crop.x,
crop.y,
crop.width,
crop.height,
]);
if (keys.has(key)) {
deduplicatedCount++;
continue;
}
if (keys.size < MAX_VISUAL_DEDUP_KEYS) keys.add(key);
else dedupDegraded = true;
candidates.push(candidate);
}
const pending = checkpoint();
if (pending) await pending;
return {
...discovery,
candidates,
candidateCount: discovery.candidates.length,
deduplicatedCount,
dedupDegraded,
};
}
@@ -1,341 +0,0 @@
import { type CdpTarget } from "@/browser-driver/frame-graph";
import type { ViewportRect } from "../geometry";
import { createCaptureCheckpoint } from "./capture-abort";
import type {
CaptureIssue,
DocumentFacts,
DocumentIdentity,
NodeFacts,
ObservationFacts,
} from "./facts";
import type { FrameOwnedAxNode } from "./frame-document";
import {
EMPTY_VISUAL_CONTEXT,
extendVisualContext,
projectVisualBox,
resolveVisualRegion,
type VisualContext,
type VisualIssueReason,
type VisualRegionResult,
visualProjectionIssue,
} from "./visual-region";
/** Shared observation-time addresses, not cached live geometry. */
export interface VisualFramePath {
readonly document: DocumentIdentity;
readonly parent?: { readonly ownerBackendNodeId: number; readonly frame: VisualFramePath };
}
export interface VisualCandidate {
readonly document: DocumentIdentity;
readonly backendNodeId: number;
readonly parentBackendNodeId: number | null;
readonly label?: string;
/** Absent when execution ancestry could not be established; discovery is still retained. */
readonly framePath?: VisualFramePath;
readonly region: Extract<VisualRegionResult, { status: "available" }>;
}
export type VisualDiscoveryIssue =
| {
readonly reason: "visual-facts-not-collected";
readonly target?: never;
readonly frameId?: never;
readonly backendNodeId?: never;
}
| {
readonly target: CdpTarget;
readonly frameId: string;
readonly backendNodeId?: number;
readonly reason: VisualIssueReason;
};
export interface VisualDiscoveryResult {
readonly candidates: readonly VisualCandidate[];
readonly issues: readonly VisualDiscoveryIssue[];
/** Preserve upstream failures, including those that do not invalidate Canvas geometry. */
readonly captureIssues: readonly CaptureIssue[];
readonly complete: boolean;
}
type Document = DocumentFacts<FrameOwnedAxNode>;
interface NodeContext {
self: VisualContext;
children: VisualContext;
}
function viewportRect(
rect: { x: number; y: number; w: number; h: number } | null | undefined,
): ViewportRect | null {
return rect ? { x: rect.x, y: rect.y, width: rect.w, height: rect.h } : null;
}
/** Client offsets are unscaled local CSS units. Only ordinary, untransformed
* overflow chains consume this adapter; other combinations are rejected by policy. */
function clientBox(node: NodeFacts, document: Document): ViewportRect | null {
const client = node.layout?.clientRect;
const bounds = node.layout?.bounds;
const border =
node.localRect ??
(bounds?.length === 4 && bounds.every(Number.isFinite) && document.geometry
? {
x:
bounds[0] / document.geometry.coordinates.layoutUnitsPerCssPixel -
document.geometry.coordinates.scrollCss.x,
y:
bounds[1] / document.geometry.coordinates.layoutUnitsPerCssPixel -
document.geometry.coordinates.scrollCss.y,
}
: null);
if (
!client ||
client.length !== 4 ||
!client.every(Number.isFinite) ||
client[2] < 0 ||
client[3] < 0 ||
!border ||
!document.geometry
)
return null;
return projectVisualBox(
{ x: border.x + client[0], y: border.y + client[1], width: client[2], height: client[3] },
document.geometry.projections,
);
}
/** Pure Facts consumer: no CDP, ref registration, selection, or semantic keep/drop.
* Both frame and node ancestry are memoized in this call, using iterative walks. */
export async function discoverVisualCandidates(
facts: ObservationFacts<FrameOwnedAxNode>,
signal?: AbortSignal,
): Promise<VisualDiscoveryResult> {
const checkpoint = createCaptureCheckpoint(signal);
if (!facts.visualFactsCollected)
return {
candidates: [],
complete: false,
captureIssues: facts.issues,
issues: [{ reason: "visual-facts-not-collected" }],
};
let work = 0;
const candidates: VisualCandidate[] = [];
const issues: VisualDiscoveryIssue[] = [];
const documents = new Map(facts.documents.map((document) => [document.frame.frameId, document]));
const framePaths = new Map<string, VisualFramePath>();
const contexts = new Map<string, Map<number, NodeContext>>();
const roots = new Map<string, VisualContext>();
async function nodeContext(document: Document, id: number): Promise<NodeContext> {
const cache = contexts.get(document.frame.frameId)!;
const path: NodeFacts[] = [];
let node = document.index.nodes.get(id);
while (node && !cache.has(node.backendNodeId)) {
if (work++ % 256 === 0) {
const pending = checkpoint();
if (pending) await pending;
}
if (
!document.index.ancestryComplete?.get(node.backendNodeId) ||
path.length >= document.index.nodes.size
) {
const incomplete = {
...roots.get(document.frame.frameId)!,
issue: roots.get(document.frame.frameId)!.issue ?? ("ancestry-incomplete" as const),
};
cache.set(node.backendNodeId, { self: incomplete, children: incomplete });
break;
}
path.push(node);
node =
node.parentBackendNodeId === null
? undefined
: document.index.nodes.get(node.parentBackendNodeId);
}
let parent = node
? cache.get(node.backendNodeId)!.children
: roots.get(document.frame.frameId)!;
for (let i = path.length - 1; i >= 0; i--) {
if (work++ % 256 === 0) {
const pending = checkpoint();
if (pending) await pending;
}
const current = path[i];
let self = parent;
let children = parent;
if (document.index.excludedBackendNodeIds.has(current.backendNodeId)) {
self = children = { ...parent, hidden: true };
} else if (current.nodeType === 1) {
if (!current.layout) {
// An element without layout is not evidence that descendants are hidden
// (e.g. a boxless ancestor). Keep the missing style evidence explicit.
self = children = { ...parent, issue: parent.issue ?? "facts-unavailable" };
} else if (document.identity) {
const ancestor = {
document: document.identity,
backendNodeId: current.backendNodeId,
styles: current.layout.styles,
clientBox: clientBox(current, document),
};
children = extendVisualContext(
parent,
ancestor,
current.tag !== "iframe" && current.tag !== "frame",
);
self = current.tag === "canvas" ? extendVisualContext(parent, ancestor, false) : children;
}
}
const result = { self, children };
cache.set(current.backendNodeId, result);
parent = children;
}
return (
cache.get(id) ?? {
self: { ...parent, issue: "ancestry-incomplete" },
children: { ...parent, issue: "ancestry-incomplete" },
}
);
}
// Process parent documents first, independent of snapshot/target completion order.
for (const document of facts.documents) {
const path: Document[] = [];
const visiting = new Set<string>();
let current: Document | undefined = document;
while (current && !roots.has(current.frame.frameId)) {
if (work++ % 256 === 0) {
const pending = checkpoint();
if (pending) await pending;
}
if (visiting.has(current.frame.frameId)) {
roots.set(current.frame.frameId, {
...EMPTY_VISUAL_CONTEXT,
issue: "ownership-unresolved",
});
contexts.set(current.frame.frameId, new Map());
break;
}
visiting.add(current.frame.frameId);
path.push(current);
current = current.frame.parentFrameId
? documents.get(current.frame.parentFrameId)
: undefined;
}
for (let i = path.length - 1; i >= 0; i--) {
const doc = path[i];
const { frame } = doc;
let root: VisualContext = roots.get(frame.frameId) ?? EMPTY_VISUAL_CONTEXT;
if (frame.parentFrameId) {
const parent = documents.get(frame.parentFrameId);
if (
!parent ||
!roots.has(parent.frame.frameId) ||
frame.ownerBackendNodeId === undefined ||
!parent.identity
) {
root = { ...root, issue: "ownership-unresolved" };
} else {
const owner = parent.index.nodes.get(frame.ownerBackendNodeId);
const ownerState = await nodeContext(parent, frame.ownerBackendNodeId);
root = {
...ownerState.children,
// CSS visibility inside a child document cannot override its hidden owner.
hidden:
ownerState.children.hidden ||
owner?.layout?.styles.visibility === "hidden" ||
owner?.layout?.styles.visibility === "collapse",
// Frame scaling is already interpreted by the shared content-quad projection.
transformed: false,
localClip: false,
unpositionedClip: false,
};
}
} else if (frame.frameId !== facts.rootFrameId)
root = { ...root, issue: "ownership-unresolved" };
root = {
...root,
issue:
root.issue ??
(!doc.identity ? "identity-unverified" : visualProjectionIssue(doc.geometry)),
};
if (doc.identity) {
const parentPath = frame.parentFrameId ? framePaths.get(frame.parentFrameId) : undefined;
if (!frame.parentFrameId && frame.frameId === facts.rootFrameId)
framePaths.set(frame.frameId, { document: doc.identity });
else if (parentPath && frame.ownerBackendNodeId !== undefined)
framePaths.set(frame.frameId, {
document: doc.identity,
parent: { ownerBackendNodeId: frame.ownerBackendNodeId, frame: parentPath },
});
}
roots.set(frame.frameId, root);
contexts.set(frame.frameId, new Map());
}
}
for (const document of facts.documents) {
for (const node of document.index.nodes.values()) {
if (work++ % 256 === 0) {
const pending = checkpoint();
if (pending) await pending;
}
if (node.tag !== "canvas" || document.index.excludedBackendNodeIds.has(node.backendNodeId))
continue;
// No layout/zero size is positive evidence of no screenshot area for this node.
if (!node.layout) continue;
const bounds = node.layout.bounds;
if (
bounds?.length === 4 &&
bounds.every(Number.isFinite) &&
(bounds[2] <= 0 || bounds[3] <= 0)
)
continue;
const context = (await nodeContext(document, node.backendNodeId)).self;
const local = viewportRect(node.localRect);
const borderBox =
local && document.geometry ? projectVisualBox(local, document.geometry.projections) : null;
const region = resolveVisualRegion({
borderBox,
frameVisibleBox: viewportRect(node.rect),
context,
visibility: node.layout.styles.visibility,
});
if (region.status === "unavailable") {
issues.push({
target: document.frame.target,
frameId: document.frame.frameId,
backendNodeId: node.backendNodeId,
reason: region.reason,
});
} else if (region.status === "available" && document.identity) {
const label = node.attrs["aria-label"]?.trim() || node.attrs.title?.trim();
candidates.push({
document: document.identity,
framePath: framePaths.get(document.frame.frameId),
backendNodeId: node.backendNodeId,
parentBackendNodeId: node.parentBackendNodeId,
...(label ? { label } : {}),
region,
});
}
}
}
// AX-only/missing documents have no Canvas nodes to carry the missing evidence.
const reportedFrames = new Set(issues.map((issue) => issue.frameId));
for (const document of facts.documents) {
const reason = roots.get(document.frame.frameId)?.issue;
if (
reason &&
!reportedFrames.has(document.frame.frameId) &&
(!document.identity || !document.index.nodes.size)
)
issues.push({ target: document.frame.target, frameId: document.frame.frameId, reason });
}
const pending = checkpoint();
if (pending) await pending;
return {
candidates,
issues,
captureIssues: facts.issues,
complete: issues.length === 0 && facts.issues.length === 0,
};
}
@@ -1,318 +0,0 @@
import {
type GeometryProjection,
type Polygon,
projectPolygon,
rectPolygon,
regionBounds,
type ViewportRect,
} from "../geometry";
import type { DocumentGeometry, DocumentIdentity } from "./facts";
export type VisualIssueReason =
| "ancestry-incomplete"
| "facts-unavailable"
| "geometry-unavailable"
| "geometry-unsupported"
| "identity-unverified"
| "ownership-unresolved";
/** Top viewport CSS coordinates. Unconstrained axes have infinite bounds. */
export interface VisualClip {
readonly left: number;
readonly top: number;
readonly right: number;
readonly bottom: number;
}
/** Shared persistent chain: one entry per clipping ancestor, not per Canvas. */
export interface VisualClipSource {
readonly document: DocumentIdentity;
readonly backendNodeId: number;
readonly x: boolean;
readonly y: boolean;
readonly overflowX: string;
readonly overflowY: string;
readonly box: ViewportRect;
readonly parent?: VisualClipSource;
}
export interface VisualContext {
readonly clip: VisualClip;
readonly clips?: VisualClipSource;
readonly hidden: boolean;
/** A local CSS scale/zoom makes snapshot client offsets ambiguous. */
readonly transformed: boolean;
readonly localClip?: boolean;
/** Overflow between an absolute target and its nearest positioned ancestor. */
readonly unpositionedClip?: boolean;
readonly issue?: VisualIssueReason;
}
export const EMPTY_VISUAL_CONTEXT: VisualContext = {
clip: { left: -Infinity, top: -Infinity, right: Infinity, bottom: Infinity },
hidden: false,
transformed: false,
};
function axisRect(polygon: Polygon): boolean {
if (polygon.length !== 4 || polygon.some((p) => !Number.isFinite(p.x) || !Number.isFinite(p.y)))
return false;
const [a, b, c, d] = polygon;
return a.y === b.y && b.x === c.x && c.y === d.y && d.x === a.x && b.x > a.x && d.y > a.y;
}
/** Check evidence before projectRectToViewport can reduce a polygon to its bounds. */
export function visualProjectionIssue(
geometry: DocumentGeometry | undefined,
): VisualIssueReason | undefined {
if (!geometry || !geometry.projections.length || geometry.pageScale === undefined)
return "geometry-unavailable";
if (!Number.isFinite(geometry.pageScale) || geometry.pageScale <= 0)
return "geometry-unavailable";
if (geometry.pageScale !== 1) return "geometry-unsupported";
const { layoutUnitsPerCssPixel, scrollCss } = geometry.coordinates;
if (
!Number.isFinite(layoutUnitsPerCssPixel) ||
layoutUnitsPerCssPixel <= 0 ||
![scrollCss.x, scrollCss.y].every(Number.isFinite)
)
return "geometry-unavailable";
for (const projection of geometry.projections) {
if (
![projection.topViewport.width, projection.topViewport.height].every(
(n) => Number.isFinite(n) && n > 0,
)
)
return "geometry-unavailable";
if (projection.sourceClips.some((clip) => !axisRect(clip))) return "geometry-unsupported";
for (const edge of projection.edges) {
if (
![edge.sourceViewport.width, edge.sourceViewport.height].every(
(n) => Number.isFinite(n) && n > 0,
)
)
return "geometry-unavailable";
if (!axisRect(edge.destinationQuad) || edge.destinationClips?.some((clip) => !axisRect(clip)))
return "geometry-unsupported";
}
}
}
/** Uses the same projection math as DOM geometry, without clipping away box origins. */
export function projectVisualBox(
box: ViewportRect,
projections: readonly GeometryProjection[],
): ViewportRect | null {
let polygon: Polygon = rectPolygon({ x: box.x, y: box.y, w: box.width, h: box.height });
for (const projection of projections)
for (const edge of projection.edges) polygon = projectPolygon(polygon, edge);
// Preserve zero-size client boxes: they can prove a fully clipped axis.
if (box.width === 0 || box.height === 0) {
const [a, , c] = polygon;
return { x: a.x, y: a.y, width: c.x - a.x, height: c.y - a.y };
}
return regionBounds([polygon]);
}
function axisTransformScale(value: string): [number, number] | null {
if (value === "none") return [1, 1];
const match = /^matrix(3d)?\(([^)]+)\)$/.exec(value);
if (!match) return null;
const values = match[2].split(",").map((part) => Number(part.trim()));
if (!values.every(Number.isFinite)) return null;
if (match[1])
return values.length === 16 &&
values[0] > 0 &&
values[5] > 0 &&
values[10] === 1 &&
values[15] === 1 &&
[1, 2, 3, 4, 6, 7, 8, 9, 11, 14].every((index) => values[index] === 0)
? [values[0], values[5]]
: null;
return values.length === 6 && values[0] > 0 && values[3] > 0 && values[1] === 0 && values[2] === 0
? [values[0], values[3]]
: null;
}
export interface VisualAncestor {
readonly document: DocumentIdentity;
readonly backendNodeId: number;
readonly styles: Readonly<Record<string, string>>;
/** Already in top viewport CSS units, from a source-specific adapter. */
readonly clientBox?: ViewportRect | null;
}
/** One ancestor step, reusable by discovery and the future live local adapter. */
export function extendVisualContext(
parent: VisualContext,
node: VisualAncestor,
clipContents = true,
): VisualContext {
const styles = node.styles;
const opacity = styles.opacity?.trim() ? Number(styles.opacity) : NaN;
const zoom = styles.zoom === "normal" ? 1 : styles.zoom?.trim() ? Number(styles.zoom) : NaN;
const scale =
styles.scale === "none" ? [1] : (styles.scale ?? "").trim().split(/\s+/).map(Number);
const scaleSupported =
scale.length >= 1 &&
scale.length <= 3 &&
scale.every((n) => Number.isFinite(n) && n > 0) &&
(scale.length < 3 || scale[2] === 1);
const transformScale = axisTransformScale(styles.transform ?? "");
// Identity/translation changes origins, which bounds already capture, but
// leaves client offsets in the same units. Only scaling makes them ambiguous.
const transformed =
parent.transformed ||
transformScale?.some((n) => n !== 1) === true ||
zoom !== 1 ||
scale.some((n) => n !== 1);
const hidden = parent.hidden || opacity === 0 || styles.display === "none";
let issue = parent.issue;
if (
!Number.isFinite(opacity) ||
!Number.isFinite(zoom) ||
zoom <= 0 ||
!styles.display ||
!styles.visibility
)
issue ??= "facts-unavailable";
if (!styles.transform || !styles["clip-path"] || !styles["mask-image"])
issue ??= "facts-unavailable";
else if (!transformScale || styles["clip-path"] !== "none" || styles["mask-image"] !== "none")
issue ??= "geometry-unsupported";
if (
!["rotate", "scale", "perspective", "clip", "contain", "overflow-clip-margin"].every(
(key) => styles[key],
)
)
issue ??= "facts-unavailable";
else if (
!scaleSupported ||
!["none", "0deg"].includes(styles.rotate) ||
styles.perspective !== "none" ||
styles.clip !== "auto" ||
/(?:^|\s)(paint|strict|content)(?:$|\s)/.test(styles.contain)
)
issue ??= "geometry-unsupported";
// Do not pretend every DOM ancestor clips out-of-flow descendants. Support
// the ordinary positioned-container case; other containing-block cases stay explicit.
if (
(styles.position === "absolute" && parent.unpositionedClip) ||
(styles.position === "fixed" && parent.localClip)
)
issue ??= "geometry-unsupported";
const overflow = [styles["overflow-x"], styles["overflow-y"]];
if (clipContents && overflow.includes("clip") && styles["overflow-clip-margin"] !== "0px")
issue ??= "geometry-unsupported";
if (
overflow.some(
(value) => !["visible", "hidden", "clip", "scroll", "auto", "overlay"].includes(value),
)
)
issue ??= "facts-unavailable";
const [x, y] = overflow.map((value) => value !== "visible");
let clip = parent.clip;
let clips = parent.clips;
if (clipContents && (x || y)) {
if (transformed) issue ??= "geometry-unsupported";
const box = node.clientBox;
if (
!box ||
![box.x, box.y, box.width, box.height].every(Number.isFinite) ||
box.width < 0 ||
box.height < 0
)
issue ??= "geometry-unavailable";
else {
clip = {
left: x ? Math.max(clip.left, box.x) : clip.left,
right: x ? Math.min(clip.right, box.x + box.width) : clip.right,
top: y ? Math.max(clip.top, box.y) : clip.top,
bottom: y ? Math.min(clip.bottom, box.y + box.height) : clip.bottom,
};
clips = {
document: node.document,
backendNodeId: node.backendNodeId,
x,
y,
overflowX: styles["overflow-x"],
overflowY: styles["overflow-y"],
box,
parent: clips,
};
}
}
const clipsHere = clipContents && (x || y);
return {
hidden,
transformed,
issue,
clip,
clips,
localClip: parent.localClip || clipsHere,
unpositionedClip:
styles.position !== "static" ||
styles.transform !== "none" ||
/(?:^|\s)layout(?:$|\s)/.test(styles.contain ?? "")
? false
: parent.unpositionedClip || clipsHere,
};
}
/** A page screenshot rectangle, not an exact mask of visible Canvas pixels.
* Rounded corners are rendered by the browser and do not invalidate this range. */
export type VisualRegionResult =
| { status: "available"; borderBox: ViewportRect; crop: ViewportRect; clips?: VisualClipSource }
| { status: "empty" }
| { status: "unavailable"; reason: VisualIssueReason };
/** Box and frameVisibleBox must come from the same box kind (Canvas border-box).
* Frame/viewport clipping is supplied by the shared geometry adapter. */
export function resolveVisualRegion(input: {
borderBox: ViewportRect | null;
frameVisibleBox: ViewportRect | null;
context: VisualContext;
visibility: string;
}): VisualRegionResult {
const { borderBox, frameVisibleBox, context, visibility } = input;
if (context.hidden || visibility === "hidden" || visibility === "collapse")
return { status: "empty" };
if (context.issue) return { status: "unavailable", reason: context.issue };
if (visibility !== "visible") return { status: "unavailable", reason: "facts-unavailable" };
if (!borderBox) return { status: "unavailable", reason: "geometry-unavailable" };
if (!frameVisibleBox) return { status: "empty" };
if (
![
borderBox.x,
borderBox.y,
borderBox.width,
borderBox.height,
frameVisibleBox.x,
frameVisibleBox.y,
frameVisibleBox.width,
frameVisibleBox.height,
].every(Number.isFinite)
)
return { status: "unavailable", reason: "geometry-unavailable" };
if (
borderBox.width <= 0 ||
borderBox.height <= 0 ||
frameVisibleBox.width < 0 ||
frameVisibleBox.height < 0 ||
Object.values(context.clip).some(Number.isNaN)
)
return { status: "unavailable", reason: "geometry-unavailable" };
const x = Math.max(frameVisibleBox.x, context.clip.left);
const y = Math.max(frameVisibleBox.y, context.clip.top);
const right = Math.min(frameVisibleBox.x + frameVisibleBox.width, context.clip.right);
const bottom = Math.min(frameVisibleBox.y + frameVisibleBox.height, context.clip.bottom);
return right > x && bottom > y
? {
status: "available",
borderBox,
crop: { x, y, width: right - x, height: bottom - y },
clips: context.clips,
}
: { status: "empty" };
}
-56
View File
@@ -24,9 +24,6 @@ export type RpcErrorReason =
| "agent_window_scope"
| "element_not_visible"
| "ref_not_found"
| "ref_kind_unsupported"
| "visual_target_changed"
| "visual_pixel_budget_exceeded"
| "selector_not_found"
| "target_not_fillable"
| "fill_value_invalid"
@@ -473,59 +470,6 @@ export interface HoverResult {
dialogs?: JavaScriptDialogInfo[];
}
export interface ScrollToParams {
session_id: string;
ref?: string;
selector?: string;
tab_id?: number;
timeout_ms?: number;
}
export interface ScrollToResult {
tab_id: number;
used_ref?: string;
used_selector?: string;
/** Clipped border-box bounds in top-level viewport CSS pixels; not an occlusion test. */
x: number;
y: number;
width: number;
height: number;
dialogs?: JavaScriptDialogInfo[];
}
export interface FocusParams {
session_id: string;
ref?: string;
selector?: string;
tab_id?: number;
timeout_ms?: number;
}
export interface FocusResult {
tab_id: number;
used_ref?: string;
used_selector?: string;
focused: boolean;
dialogs?: JavaScriptDialogInfo[];
}
export interface BlurParams {
session_id: string;
ref?: string;
selector?: string;
tab_id?: number;
timeout_ms?: number;
}
export interface BlurResult {
tab_id: number;
used_ref?: string;
used_selector?: string;
was_focused: boolean;
focused: boolean;
dialogs?: JavaScriptDialogInfo[];
}
export interface FillParams {
session_id: string;
value: string;
+2 -12
View File
@@ -55,20 +55,10 @@ Use this default loop:
```text
bsk navigate <url> --session <id>
bsk observe --session <id>
bsk click|hover|scroll-to|focus|blur|fill|select|press ... --session <id>
bsk click|hover|fill|select|press ... --session <id>
bsk observe --session <id> # after navigation or a meaningful DOM change
```
`bsk scroll-to <ref-or-selector> --session <id>` scrolls an element and its frame owners into view.
Use a fresh element ref for iframe/shadow-root targets; CSS selectors search the main document.
The result is the visible border-box portion's bounds in top-level viewport CSS pixels after
ancestor clipping. Partial visibility is enough; hidden or fully clipped targets fail with
`permission_denied` and `data.reason=element_not_visible`. This does not test occlusion by other elements.
For a specific tab or deadline: `bsk scroll-to @e3 --session <id> --tab-id 42 --timeout 5s`.
`bsk focus <ref>` explicitly focuses a target; `bsk blur <ref>` removes focus and reports whether
it was focused. Use these for UI states triggered by focus changes.
Prefer fresh `@eN` refs over CSS selectors. Navigation invalidates refs; large DOM changes may also
make them stale. Observe again before the next interaction.
@@ -124,7 +114,7 @@ This list of names is complete. Never invent a command outside it; read
session start|stop|list browsers status doctor update logs
navigate navigate-back navigate-forward reload wait-for-navigation wait-ms
observe snapshot get-html screenshot console network
click hover scroll-to focus blur fill select press evaluate
click hover fill select press evaluate
tab list|create|close|select|borrow|return window resize emulate
upload download request-help record start|stop
```
+3 -123
View File
@@ -1,5 +1,4 @@
//! `bsk click` / `bsk focus` / `bsk blur` / `bsk fill` / `bsk press`
//! interaction tools.
//! `bsk click` / `bsk fill` / `bsk press` (M7 interaction tools).
//!
//! The `<target>` positional accepts either a snapshot ref (`@e3`,
//! `e3`) or a CSS selector. Use `--ref` / `--selector` explicitly when
@@ -11,9 +10,8 @@ use std::time::Duration;
use anyhow::Context;
use bsk_protocol::Method;
use bsk_protocol::tools::{
BlurParams, BlurResult, ClickParams, ClickResult, FillParams, FillResult, FocusParams,
FocusResult, HoverParams, HoverResult, KeyModifier, MouseButton, PressParams, PressResult,
SelectParams, SelectResult,
ClickParams, ClickResult, FillParams, FillResult, HoverParams, HoverResult, KeyModifier,
MouseButton, PressParams, PressResult, SelectParams, SelectResult,
};
use clap::{Args, ValueEnum};
@@ -276,124 +274,6 @@ pub fn dispatch_hover(args: HoverArgs, format: Format) -> Result<(), CliError> {
Ok(())
}
// ---------------------------------------------------------------------------
// bsk focus / bsk blur
// ---------------------------------------------------------------------------
#[derive(Debug, Clone, Args)]
pub struct FocusArgs {
/// Snapshot ref (`@e3`, `e3`) or CSS selector. Optional when `--ref`/`--selector` is used.
pub target: Option<String>,
#[arg(long = "ref")]
pub ref_: Option<String>,
#[arg(long = "selector")]
pub selector: Option<String>,
#[arg(long)]
pub session: String,
#[arg(long = "tab-id")]
pub tab_id: Option<i64>,
#[arg(long, default_value = "30s", value_parser = parse_timeout_ms)]
pub timeout: u32,
}
pub fn dispatch_focus(args: FocusArgs, format: Format) -> Result<(), CliError> {
let info = ensure_daemon().context("ensure daemon is running")?;
let (ref_, selector) = split_target(args.target, args.ref_, args.selector)?;
let params = FocusParams {
session_id: args.session,
ref_,
selector,
tab_id: args.tab_id,
timeout_ms: Some(args.timeout),
};
let reply: FocusResult = call(
info.sock_path,
Method::ToolFocus,
params,
"focus-1",
args.timeout,
)?;
match format {
Format::Json => println!(
"{}",
serde_json::to_string_pretty(&reply)
.map_err(|e| CliError::Local(anyhow::anyhow!(e)))?
),
Format::Human => {
let target =
format_used_target(reply.used_ref.as_deref(), reply.used_selector.as_deref());
println!(
"focus ok tab={} target={target} focused={}",
reply.tab_id, reply.focused
);
print_dialog_summaries(&reply.dialogs);
}
}
Ok(())
}
#[derive(Debug, Clone, Args)]
pub struct BlurArgs {
/// Snapshot ref (`@e3`, `e3`) or CSS selector. Optional when `--ref`/`--selector` is used.
pub target: Option<String>,
#[arg(long = "ref")]
pub ref_: Option<String>,
#[arg(long = "selector")]
pub selector: Option<String>,
#[arg(long)]
pub session: String,
#[arg(long = "tab-id")]
pub tab_id: Option<i64>,
#[arg(long, default_value = "30s", value_parser = parse_timeout_ms)]
pub timeout: u32,
}
pub fn dispatch_blur(args: BlurArgs, format: Format) -> Result<(), CliError> {
let info = ensure_daemon().context("ensure daemon is running")?;
let (ref_, selector) = split_target(args.target, args.ref_, args.selector)?;
let params = BlurParams {
session_id: args.session,
ref_,
selector,
tab_id: args.tab_id,
timeout_ms: Some(args.timeout),
};
let reply: BlurResult = call(
info.sock_path,
Method::ToolBlur,
params,
"blur-1",
args.timeout,
)?;
match format {
Format::Json => println!(
"{}",
serde_json::to_string_pretty(&reply)
.map_err(|e| CliError::Local(anyhow::anyhow!(e)))?
),
Format::Human => {
let target =
format_used_target(reply.used_ref.as_deref(), reply.used_selector.as_deref());
println!(
"blur ok tab={} target={target} was_focused={} focused={}",
reply.tab_id, reply.was_focused, reply.focused
);
print_dialog_summaries(&reply.dialogs);
}
}
Ok(())
}
// ---------------------------------------------------------------------------
// bsk fill
// ---------------------------------------------------------------------------
+1 -15
View File
@@ -28,7 +28,6 @@ pub mod record_recovery;
pub mod record_state;
pub mod render_error;
pub mod screenshot;
pub mod scroll;
pub mod session;
pub mod snapshot;
pub mod status;
@@ -48,15 +47,12 @@ use crate::cli::evaluate::EvaluateArgs;
use crate::cli::get_html::GetHtmlArgs;
use crate::cli::human_loop::RequestHelpArgs;
use crate::cli::install_skill::InstallSkillArgs;
use crate::cli::interaction::{
BlurArgs, ClickArgs, FillArgs, FocusArgs, HoverArgs, PressArgs, SelectArgs,
};
use crate::cli::interaction::{ClickArgs, FillArgs, HoverArgs, PressArgs, SelectArgs};
use crate::cli::navigate::{NavigateCommand, NavigateHistoryArgs, ReloadArgs};
use crate::cli::network::NetworkArgs;
use crate::cli::observe::ObserveArgs;
use crate::cli::record::RecordCmd;
use crate::cli::screenshot::ScreenshotArgs;
use crate::cli::scroll::ScrollToArgs;
use crate::cli::session::SessionCmd;
use crate::cli::snapshot::SnapshotArgs;
use crate::cli::tab::TabCmd;
@@ -177,16 +173,6 @@ pub enum Command {
/// Hover a snapshot ref or CSS selector.
Hover(HoverArgs),
/// Scroll a snapshot ref or CSS selector into the visible viewport.
#[command(name = "scroll-to")]
ScrollTo(ScrollToArgs),
/// Focus a snapshot ref or CSS selector.
Focus(FocusArgs),
/// Remove focus from a snapshot ref or CSS selector.
Blur(BlurArgs),
/// Fill an input / textarea / contenteditable.
Fill(FillArgs),
-41
View File
@@ -40,9 +40,6 @@ use bsk_protocol::ErrorCode;
pub mod reason {
pub const AGENT_WINDOW_SCOPE: &str = "agent_window_scope";
pub const ELEMENT_NOT_VISIBLE: &str = "element_not_visible";
pub const VISUAL_TARGET_CHANGED: &str = "visual_target_changed";
pub const VISUAL_PIXEL_BUDGET_EXCEEDED: &str = "visual_pixel_budget_exceeded";
pub const REF_KIND_UNSUPPORTED: &str = "ref_kind_unsupported";
pub const REF_NOT_FOUND: &str = "ref_not_found";
pub const SELECTOR_NOT_FOUND: &str = "selector_not_found";
pub const TARGET_NOT_FILLABLE: &str = "target_not_fillable";
@@ -250,25 +247,6 @@ pub fn info_for_error(code: ErrorCode, data: Option<&serde_json::Value>) -> Rend
),
exit_code: base.exit_code,
},
(ErrorCode::NotFound, reason::VISUAL_TARGET_CHANGED) => RenderInfo {
summary: "visual target needs a fresh observation",
hint: Some(
"observe the page again and use the current visual ref; the previous identity or region cannot be used",
),
exit_code: base.exit_code,
},
(ErrorCode::CdpFailed, reason::VISUAL_PIXEL_BUDGET_EXCEEDED) => RenderInfo {
summary: "visual screenshot exceeds the image size limit",
hint: Some("the screenshot could not be reduced within the pixel budget"),
exit_code: base.exit_code,
},
(ErrorCode::Unsupported, reason::REF_KIND_UNSUPPORTED) => RenderInfo {
summary: "this tool does not support the ref target type",
hint: Some(
"use a DOM ref for DOM operations; visual-region refs require an available visual screenshot path",
),
exit_code: base.exit_code,
},
(ErrorCode::NotFound, reason::REF_NOT_FOUND) => RenderInfo {
summary: "snapshot ref was not found for this tab",
hint: Some("rerun `bsk snapshot` for the current tab and use one of the returned refs"),
@@ -686,25 +664,6 @@ mod tests {
assert!(info.summary.contains("timed out after browser dispatch"));
}
#[test]
fn visual_screenshot_failures_have_specific_guidance() {
let changed = serde_json::json!({ "reason": reason::VISUAL_TARGET_CHANGED });
let info = info_for_error(ErrorCode::NotFound, Some(&changed));
assert!(info.hint.unwrap().contains("observe"));
let budget = serde_json::json!({ "reason": reason::VISUAL_PIXEL_BUDGET_EXCEEDED });
let info = info_for_error(ErrorCode::CdpFailed, Some(&budget));
assert!(info.summary.contains("image size limit"));
}
#[test]
fn visual_ref_type_error_has_specific_guidance() {
let data = serde_json::json!({ "reason": reason::REF_KIND_UNSUPPORTED });
let info = info_for_error(ErrorCode::Unsupported, Some(&data));
assert!(info.summary.contains("ref target type"));
assert!(info.hint.unwrap().contains("DOM ref"));
assert_eq!(info.exit_code, info_for(ErrorCode::Unsupported).exit_code);
}
#[test]
fn ref_not_found_overrides_not_found_copy() {
let data = serde_json::json!({ "reason": reason::REF_NOT_FOUND });
-100
View File
@@ -1,100 +0,0 @@
//! `bsk scroll-to` — bring a target element into the visible viewport.
use std::time::Duration;
use anyhow::Context;
use bsk_protocol::Method;
use bsk_protocol::tools::{ScrollToParams, ScrollToResult};
use clap::Args;
use crate::cli::dialogs::print_dialog_summaries;
use crate::cli::ensure_daemon::ensure_daemon;
use crate::cli::error::{CliError, Format};
use crate::cli::interaction::split_target;
use crate::cli::navigate::parse_timeout_ms;
#[derive(Debug, Clone, Args)]
pub struct ScrollToArgs {
/// Snapshot ref (`@e3`, `e3`) or CSS selector. Optional when `--ref`/`--selector` is used.
pub target: Option<String>,
#[arg(long = "ref")]
pub ref_: Option<String>,
#[arg(long = "selector")]
pub selector: Option<String>,
#[arg(long)]
pub session: String,
#[arg(long = "tab-id")]
pub tab_id: Option<i64>,
#[arg(long, default_value = "30s", value_parser = parse_timeout_ms)]
pub timeout: u32,
}
pub fn dispatch(args: ScrollToArgs, format: Format) -> Result<(), CliError> {
let (ref_, selector) = split_target(args.target, args.ref_, args.selector)?;
let info = ensure_daemon().context("ensure daemon is running")?;
let params = ScrollToParams {
session_id: args.session,
ref_,
selector,
tab_id: args.tab_id,
timeout_ms: Some(args.timeout),
};
let reply: ScrollToResult = crate::cli::business_rpc::call(
info.sock_path,
"scroll-to-1",
Method::ToolScrollTo,
Some(params),
ipc_timeout(args.timeout),
)?;
match format {
Format::Json => println!(
"{}",
serde_json::to_string_pretty(&reply)
.map_err(|e| CliError::Local(anyhow::anyhow!(e)))?
),
Format::Human => {
let target =
format_used_target(reply.used_ref.as_deref(), reply.used_selector.as_deref());
println!(
"scroll-to ok tab={} target={target} bounds=({}, {}, {}, {})",
reply.tab_id, reply.x, reply.y, reply.width, reply.height
);
print_dialog_summaries(&reply.dialogs);
}
}
Ok(())
}
fn format_used_target(used_ref: Option<&str>, used_selector: Option<&str>) -> String {
used_ref
.map(|r| format!("@{r}"))
.or_else(|| used_selector.map(str::to_string))
.unwrap_or_else(|| "?".into())
}
fn ipc_timeout(timeout_ms: u32) -> Duration {
Duration::from_millis(u64::from(timeout_ms))
.checked_add(Duration::from_secs(15))
.unwrap_or(Duration::from_secs(u64::from(timeout_ms / 1_000) + 15))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn split_target_detects_refs_and_selectors() {
let (ref_, selector) = split_target(Some("@e3".into()), None, None).unwrap();
assert_eq!(ref_.as_deref(), Some("@e3"));
assert!(selector.is_none());
let (ref_, selector) = split_target(Some("#target".into()), None, None).unwrap();
assert!(ref_.is_none());
assert_eq!(selector.as_deref(), Some("#target"));
}
}
-3
View File
@@ -253,9 +253,6 @@ pub fn full_handler(status: DaemonStatus, state: Arc<DaemonState>) -> RpcHandler
| Method::ToolReload
| Method::ToolClick
| Method::ToolHover
| Method::ToolScrollTo
| Method::ToolFocus
| Method::ToolBlur
| Method::ToolFill
| Method::ToolPress
| Method::ToolSelect
-3
View File
@@ -94,9 +94,6 @@ fn dispatch(cli: Cli, format: Format) -> Result<(), CliError> {
Command::Reload(args) => cli::navigate::dispatch_reload(args, format),
Command::Click(args) => cli::interaction::dispatch_click(args, format),
Command::Hover(args) => cli::interaction::dispatch_hover(args, format),
Command::ScrollTo(args) => cli::scroll::dispatch(args, format),
Command::Focus(args) => cli::interaction::dispatch_focus(args, format),
Command::Blur(args) => cli::interaction::dispatch_blur(args, format),
Command::Fill(args) => cli::interaction::dispatch_fill(args, format),
Command::Press(args) => cli::interaction::dispatch_press(args, format),
Command::Select(args) => cli::interaction::dispatch_select(args, format),
-59
View File
@@ -255,65 +255,6 @@ fn parses_hover_with_settle() {
assert_eq!(args.settle, 300);
}
#[test]
fn parses_scroll_to_target() {
let cli = parse(&["bsk", "scroll-to", "@e2", "--session", "s1"]);
let Command::ScrollTo(args) = cli.command else {
panic!("expected scroll-to command");
};
assert_eq!(args.target.as_deref(), Some("@e2"));
}
#[test]
fn parses_scroll_to_explicit_target_tab_and_timeout() {
for flag in ["--ref", "--selector"] {
let cli = parse(&[
"bsk",
"scroll-to",
flag,
"e3",
"--session",
"s1",
"--tab-id",
"42",
"--timeout",
"5s",
]);
let Command::ScrollTo(args) = cli.command else {
panic!("expected scroll-to command");
};
assert_eq!(args.tab_id, Some(42));
assert_eq!(args.timeout, 5_000);
assert_eq!(
if flag == "--ref" {
args.ref_
} else {
args.selector
}
.as_deref(),
Some("e3")
);
}
}
#[test]
fn parses_focus_target() {
let cli = parse(&["bsk", "focus", "@e2", "--session", "s1"]);
let Command::Focus(args) = cli.command else {
panic!("expected focus command");
};
assert_eq!(args.target.as_deref(), Some("@e2"));
}
#[test]
fn parses_blur_selector() {
let cli = parse(&["bsk", "blur", "--selector", "#search", "--session", "s1"]);
let Command::Blur(args) = cli.command else {
panic!("expected blur command");
};
assert_eq!(args.selector.as_deref(), Some("#search"));
}
#[test]
fn rejects_zero_click_count() {
assert!(
+13 -23
View File
@@ -293,15 +293,6 @@ async fn session_user_interrupt_event_cancels_inflight_with_user_aborted() {
/// catch a tool dispatched arbitrarily later.
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn user_interrupt_rejects_next_mutating_tool_call_when_session_was_idle() {
assert_idle_interrupt_rejects(Method::ToolClick).await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn user_interrupt_rejects_scroll_to_before_forwarding() {
assert_idle_interrupt_rejects(Method::ToolScrollTo).await;
}
async fn assert_idle_interrupt_rejects(method: Method) {
let (handle, sock) = spawn_daemon().await;
let mut ws = connect_ext(handle.ws_addr()).await;
let _ = handshake_as_ext(&mut ws).await;
@@ -310,11 +301,10 @@ async fn assert_idle_interrupt_rejects(method: Method) {
// Fake extension: answer tool.session_start, observe other
// requests but never reply. We expect the daemon to reject the
// tool call BEFORE it ever forwards a browser-input frame.
// tool call BEFORE it ever forwards a tool.click frame.
let ws_sink_for_responder = Arc::clone(&ws_sink);
let observed_call_count = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let observed_call_count_clone = Arc::clone(&observed_call_count);
let forwarded_method = method.clone();
let observed_click_count = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let observed_click_count_clone = Arc::clone(&observed_click_count);
let responder = tokio::spawn(async move {
let mut ws_stream = ws_stream;
while let Some(Ok(msg)) = ws_stream.next().await {
@@ -339,8 +329,8 @@ async fn assert_idle_interrupt_rejects(method: Method) {
g.send(Message::Text(serde_json::to_string(&reply).unwrap()))
.await
.unwrap();
} else if req.method == forwarded_method {
observed_call_count_clone.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
} else if req.method == Method::ToolClick {
observed_click_count_clone.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
// Deliberately do NOT reply — daemon should reject before
// ever forwarding to us.
let _ = req;
@@ -387,12 +377,12 @@ async fn assert_idle_interrupt_rejects(method: Method) {
let state = handle.state();
wait_for_session_interrupt_pending(&state, &start.session_id).await;
// The CLI now issues a browser-input (mutating). The daemon must
// The CLI now issues a tool.click (mutating). The daemon must
// reject it WITHOUT forwarding to the extension.
let outcome = ipc
let click_outcome = ipc
.call::<_, serde_json::Value>(
"input-after-interrupt",
method,
"click-after-interrupt",
Method::ToolClick,
Some(json!({
"session_id": start.session_id,
"ref": "fake-ref-1",
@@ -401,7 +391,7 @@ async fn assert_idle_interrupt_rejects(method: Method) {
)
.await
.unwrap();
let err = outcome.expect_err("browser-input must be rejected");
let err = click_outcome.expect_err("tool.click must be rejected");
assert_eq!(
err.code,
ErrorCode::UserAborted,
@@ -409,11 +399,11 @@ async fn assert_idle_interrupt_rejects(method: Method) {
err
);
// Crucially: the extension never saw a browser-input frame.
// Crucially: the extension never saw a tool.click frame.
assert_eq!(
observed_call_count.load(std::sync::atomic::Ordering::SeqCst),
observed_click_count.load(std::sync::atomic::Ordering::SeqCst),
0,
"browser-input must NOT have been forwarded to the extension"
"tool.click must NOT have been forwarded to the extension"
);
responder.abort();
+6 -138
View File
@@ -1,4 +1,5 @@
//! M7 integration: drive interaction and navigation tool RPCs
//! M7 integration: drive the 7 new tool RPCs (navigate /
//! navigate_back / navigate_forward / reload / click / fill / press)
//! through the IPC + per-session queue + fake extension and assert
//! the wire shapes line up. The extension stub mirrors each method's
//! `*Result` so we exercise the daemon's serialise → forward →
@@ -12,13 +13,11 @@ use bsk::daemon::{self, DaemonConfig};
use bsk::ipc_client::IpcClient;
use bsk_protocol::system::{HandshakeParams, HandshakeResult};
use bsk_protocol::tools::{
BlurParams, BlurResult, ClickParams, ClickResult, FillParams, FillResult, FocusParams,
FocusResult, KeyModifier, MouseButton, NavigateBackParams, NavigateBackResult,
NavigateForwardParams, NavigateForwardResult, NavigateParams, NavigateResult, PressParams,
PressResult, ReloadParams, ReloadResult, SelectParams, SelectResult, SessionStartParams,
SessionStartResult, WaitUntil,
ClickParams, ClickResult, FillParams, FillResult, KeyModifier, MouseButton, NavigateBackParams,
NavigateBackResult, NavigateForwardParams, NavigateForwardResult, NavigateParams,
NavigateResult, PressParams, PressResult, ReloadParams, ReloadResult, SelectParams,
SelectResult, SessionStartParams, SessionStartResult, WaitUntil,
};
use bsk_protocol::tools::{ScrollToParams, ScrollToResult};
use bsk_protocol::{
BrowserPeerInfo, ErrorCode, Frame, Method, RequestFrame, ResponseBody, ResponseFrame, RpcError,
};
@@ -383,137 +382,6 @@ async fn click_round_trips_ref_and_modifiers() {
handle.shutdown().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn focus_round_trips_verified_state() {
let (handle, sock) = spawn_daemon().await;
let mut ws = connect_ext(handle.ws_addr()).await;
let _ = do_handshake(&mut ws).await;
run_extension(ws, |req| {
assert_eq!(req.method, Method::ToolFocus);
let params = req.params.clone().unwrap();
let p: FocusParams = serde_json::from_value(params).unwrap();
assert_eq!(p.ref_.as_deref(), Some("@e3"));
ResponseBody::Ok(
serde_json::to_value(FocusResult {
tab_id: 9,
used_ref: Some("e3".into()),
used_selector: None,
focused: true,
dialogs: vec![],
})
.unwrap(),
)
});
let session_id = ipc_session_start(&sock).await;
let result: FocusResult = ipc_tool_call(
&sock,
Method::ToolFocus,
FocusParams {
session_id,
ref_: Some("@e3".into()),
selector: None,
tab_id: None,
timeout_ms: Some(5_000),
},
)
.await
.expect("focus ok");
assert!(result.focused);
assert_eq!(result.used_ref.as_deref(), Some("e3"));
handle.shutdown().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn blur_round_trips_previous_focus_state() {
let (handle, sock) = spawn_daemon().await;
let mut ws = connect_ext(handle.ws_addr()).await;
let _ = do_handshake(&mut ws).await;
run_extension(ws, |req| {
assert_eq!(req.method, Method::ToolBlur);
let params = req.params.clone().unwrap();
let p: BlurParams = serde_json::from_value(params).unwrap();
assert_eq!(p.selector.as_deref(), Some("#search"));
ResponseBody::Ok(
serde_json::to_value(BlurResult {
tab_id: 9,
used_ref: None,
used_selector: Some("#search".into()),
was_focused: true,
focused: false,
dialogs: vec![],
})
.unwrap(),
)
});
let session_id = ipc_session_start(&sock).await;
let result: BlurResult = ipc_tool_call(
&sock,
Method::ToolBlur,
BlurParams {
session_id,
ref_: None,
selector: Some("#search".into()),
tab_id: None,
timeout_ms: Some(5_000),
},
)
.await
.expect("blur ok");
assert!(result.was_focused);
assert!(!result.focused);
assert_eq!(result.used_selector.as_deref(), Some("#search"));
handle.shutdown().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn scroll_to_round_trips_visible_bounds() {
let (handle, sock) = spawn_daemon().await;
let mut ws = connect_ext(handle.ws_addr()).await;
let _ = do_handshake(&mut ws).await;
run_extension(ws, |req| {
assert_eq!(req.method, Method::ToolScrollTo);
let params = req.params.clone().unwrap();
let parsed: ScrollToParams = serde_json::from_value(params).unwrap();
assert_eq!(parsed.ref_.as_deref(), Some("@e3"));
ResponseBody::Ok(
serde_json::to_value(ScrollToResult {
tab_id: 9,
used_ref: Some("e3".into()),
used_selector: None,
x: 10.0,
y: 20.0,
width: 100.0,
height: 40.0,
dialogs: vec![],
})
.unwrap(),
)
});
let session_id = ipc_session_start(&sock).await;
let result: ScrollToResult = ipc_tool_call(
&sock,
Method::ToolScrollTo,
ScrollToParams {
session_id,
ref_: Some("@e3".into()),
selector: None,
tab_id: None,
timeout_ms: Some(5_000),
},
)
.await
.expect("scroll-to ok");
assert_eq!(result.used_ref.as_deref(), Some("e3"));
assert_eq!(result.width, 100.0);
handle.shutdown().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn fill_round_trips_clear_before_default() {
let (handle, sock) = spawn_daemon().await;
@@ -1,42 +0,0 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "BlurParams",
"type": "object",
"required": [
"session_id"
],
"properties": {
"ref": {
"description": "Optional `@e<N>` ref allocated by the last observation. Mutually exclusive with `selector`.",
"type": [
"string",
"null"
]
},
"selector": {
"type": [
"string",
"null"
]
},
"session_id": {
"type": "string"
},
"tab_id": {
"description": "Target tab. Defaults to the Agent Window's active tab.",
"type": [
"integer",
"null"
],
"format": "int64"
},
"timeout_ms": {
"type": [
"integer",
"null"
],
"format": "uint32",
"minimum": 1.0
}
}
}
@@ -1,112 +0,0 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "BlurResult",
"type": "object",
"required": [
"focused",
"tab_id",
"was_focused"
],
"properties": {
"dialogs": {
"type": "array",
"items": {
"$ref": "#/definitions/JavaScriptDialogInfo"
}
},
"focused": {
"description": "Always false for a successful blur; included for machine verification.",
"type": "boolean"
},
"tab_id": {
"type": "integer",
"format": "int64"
},
"used_ref": {
"type": [
"string",
"null"
]
},
"used_selector": {
"type": [
"string",
"null"
]
},
"was_focused": {
"description": "Whether the target held focus before the call.",
"type": "boolean"
}
},
"definitions": {
"JavaScriptDialogHandledAction": {
"description": "How the extension resolved the dialog so CDP could continue.",
"type": "string",
"enum": [
"accepted",
"dismissed"
]
},
"JavaScriptDialogInfo": {
"description": "One observed + handled JavaScript dialog during a tool call.",
"type": "object",
"required": [
"handled",
"message",
"sequence",
"tab_id",
"type"
],
"properties": {
"default_prompt": {
"type": [
"string",
"null"
]
},
"handled": {
"$ref": "#/definitions/JavaScriptDialogHandledAction"
},
"has_browser_handler": {
"type": [
"boolean",
"null"
]
},
"message": {
"type": "string"
},
"sequence": {
"description": "Monotonic per-tab sequence for ordering within a session.",
"type": "integer",
"format": "uint64",
"minimum": 0.0
},
"tab_id": {
"type": "integer",
"format": "int64"
},
"type": {
"$ref": "#/definitions/JavaScriptDialogType"
},
"url": {
"type": [
"string",
"null"
]
}
}
},
"JavaScriptDialogType": {
"description": "Native JS dialog kind reported by CDP.",
"type": "string",
"enum": [
"alert",
"confirm",
"prompt",
"beforeunload"
]
}
}
}
@@ -1,42 +0,0 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "FocusParams",
"type": "object",
"required": [
"session_id"
],
"properties": {
"ref": {
"description": "Optional `@e<N>` ref allocated by the last observation. Mutually exclusive with `selector`.",
"type": [
"string",
"null"
]
},
"selector": {
"type": [
"string",
"null"
]
},
"session_id": {
"type": "string"
},
"tab_id": {
"description": "Target tab. Defaults to the Agent Window's active tab.",
"type": [
"integer",
"null"
],
"format": "int64"
},
"timeout_ms": {
"type": [
"integer",
"null"
],
"format": "uint32",
"minimum": 1.0
}
}
}
@@ -1,107 +0,0 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "FocusResult",
"type": "object",
"required": [
"focused",
"tab_id"
],
"properties": {
"dialogs": {
"type": "array",
"items": {
"$ref": "#/definitions/JavaScriptDialogInfo"
}
},
"focused": {
"description": "Whether the target holds DOM focus in its document or shadow root after the call.",
"type": "boolean"
},
"tab_id": {
"type": "integer",
"format": "int64"
},
"used_ref": {
"type": [
"string",
"null"
]
},
"used_selector": {
"type": [
"string",
"null"
]
}
},
"definitions": {
"JavaScriptDialogHandledAction": {
"description": "How the extension resolved the dialog so CDP could continue.",
"type": "string",
"enum": [
"accepted",
"dismissed"
]
},
"JavaScriptDialogInfo": {
"description": "One observed + handled JavaScript dialog during a tool call.",
"type": "object",
"required": [
"handled",
"message",
"sequence",
"tab_id",
"type"
],
"properties": {
"default_prompt": {
"type": [
"string",
"null"
]
},
"handled": {
"$ref": "#/definitions/JavaScriptDialogHandledAction"
},
"has_browser_handler": {
"type": [
"boolean",
"null"
]
},
"message": {
"type": "string"
},
"sequence": {
"description": "Monotonic per-tab sequence for ordering within a session.",
"type": "integer",
"format": "uint64",
"minimum": 0.0
},
"tab_id": {
"type": "integer",
"format": "int64"
},
"type": {
"$ref": "#/definitions/JavaScriptDialogType"
},
"url": {
"type": [
"string",
"null"
]
}
}
},
"JavaScriptDialogType": {
"description": "Native JS dialog kind reported by CDP.",
"type": "string",
"enum": [
"alert",
"confirm",
"prompt",
"beforeunload"
]
}
}
}
@@ -1,42 +0,0 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "ScrollToParams",
"type": "object",
"required": [
"session_id"
],
"properties": {
"ref": {
"description": "Optional `@e<N>` ref allocated by the last observation. Mutually exclusive with `selector`.",
"type": [
"string",
"null"
]
},
"selector": {
"type": [
"string",
"null"
]
},
"session_id": {
"type": "string"
},
"tab_id": {
"description": "Target tab. Defaults to the Agent Window's active tab.",
"type": [
"integer",
"null"
],
"format": "int64"
},
"timeout_ms": {
"type": [
"integer",
"null"
],
"format": "uint32",
"minimum": 1.0
}
}
}
@@ -1,123 +0,0 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "ScrollToResult",
"type": "object",
"required": [
"height",
"tab_id",
"width",
"x",
"y"
],
"properties": {
"dialogs": {
"type": "array",
"items": {
"$ref": "#/definitions/JavaScriptDialogInfo"
}
},
"height": {
"type": "number",
"format": "double"
},
"tab_id": {
"type": "integer",
"format": "int64"
},
"used_ref": {
"type": [
"string",
"null"
]
},
"used_selector": {
"type": [
"string",
"null"
]
},
"width": {
"type": "number",
"format": "double"
},
"x": {
"description": "Bounding rectangle of the element's visible border-box portion, in top-level viewport CSS pixels, after ancestor and viewport clipping. Partial visibility is sufficient. This is not an occlusion or hit test.",
"type": "number",
"format": "double"
},
"y": {
"type": "number",
"format": "double"
}
},
"definitions": {
"JavaScriptDialogHandledAction": {
"description": "How the extension resolved the dialog so CDP could continue.",
"type": "string",
"enum": [
"accepted",
"dismissed"
]
},
"JavaScriptDialogInfo": {
"description": "One observed + handled JavaScript dialog during a tool call.",
"type": "object",
"required": [
"handled",
"message",
"sequence",
"tab_id",
"type"
],
"properties": {
"default_prompt": {
"type": [
"string",
"null"
]
},
"handled": {
"$ref": "#/definitions/JavaScriptDialogHandledAction"
},
"has_browser_handler": {
"type": [
"boolean",
"null"
]
},
"message": {
"type": "string"
},
"sequence": {
"description": "Monotonic per-tab sequence for ordering within a session.",
"type": "integer",
"format": "uint64",
"minimum": 0.0
},
"tab_id": {
"type": "integer",
"format": "int64"
},
"type": {
"$ref": "#/definitions/JavaScriptDialogType"
},
"url": {
"type": [
"string",
"null"
]
}
}
},
"JavaScriptDialogType": {
"description": "Native JS dialog kind reported by CDP.",
"type": "string",
"enum": [
"alert",
"confirm",
"prompt",
"beforeunload"
]
}
}
}
@@ -79,12 +79,6 @@ fn main() {
dump!(ClickResult, "tool_click_result");
dump!(HoverParams, "tool_hover_params");
dump!(HoverResult, "tool_hover_result");
dump!(ScrollToParams, "tool_scroll_to_params");
dump!(ScrollToResult, "tool_scroll_to_result");
dump!(FocusParams, "tool_focus_params");
dump!(FocusResult, "tool_focus_result");
dump!(BlurParams, "tool_blur_params");
dump!(BlurResult, "tool_blur_result");
dump!(FillParams, "tool_fill_params");
dump!(FillResult, "tool_fill_result");
dump!(PressParams, "tool_press_params");
-18
View File
@@ -72,12 +72,6 @@ pub enum Method {
ToolClick,
#[serde(rename = "tool.hover")]
ToolHover,
#[serde(rename = "tool.scroll_to")]
ToolScrollTo,
#[serde(rename = "tool.focus")]
ToolFocus,
#[serde(rename = "tool.blur")]
ToolBlur,
#[serde(rename = "tool.fill")]
ToolFill,
#[serde(rename = "tool.press")]
@@ -174,9 +168,6 @@ impl Method {
| Method::ToolNavigateForward
| Method::ToolReload
| Method::ToolClick
| Method::ToolScrollTo
| Method::ToolFocus
| Method::ToolBlur
| Method::ToolFill
| Method::ToolPress
| Method::ToolSelect
@@ -314,9 +305,6 @@ mod tests {
assert!(Method::ToolNavigateForward.is_mutating());
assert!(Method::ToolReload.is_mutating());
assert!(Method::ToolClick.is_mutating());
assert!(Method::ToolScrollTo.is_mutating());
assert!(Method::ToolFocus.is_mutating());
assert!(Method::ToolBlur.is_mutating());
assert!(Method::ToolFill.is_mutating());
assert!(Method::ToolPress.is_mutating());
assert!(Method::ToolSelect.is_mutating());
@@ -361,9 +349,6 @@ mod tests {
assert_eq!(Method::ToolHover.effect(), MethodEffect::TransientInput);
assert_eq!(Method::ToolObserve.effect(), MethodEffect::TransientInput);
assert_eq!(Method::ToolClick.effect(), MethodEffect::BrowserMutation);
assert_eq!(Method::ToolScrollTo.effect(), MethodEffect::BrowserMutation);
assert_eq!(Method::ToolFocus.effect(), MethodEffect::BrowserMutation);
assert_eq!(Method::ToolBlur.effect(), MethodEffect::BrowserMutation);
assert_eq!(Method::Cancel.effect(), MethodEffect::ControlPlane);
}
@@ -373,9 +358,6 @@ mod tests {
assert!(Method::ToolHover.requires_interrupt_gate());
assert!(Method::ToolObserve.requires_interrupt_gate());
assert!(Method::ToolClick.requires_interrupt_gate());
assert!(Method::ToolScrollTo.requires_interrupt_gate());
assert!(Method::ToolFocus.requires_interrupt_gate());
assert!(Method::ToolBlur.requires_interrupt_gate());
assert!(!Method::Cancel.requires_interrupt_gate());
}
}
+2 -109
View File
@@ -1,5 +1,5 @@
//! DOM interaction tools (`tool.click`, `tool.hover`, `tool.focus`,
//! `tool.blur`, `tool.fill`, `tool.press`, `tool.select`).
//! DOM interaction tools (`tool.click`, `tool.fill`, `tool.press`,
//! `tool.select`).
//!
//! Element-targeted tools accept either a snapshot `ref` (`@e<N>` form,
//! normalised against the session's RefStore) **or** a CSS selector
@@ -131,82 +131,6 @@ pub struct HoverResult {
pub dialogs: Vec<JavaScriptDialogInfo>,
}
// ---------------------------------------------------------------------------
// focus / blur
// ---------------------------------------------------------------------------
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct FocusParams {
pub session_id: String,
/// Optional `@e<N>` ref allocated by the last observation.
/// Mutually exclusive with `selector`.
#[serde(
rename = "ref",
alias = "ref_",
default,
skip_serializing_if = "Option::is_none"
)]
pub ref_: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub selector: Option<String>,
/// Target tab. Defaults to the Agent Window's active tab.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub tab_id: Option<i64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(range(min = 1))]
pub timeout_ms: Option<u32>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct FocusResult {
pub tab_id: i64,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub used_ref: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub used_selector: Option<String>,
/// Whether the target holds DOM focus in its document or shadow root after the call.
pub focused: bool,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub dialogs: Vec<JavaScriptDialogInfo>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct BlurParams {
pub session_id: String,
/// Optional `@e<N>` ref allocated by the last observation.
/// Mutually exclusive with `selector`.
#[serde(
rename = "ref",
alias = "ref_",
default,
skip_serializing_if = "Option::is_none"
)]
pub ref_: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub selector: Option<String>,
/// Target tab. Defaults to the Agent Window's active tab.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub tab_id: Option<i64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(range(min = 1))]
pub timeout_ms: Option<u32>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct BlurResult {
pub tab_id: i64,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub used_ref: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub used_selector: Option<String>,
/// Whether the target held focus before the call.
pub was_focused: bool,
/// Always false for a successful blur; included for machine verification.
pub focused: bool,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub dialogs: Vec<JavaScriptDialogInfo>,
}
// ---------------------------------------------------------------------------
// fill
// ---------------------------------------------------------------------------
@@ -401,37 +325,6 @@ mod tests {
assert_eq!(round, p);
}
#[test]
fn focus_params_serialise_ref_field_name() {
let p = FocusParams {
session_id: "abcd".into(),
ref_: Some("@e4".into()),
selector: None,
tab_id: Some(42),
timeout_ms: Some(5_000),
};
let v = serde_json::to_value(&p).unwrap();
assert_eq!(v.get("ref").and_then(|v| v.as_str()), Some("@e4"));
assert!(v.get("ref_").is_none());
let round: FocusParams = serde_json::from_value(v).unwrap();
assert_eq!(round, p);
}
#[test]
fn blur_result_round_trips_focus_state() {
let result = BlurResult {
tab_id: 42,
used_ref: Some("e4".into()),
used_selector: None,
was_focused: true,
focused: false,
dialogs: vec![],
};
let value = serde_json::to_value(&result).unwrap();
let round: BlurResult = serde_json::from_value(value).unwrap();
assert_eq!(round, result);
}
#[test]
fn modifiers_render_as_lowercase_strings() {
let v = serde_json::to_value(KeyModifier::Ctrl).unwrap();
-2
View File
@@ -14,7 +14,6 @@ mod record_common;
mod record_v2;
mod record_v3;
pub mod script;
pub mod scroll;
pub mod session;
pub mod tabs;
pub mod waits;
@@ -31,7 +30,6 @@ pub use network::*;
pub use observation::*;
pub use record::*;
pub use script::*;
pub use scroll::*;
pub use session::*;
pub use tabs::*;
pub use waits::*;
-67
View File
@@ -1,67 +0,0 @@
//! Element scrolling primitive (`tool.scroll_to`).
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
use super::JavaScriptDialogInfo;
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct ScrollToParams {
pub session_id: String,
/// Optional `@e<N>` ref allocated by the last observation.
/// Mutually exclusive with `selector`.
#[serde(
rename = "ref",
alias = "ref_",
default,
skip_serializing_if = "Option::is_none"
)]
pub ref_: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub selector: Option<String>,
/// Target tab. Defaults to the Agent Window's active tab.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub tab_id: Option<i64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(range(min = 1))]
pub timeout_ms: Option<u32>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct ScrollToResult {
pub tab_id: i64,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub used_ref: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub used_selector: Option<String>,
/// Bounding rectangle of the element's visible border-box portion, in
/// top-level viewport CSS pixels, after ancestor and viewport clipping.
/// Partial visibility is sufficient. This is not an occlusion or hit test.
pub x: f64,
pub y: f64,
pub width: f64,
pub height: f64,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub dialogs: Vec<JavaScriptDialogInfo>,
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn params_serialise_ref_field_name() {
let params = ScrollToParams {
session_id: "abcd".into(),
ref_: Some("@e3".into()),
selector: None,
tab_id: Some(42),
timeout_ms: Some(5_000),
};
let value = serde_json::to_value(&params).unwrap();
assert_eq!(value.get("ref").and_then(|v| v.as_str()), Some("@e3"));
assert!(value.get("ref_").is_none());
let round: ScrollToParams = serde_json::from_value(value).unwrap();
assert_eq!(round, params);
}
}
-5
View File
@@ -84,11 +84,6 @@ Shared Rust types + JSON Schema generation. TypeScript mirrors frame shapes in
4. Extension dispatcher validates sandbox rules, invokes CDP via `BrowserDriver`.
5. Response travels CLI ← daemon ← extension; CLI prints result and exits.
The [scroll-to primitive reference](scroll-to.md) documents `tool.scroll_to`,
including its CLI/plugin mappings, visible-bounds contract and cancellation
behavior. It follows the same routing path and is classified as a browser
mutation for session queueing and user-interruption gating.
## Session and sandbox model
- **Session** = opaque ID (4 lowercase letters in v0.1) + dedicated **Agent Window**
-131
View File
@@ -1,131 +0,0 @@
# Scroll-to primitive
`scroll-to` brings an existing element and its containing frames into view, then
returns the visible portion's bounds. Use it to reveal a target before inspecting
the page or taking a screenshot. It accepts an element target rather than a pixel
distance or scroll direction.
| Entry point | Name |
| --- | --- |
| CLI | `bsk scroll-to` |
| Wire protocol | `tool.scroll_to` |
| DeepSeek Harness | `browser_interact` with `action: "scroll-to"` |
## CLI
Use a session you started and a fresh ref from an observation. The ids below are
examples; substitute the session, tab and ref returned by your own calls.
```sh
bsk scroll-to @e3 --session abcd --tab-id 42 --timeout 5s --json
bsk scroll-to --selector '#details' --session abcd
```
Supply exactly one target: a positional ref or selector, `--ref`, or `--selector`.
Refs accept `@e3` and `e3`. CSS selectors search only the main document; use refs
for elements in iframes (including out-of-process iframes) or shadow roots.
Refs belong to one session and tab and may become stale after page changes.
`--session` is required. `--tab-id` defaults to the Agent Window's active tab.
`--timeout` defaults to `30s` and must be positive; durations such as `5000ms` and
`5s` are accepted. The command operates on Agent Window tabs, including user tabs
explicitly borrowed into that window.
Example JSON output:
```json
{
"tab_id": 42,
"used_ref": "e3",
"x": 10,
"y": 20,
"width": 100,
"height": 80
}
```
`used_ref` or `used_selector` identifies the resolved target. The optional
`dialogs` array reports JavaScript dialogs handled during the call.
## Result contract
`x` and `y` are the upper-left corner, and `width` and `height` are the size of
the visible border-box portion's bounding rectangle. All four values use
**top-level viewport CSS pixels**, after scrolling and clipping by ancestor
containers and frame viewports. They are not document or screenshot pixel
coordinates.
Partial visibility is sufficient: a 400px-high element clipped by an 80px-high
scroll container can succeed with `height: 80`. Hidden or fully clipped targets
fail. The rectangle is not an occlusion or hit test: another element can cover
the target, and the rectangle's center is not a guaranteed click point. Layout
can change after the result; use fresh refs for subsequent interactions.
## Protocol and plugin
The CLI sends a normal request through the daemon to the extension:
```json
{
"id": "scroll-1",
"method": "tool.scroll_to",
"params": {
"session_id": "abcd",
"ref": "@e3",
"tab_id": 42,
"timeout_ms": 5000
}
}
```
Protocol callers must supply exactly one nonempty `ref` or `selector`.
`tab_id` is optional; `timeout_ms` is a positive integer and defaults to 30000.
These target constraints are enforced by the handler. See the generated
[parameter schema](../crates/bsk-protocol/schema/tool_scroll_to_params.json) and
[result schema](../crates/bsk-protocol/schema/tool_scroll_to_result.json) for field
types. The wire response wraps the result above in `{ "id": "scroll-1", "result": ... }`.
The plugin exposes the same action with its usual camelCase parameters:
```json
{
"action": "scroll-to",
"session": "abcd",
"target": "@e3",
"tabId": 42,
"timeoutMs": 5000
}
```
Call this through `browser_interact` using a plugin-owned session. Its result
contains `session`, `tabId`, `x`, `y`, `width` and `height` with the same bounds
semantics. Omitted `session` uses the plugin's current session.
## Errors and interruption
The wire response uses the normal `error.code`, `error.message` and optional
`error.data.reason` fields. Common cases are:
| Code | `data.reason` | Meaning |
| --- | --- | --- |
| `invalid_params` | — | Missing/conflicting target or invalid timeout/tab id |
| `not_found` | `ref_not_found` | Ref is unknown, expired or belongs to another tab |
| `not_found` | `selector_not_found` | Main-document selector matched no element |
| `permission_denied` | `agent_window_scope` | Target tab has not been borrowed into the Agent Window |
| `permission_denied` | `element_not_visible` | No visible area remains after scrolling |
| `cancelled` | — | The call was cancelled |
| `timeout` | — | The action deadline expired |
| `cdp_failed` | varies | Browser command, frame geometry or visibility measurement failed |
The protocol classifies scroll-to as a browser mutation, so it uses the existing
per-session queue and user-interruption gate. The extension releases any retained
hover state before dispatch. Cancellation and deadline checks surround each CDP
step, including frame lookup, scrolling and measurement. Once detected, they
prevent further scrolling and a successful response; remote-object cleanup still
runs. Already performed scrolling is not rolled back, so inspect the current
page before retrying an interrupted call.
The implementation is scoped to scroll-to: it reuses frame scrolling and
coordinate projection, with a separate visibility measurement that accounts for
ancestor clipping and hidden frame owners. Shared click, hover and screenshot
geometry is unchanged.
+57 -34
View File
@@ -38,11 +38,17 @@ function Write-Die {
# ── Platform / architecture detection ─────────────────────────────────────────
function Get-PlatformTriple {
$arch = [System.Runtime.InteropServices.RuntimeInformation]::ProcessArchitecture
# RuntimeInformation.ProcessArchitecture can be unavailable in Windows PowerShell 5.1.
# WOW64 exposes the native architecture separately from the 32-bit process.
$arch = $env:PROCESSOR_ARCHITEW6432
if (-not $arch) { $arch = $env:PROCESSOR_ARCHITECTURE }
if (-not $arch) {
Write-Die "could not detect Windows architecture: PROCESSOR_ARCHITEW6432 and PROCESSOR_ARCHITECTURE are empty"
}
switch ($arch) {
"X64" { $archId = "x64" }
"Arm64" { $archId = "arm64" }
"AMD64" { $archId = "x64" }
"ARM64" { $archId = "arm64" }
default { Write-Die "unsupported architecture: $arch (x64 and ARM64 only)" }
}
@@ -67,25 +73,21 @@ function Add-ToUserPath {
$currentUserPath = @([Environment]::GetEnvironmentVariable("PATH", "User") -split ";" | Where-Object { $_ })
if ($currentUserPath -contains $Dir) {
Write-Log "$Dir is already in your user PATH"
$newUserPath = (@($Dir) + @($currentUserPath | Where-Object { $_ -ine $Dir })) -join ";"
if (($currentUserPath -join ";") -ceq $newUserPath) {
Write-Log "$Dir is already first in your user PATH"
return
}
$newUserPath = ($currentUserPath + $Dir) -join ";"
[Environment]::SetEnvironmentVariable("PATH", $newUserPath, "User")
Write-Log "added ${Dir} to user PATH"
Write-Log "placed ${Dir} first in user PATH"
}
function Add-ToSessionPath {
param([string]$Dir)
$pathEntries = $env:PATH -split ";" | Where-Object { $_ }
if ($pathEntries -contains $Dir) {
return
}
$env:PATH = "$Dir;$env:PATH"
$pathEntries = @($env:PATH -split ";" | Where-Object { $_ -and $_ -ine $Dir })
$env:PATH = (@($Dir) + $pathEntries) -join ";"
}
# ── Git Bash (bash environment) PATH helper ──────────────────────────────────
@@ -126,14 +128,19 @@ function Install-Binary {
$staged = "$Target.install-$([Guid]::NewGuid().ToString('N'))"
try {
[System.IO.File]::Copy($Source, $staged)
# The running daemon may come from another installation directory.
# Use the downloaded CLI: older versions have broken Windows liveness checks.
# Stop verifies daemon identity and uses the current BSK_HOME.
# Fail closed for both new installs and replacements; never discard daemon metadata here.
& $Source daemon stop
if ($LASTEXITCODE -ne 0) {
$daemonHome = if ($env:BSK_HOME) { $env:BSK_HOME } else { Join-Path $HOME ".bsk" }
$daemonInfoPath = $ExecutionContext.SessionState.Path.GetUnresolvedProviderPathFromPSPath((Join-Path $daemonHome "daemon.json"))
throw "could not stop bsk daemon; installation was not changed. Stop any running bsk daemon, remove '$daemonInfoPath', then retry the installer."
}
if ([System.IO.File]::Exists($Target)) {
# Use the downloaded CLI: older versions have broken Windows
# liveness checks. Stop verifies daemon identity before terminating it.
& $Source daemon stop
if ($LASTEXITCODE -ne 0) { throw "could not stop bsk daemon; existing installation was not replaced" }
# PowerShell 5.1 converts $null to an empty path for string parameters.
[System.IO.File]::Replace($staged, $Target, [NullString]::Value)
Write-Log "daemon will restart automatically on the next browser command"
}
else {
[System.IO.File]::Move($staged, $Target)
@@ -166,14 +173,20 @@ function Main {
Write-Log "latest version is ${version}"
}
$platformKey = $platform.PlatformKey
$asset = $null
if ($manifest -and $manifest.assets) {
$asset = $manifest.assets.$platformKey
}
# ARM64 is not in the current release matrix. Require a published entry
# before attempting it, while preserving legacy x64 installs without a manifest.
if ($platform.ArchId -eq "arm64" -and -not $asset) {
Write-Die "version.json does not list a Windows ARM64 package for bsk $version"
}
$archiveName = "bsk-v${version}-$($platform.TargetTriple).zip"
$downloadUrl = "${GitHub}/releases/download/${tag}/${archiveName}"
$expectedSha = $null
$platformKey = $platform.PlatformKey
if ($manifest -and $manifest.assets) {
$expectedSha = $manifest.assets.$platformKey.sha256
}
$expectedSha = if ($asset) { $asset.sha256 } else { $null }
if (-not $expectedSha) {
if (-not $manifest) {
Write-Log "warning: could not fetch version.json; skipping checksum verification"
@@ -184,17 +197,21 @@ function Main {
}
$tempDir = Join-Path ([System.IO.Path]::GetTempPath()) ([System.IO.Path]::GetRandomFileName())
New-Item -ItemType Directory -Path $tempDir -Force | Out-Null
[System.IO.Directory]::CreateDirectory($tempDir) | Out-Null
try {
$archivePath = Join-Path $tempDir $archiveName
Write-Log "downloading ${downloadUrl}"
Invoke-WebRequest -Uri $downloadUrl -OutFile $archivePath -UseBasicParsing -ErrorAction Stop
# PowerShell 5.1's -OutFile treats brackets as wildcards. Write raw HTTP bytes literally.
$response = Invoke-WebRequest -Uri $downloadUrl -UseBasicParsing -ErrorAction Stop
try {
[System.IO.File]::WriteAllBytes($archivePath, $response.RawContentStream.ToArray())
} finally { $response.RawContentStream.Dispose() }
if ($expectedSha) {
Write-Log "verifying checksum"
$actualSha = (Get-FileHash -Algorithm SHA256 -Path $archivePath).Hash
$actualSha = (Get-FileHash -Algorithm SHA256 -LiteralPath $archivePath).Hash
if ($actualSha -ieq $expectedSha) {
Write-Log "checksum OK"
}
@@ -204,15 +221,15 @@ function Main {
}
Write-Log "extracting ${archiveName}"
Expand-Archive -Path $archivePath -DestinationPath $tempDir -Force
# PowerShell 5.1's Expand-Archive treats the destination as a wildcard path.
Add-Type -AssemblyName System.IO.Compression.FileSystem
[System.IO.Compression.ZipFile]::ExtractToDirectory($archivePath, $tempDir)
if (-not (Test-Path (Join-Path $tempDir "bsk.exe"))) {
if (-not (Test-Path -LiteralPath (Join-Path $tempDir "bsk.exe"))) {
Write-Die "bsk.exe not found in archive"
}
if (-not (Test-Path $InstallDir)) {
New-Item -ItemType Directory -Path $InstallDir -Force | Out-Null
}
[System.IO.Directory]::CreateDirectory($InstallDir) | Out-Null
Install-Binary -Source (Join-Path $tempDir "bsk.exe") -Target (Join-Path $InstallDir "bsk.exe")
@@ -232,12 +249,18 @@ function Main {
& $bskPath --version
if ($LASTEXITCODE -ne 0) { throw "installed bsk failed verification" }
$command = Get-Command bsk -ErrorAction SilentlyContinue
if (-not $command -or $command.CommandType -ne "Application" -or $command.Source -ine $bskPath) {
Write-Log "warning: 'bsk' does not resolve to $bskPath in this session; check Get-Command bsk -All for a conflicting command"
}
Write-Log "done"
Write-Host ""
Write-Host "Open a new terminal (PowerShell / Git Bash) for PATH changes to take full effect."
Write-Host "This PATH check covers the current session only; a new terminal may prefer a Machine PATH entry or alias."
Write-Host "In a new PowerShell terminal, run Get-Command bsk -All and confirm the first result is $bskPath."
}
finally {
Remove-Item -Recurse -Force $tempDir -ErrorAction SilentlyContinue
Remove-Item -LiteralPath $tempDir -Recurse -Force -ErrorAction SilentlyContinue
}
}
+1 -4
View File
@@ -54,13 +54,10 @@ the `bsk` CLI and browser extension separately when a release requires it.
| `browser_session` | `start`, `stop`, `list` | Manage plugin-owned Agent Window sessions. |
| `browser_page` | `navigate`, `back`, `forward`, `reload`, `wait` | Navigate the active tab and wait for page lifecycle events. |
| `browser_inspect` | `observe`, `snapshot`, `html`, `screenshot`, `console`, `network` | Read semantic or diagnostic page state and capture screenshots. |
| `browser_interact` | `click`, `hover`, `scroll-to`, `focus`, `blur`, `fill`, `select`, `press` | Interact with controls using fresh refs or selectors. |
| `browser_interact` | `click`, `hover`, `fill`, `select`, `press` | Interact with controls using fresh refs or selectors. |
| `browser_tabs` | `list`, `create`, `select`, `close`, `borrow`, `return` | Manage Agent Window tabs and temporarily borrow user tabs. |
| `browser_assist` | `resize`, `emulate`, `request-help` | Resize or emulate the browser and pause for human-only steps. |
For `browser_interact` with `action: "scroll-to"`, see the
[scroll-to reference](../../docs/scroll-to.md) for parameters, visible bounds and errors.
Arbitrary page-script evaluation and interaction recording are not supported.
## Multi-session model
@@ -51,13 +51,8 @@ Use `browser_inspect` action `observe` as the primary semantic page view. It ret
text, and `@eN` refs. Prefer fresh refs over raw selectors. Refs invalidate after navigation and may
also become stale after large DOM changes, so observe again before the next interaction.
Use `browser_interact` with `action=scroll-to` to bring an element into view. Its bounds are the
visible border-box portion in top-level viewport CSS pixels after ancestor clipping; partial
visibility is enough. Hidden or fully clipped targets fail. This does not test occlusion by other
elements. Use fresh element refs for iframe/shadow-root targets; selectors search the main document.
Use `browser_interact` for click, hover, scroll-to, focus, blur, fill, select, and key actions. Focus and blur
explicitly enter or leave UI states triggered by focus changes. An observation marks a hover-only surface as `@e1 button "Products" [hover first: Shoes | Bags]`. The listed items are
Use `browser_interact` for click, hover, fill, select, and key actions. An observation marks a
hover-only surface as `@e1 button "Products" [hover first: Shoes | Bags]`. The listed items are
labels, not usable refs: hover the trigger, observe again, then act on the revealed item's own ref.
Do not click the trigger itself unless the user wants the trigger's action.
@@ -167,15 +167,12 @@ const BROWSER_TOOL_SPECS: BrowserToolSpec[] = [
{
name: "browser_interact",
description:
"Interact with an element in the active Agent Window tab. Actions: click, hover, scroll-to, focus, blur, fill, select, " +
"press. click/hover/scroll-to/focus/blur/fill/select require target; fill also requires value; select requires " +
"Interact with an element in the active Agent Window tab. Actions: click, hover, fill, select, " +
"press. click/hover/fill/select require target; fill also requires value; select requires " +
"values; press requires key and may optionally focus target first.",
actions: {
click: "interact.click",
hover: "interact.hover",
"scroll-to": "interact.scroll-to",
focus: "interact.focus",
blur: "interact.blur",
fill: "interact.fill",
select: "interact.select",
press: "interact.press",
@@ -562,10 +562,6 @@ export function actionForLabel(label: string): string {
return "clicking";
case "hover":
return "hovering";
case "focus":
return "focusing";
case "blur":
return "blurring";
case "fill":
return "filling";
case "select":
@@ -13,7 +13,7 @@ import type { ToolDeps } from "./tools";
const MODIFIERS = ["alt", "ctrl", "meta", "shift"] as const;
/** Add interaction primitives without bypassing session ownership or observation. */
/** Add hover and select without bypassing session ownership or observation. */
export function registerPhaseOneInteractionTools(
deps: ToolDeps,
register: ToolRegistrar,
@@ -100,171 +100,6 @@ export function registerPhaseOneInteractionTools(
}),
);
register(
defineTool({
name: "interact.scroll-to",
description:
"Scroll an element and its frame owners into view. Returns the visible portion's " +
"bounds in top-level viewport CSS pixels; fails if no visible area remains. " +
"Use a fresh ref for iframe targets; selectors search the main document.",
parameters: {
target: {
type: "string",
required: true,
description: "Element ref (@e3 / e3) or main-document CSS selector to scroll into view.",
},
session: SESSION_PARAM,
tabId: TAB_ID_PARAM,
timeoutMs: TIMEOUT_MS_PARAM,
},
output: {
schema: {
type: "object",
additionalProperties: false,
properties: {
session: { type: "string", required: true },
tabId: { type: "integer", required: true },
x: { type: "number", required: true },
y: { type: "number", required: true },
width: { type: "number", required: true },
height: { type: "number", required: true },
},
},
render: (_args, value) => [
{
type: "text",
text:
`[session ${value.session}] scrolled into view on tab ${value.tabId}: ` +
`(${value.x}, ${value.y}, ${value.width}, ${value.height})`,
},
],
},
async execute(args, exec) {
requireNonEmpty(args.target, "target");
requirePositive(args.timeoutMs, "timeoutMs");
const sessionId = registry.resolve(args.session, "browser_interact(action=scroll-to)");
const cmdArgs = ["scroll-to", "--session", sessionId];
appendTabId(cmdArgs, args.tabId);
if (args.timeoutMs !== undefined) cmdArgs.push("--timeout", `${args.timeoutMs}ms`);
appendTarget(cmdArgs, args.target);
const reply = (await runtime.run(
exec,
cmdArgs,
"scroll-to",
sessionId,
runnerTimeout(deps, args.timeoutMs),
)) as {
tab_id: number;
x: number;
y: number;
width: number;
height: number;
};
return {
session: sessionId,
tabId: reply.tab_id,
x: reply.x,
y: reply.y,
width: reply.width,
height: reply.height,
};
},
presentCall: (args) => ({
card: "terminal",
title: runtime.commandLine([
"scroll-to",
args.target,
"--session",
args.session ?? "(current)",
]),
description: "Scroll an element into view",
}),
presentResult: runtime.presentTerminalResult,
}),
);
for (const action of ["focus", "blur"] as const) {
register(
defineTool({
name: `interact.${action}`,
description:
action === "focus"
? "Focus an element and verify its DOM focus state."
: "Remove focus from an element and report whether it was focused.",
parameters: {
target: {
type: "string",
required: true,
description: "Snapshot ref (@e3 / e3) or CSS selector of the element.",
},
session: SESSION_PARAM,
tabId: TAB_ID_PARAM,
timeoutMs: TIMEOUT_MS_PARAM,
},
output: {
schema: {
type: "object",
additionalProperties: false,
properties: {
session: { type: "string", required: true },
tabId: { type: "integer", required: true },
focused: { type: "boolean", required: true },
wasFocused: {
type: "boolean",
description: "Whether the target was focused before blur.",
},
},
},
render: (_args, value) => [
{
type: "text",
text:
`[session ${value.session}] ${action} on tab ${value.tabId}: focused=${value.focused}` +
(value.wasFocused === undefined ? "" : `, wasFocused=${value.wasFocused}`),
},
],
},
async execute(args, exec) {
requireNonEmpty(args.target, "target");
requirePositive(args.timeoutMs, "timeoutMs");
const sessionId = registry.resolve(args.session, `browser_interact(action=${action})`);
const cmdArgs = [action, "--session", sessionId];
appendTabId(cmdArgs, args.tabId);
if (args.timeoutMs !== undefined) cmdArgs.push("--timeout", `${args.timeoutMs}ms`);
appendTarget(cmdArgs, args.target);
const reply = (await runtime.run(
exec,
cmdArgs,
action,
sessionId,
runnerTimeout(deps, args.timeoutMs),
)) as {
tab_id: number;
focused: boolean;
was_focused?: boolean;
};
return {
session: sessionId,
tabId: reply.tab_id,
focused: reply.focused,
...(action === "blur" ? { wasFocused: reply.was_focused } : {}),
};
},
presentCall: (args) => ({
card: "terminal",
title: runtime.commandLine([
action,
args.target,
"--session",
args.session ?? "(current)",
]),
description: action === "focus" ? "Focus an element" : "Remove focus from an element",
}),
presentResult: runtime.presentTerminalResult,
}),
);
}
register(
defineTool({
name: "interact.select",
@@ -102,9 +102,6 @@ const ACTION_ROUTES: Record<string, readonly [string, string]> = {
"inspect.network": ["browser_inspect", "network"],
"interact.click": ["browser_interact", "click"],
"interact.hover": ["browser_interact", "hover"],
"interact.scroll-to": ["browser_interact", "scroll-to"],
"interact.focus": ["browser_interact", "focus"],
"interact.blur": ["browser_interact", "blur"],
"interact.fill": ["browser_interact", "fill"],
"interact.select": ["browser_interact", "select"],
"interact.press": ["browser_interact", "press"],
@@ -203,7 +200,7 @@ const EXPECTED_ACTIONS = {
browser_session: ["start", "stop", "list"],
browser_page: ["navigate", "back", "forward", "reload", "wait"],
browser_inspect: ["observe", "snapshot", "html", "screenshot", "console", "network"],
browser_interact: ["click", "hover", "scroll-to", "focus", "blur", "fill", "select", "press"],
browser_interact: ["click", "hover", "fill", "select", "press"],
browser_tabs: ["list", "create", "select", "close", "borrow", "return"],
browser_assist: ["resize", "emulate", "request-help"],
} as const;
@@ -502,85 +499,6 @@ describe("interaction tools", () => {
]);
});
it.each([
"focus",
"blur",
] as const)("interact.%s maps focus state, target, tab and timeout", async (action) => {
const { tools, calls } = setup({
"session start": START_REPLY("s1"),
[action]: {
tab_id: 7,
focused: action === "focus",
...(action === "blur" ? { was_focused: true } : {}),
},
});
await startSession(tools);
const target = action === "focus" ? "@e3" : "#field";
const value = await tools
.get(`interact.${action}`)
?.execute({ target, tabId: 7, timeoutMs: 150_000 }, makeExec());
expect(value).toEqual({
session: "s1",
tabId: 7,
focused: action === "focus",
...(action === "blur" ? { wasFocused: true } : {}),
});
expect(calls[1].args).toEqual([
action,
"--session",
"s1",
"--tab-id",
"7",
"--timeout",
"150000ms",
target,
]);
expect(calls[1].options.timeoutMs).toBe(165_000);
});
it.each([
"focus",
"blur",
"scroll-to",
] as const)("interact.%s validates arguments and session ownership before running", async (action) => {
const { tools, calls } = setup({ "session start": START_REPLY("s1") });
await startSession(tools);
const tool = tools.get(`interact.${action}`)!;
await expect(tool.execute({}, makeExec())).rejects.toThrow(/invalid arguments/i);
await expect(tool.execute({ target: " " }, makeExec())).rejects.toThrow(/non-empty/i);
await expect(tool.execute({ target: "e1", timeoutMs: 0 }, makeExec())).rejects.toThrow(
/greater than zero/i,
);
await expect(tool.execute({ target: "e1", session: "foreign" }, makeExec())).rejects.toThrow(
/session/i,
);
expect(calls).toHaveLength(1);
});
it.each(["@e3", "#target"])("interact.scroll-to maps %s, bounds and timeout", async (target) => {
const { tools, calls } = setup({
"session start": START_REPLY("s1"),
"scroll-to": { tab_id: 7, x: 10, y: 20, width: 100, height: 80 },
});
await startSession(tools);
expect(
await tools
.get("interact.scroll-to")
?.execute({ target, tabId: 7, timeoutMs: 150_000 }, makeExec()),
).toEqual({ session: "s1", tabId: 7, x: 10, y: 20, width: 100, height: 80 });
expect(calls[1].args).toEqual([
"scroll-to",
"--session",
"s1",
"--tab-id",
"7",
"--timeout",
"150000ms",
target,
]);
expect(calls[1].options.timeoutMs).toBe(165_000);
});
it("interact.fill passes value and --no-clear", async () => {
const { tools, calls } = setup(responses);
await startSession(tools);
+234 -14
View File
@@ -1,8 +1,11 @@
#Requires -Version 5.1
param([string]$BskPath = (Join-Path $PSScriptRoot "../target/debug/bsk.exe"))
param(
[string]$BskPath = (Join-Path $PSScriptRoot "../target/debug/bsk.exe"),
[switch]$HelpersOnly
)
$ErrorActionPreference = "Stop"
# Load definitions, never Main: no downloads or changes to the real user PATH.
# Load definitions without running Main. Registry PATH writes are isolated below.
$installer = Join-Path $PSScriptRoot "../install.ps1"
$tokens = $null
$errors = $null
@@ -21,27 +24,88 @@ foreach ($statement in $ast.EndBlock.Statements) {
function Assert-Equal($Actual, $Expected) {
if ($Actual -cne $Expected) { throw "expected [$Expected], got [$Actual]" }
}
function Assert-Fails([scriptblock]$Action) {
function Assert-Fails([scriptblock]$Action, [string]$MessageContains) {
$failed = $false
try { & $Action } catch { $failed = $true }
try { & $Action } catch {
$failed = $true
if ($MessageContains -and -not $_.Exception.Message.Contains($MessageContains)) { throw }
}
if (-not $failed) { throw "expected failure" }
}
# Isolate the fatal-error stub and restore the real process environment.
& {
function Write-Die([string]$Message) { throw $Message }
$oldProcessArch = $env:PROCESSOR_ARCHITECTURE
$oldNativeArch = $env:PROCESSOR_ARCHITEW6432
try {
$cases = @(
@{ Process = 'AMD64'; Native = $null; Arch = 'x64'; Triple = 'x86_64-pc-windows-msvc' }
@{ Process = 'ARM64'; Native = $null; Arch = 'arm64'; Triple = 'aarch64-pc-windows-msvc' }
@{ Process = 'amd64'; Native = $null; Arch = 'x64'; Triple = 'x86_64-pc-windows-msvc' }
@{ Process = 'x86'; Native = 'AMD64'; Arch = 'x64'; Triple = 'x86_64-pc-windows-msvc' }
@{ Process = 'x86'; Native = 'ARM64'; Arch = 'arm64'; Triple = 'aarch64-pc-windows-msvc' }
@{ Process = $null; Native = 'AMD64'; Arch = 'x64'; Triple = 'x86_64-pc-windows-msvc' }
@{ Process = 'x86'; Native = $null; Error = 'unsupported architecture: x86 (x64 and ARM64 only)' }
@{ Process = 'AMD64'; Native = 'IA64'; Error = 'unsupported architecture: IA64 (x64 and ARM64 only)' }
@{ Process = $null; Native = $null; Error = 'could not detect Windows architecture: PROCESSOR_ARCHITEW6432 and PROCESSOR_ARCHITECTURE are empty' }
)
foreach ($case in $cases) {
$env:PROCESSOR_ARCHITECTURE = $case.Process
$env:PROCESSOR_ARCHITEW6432 = $case.Native
if ($case.Error) {
$message = $null
try { Get-PlatformTriple | Out-Null } catch { $message = $_.Exception.Message }
Assert-Equal $message $case.Error
}
else {
$platform = Get-PlatformTriple
Assert-Equal $platform.ArchId $case.Arch
Assert-Equal $platform.TargetTriple $case.Triple
Assert-Equal $platform.PlatformKey "windows-$($case.Arch)"
}
}
Write-Host "Windows installer architecture regressions passed ($($PSVersionTable.PSVersion))"
}
finally {
$env:PROCESSOR_ARCHITECTURE = $oldProcessArch
$env:PROCESSOR_ARCHITEW6432 = $oldNativeArch
}
}
$dir = 'C:\Users\Alice\.local\bin'
foreach ($existing in @('', 'C:\WindowsApps', 'C:\A;C:\B')) {
$script:UserPath = $existing
Add-ToUserPath $dir
$expected = if ($existing) { "$existing;$dir" } else { $dir }
Assert-Equal $script:UserPath $expected
Add-ToUserPath $dir
Assert-Equal $script:UserPath $expected
$pathCases = @(
@{ Existing = ''; Expected = $dir }
@{ Existing = 'C:\WindowsApps'; Expected = "$dir;C:\WindowsApps" }
@{ Existing = 'C:\A;C:\B'; Expected = "$dir;C:\A;C:\B" }
@{ Existing = "C:\old-bsk;$dir;C:\B;$($dir.ToUpperInvariant())"; Expected = "$dir;C:\old-bsk;C:\B" }
)
$oldPath = $env:PATH
try {
foreach ($case in $pathCases) {
$script:UserPath = $case.Existing
$env:PATH = $case.Existing
foreach ($attempt in 1..2) {
Add-ToUserPath $dir
Add-ToSessionPath $dir
Assert-Equal $script:UserPath $case.Expected
Assert-Equal $env:PATH $case.Expected
}
}
} finally { $env:PATH = $oldPath }
if ($HelpersOnly) {
Write-Host "Windows installer helper regressions passed ($($PSVersionTable.PSVersion))"
return
}
$root = Join-Path ([IO.Path]::GetTempPath()) ("bsk-install-test-" + [Guid]::NewGuid().ToString('N'))
[IO.Directory]::CreateDirectory($root) | Out-Null
$oldBskHome = $env:BSK_HOME
$oldAutoUpdate = $env:BSK_AUTO_UPDATE
$oldVersion = $env:BSK_VERSION
$oldProcessArch = $env:PROCESSOR_ARCHITECTURE
$oldNativeArch = $env:PROCESSOR_ARCHITEW6432
$daemon = $null
$script:DownloadServer = $null
try {
$utf8 = New-Object System.Text.UTF8Encoding($false)
$bashRc = Join-Path $root ".bashrc"
@@ -66,7 +130,7 @@ try {
if ($LASTEXITCODE -ne 0) { throw "Git Bash failed" }
Assert-Equal $actual ('/c' + $special.Substring(2).Replace('\', '/'))
$env:BSK_HOME = Join-Path $root "home"
$env:BSK_HOME = Join-Path $root "home [state]"
$env:BSK_AUTO_UPDATE = "off"
[IO.Directory]::CreateDirectory($env:BSK_HOME) | Out-Null
$sourceDir = Join-Path $root "download"
@@ -76,7 +140,8 @@ try {
$targetDir = Join-Path $root "中文 space [x] & install"
[IO.Directory]::CreateDirectory($targetDir) | Out-Null
$target = Join-Path $targetDir "bsk.exe"
Install-Binary $source $target
$firstInstallOutput = Install-Binary $source $target 6>&1 | Out-String
if ($firstInstallOutput -match 'daemon will restart') { throw 'fresh install reported a daemon restart' }
Assert-Equal (Get-FileHash -LiteralPath $target).Hash (Get-FileHash -LiteralPath $source).Hash
$daemon = Start-Process -FilePath $target -ArgumentList @('daemon', 'start', '--foreground', '--port', '0') -WindowStyle Hidden -PassThru
@@ -99,18 +164,165 @@ try {
& $target --version
if ($LASTEXITCODE -ne 0) { throw "replacement is not executable" }
# Installing into a new directory must also stop a daemon from the old one.
$newTargetDir = Join-Path $root "new install"
[IO.Directory]::CreateDirectory($newTargetDir) | Out-Null
$newTarget = Join-Path $newTargetDir "bsk.exe"
$before = (Get-FileHash -LiteralPath $target).Hash
$daemon = Start-Process -FilePath $target -ArgumentList @('daemon', 'start', '--foreground', '--port', '0') -WindowStyle Hidden -PassThru
$deadline = [DateTime]::UtcNow.AddSeconds(15)
while (-not [IO.File]::Exists($infoPath)) {
if ($daemon.HasExited -or [DateTime]::UtcNow -gt $deadline) { throw "old-directory daemon failed to start" }
Start-Sleep -Milliseconds 50
}
Install-Binary $source $newTarget
if (-not $daemon.WaitForExit(5000)) { throw "new-directory install left the old daemon running" }
Assert-Equal (Get-FileHash -LiteralPath $target).Hash $before
Assert-Equal (Get-FileHash -LiteralPath $newTarget).Hash (Get-FileHash -LiteralPath $source).Hash
& $newTarget --version
if ($LASTEXITCODE -ne 0) { throw "new-directory installation is not executable" }
if (@(Get-ChildItem -LiteralPath $newTargetDir -Filter "*.install-*").Count) { throw "new-directory staging files leaked" }
# Refuse replacement when daemon identity cannot be verified.
[IO.File]::WriteAllText($infoPath, 'invalid daemon metadata')
$before = (Get-FileHash -LiteralPath $target).Hash
Assert-Fails { Install-Binary $source $target }
Assert-Fails { Install-Binary $source $target } -MessageContains $infoPath
Assert-Equal (Get-FileHash -LiteralPath $target).Hash $before
# A failed stop must also prevent installation to a previously empty target.
$blockedTarget = Join-Path $newTargetDir "blocked.exe"
Assert-Fails { Install-Binary $source $blockedTarget } -MessageContains $infoPath
if (Test-Path -LiteralPath $blockedTarget) { throw "failed stop created an installation" }
if (@(Get-ChildItem -LiteralPath $newTargetDir -Filter "*.install-*").Count) { throw "failed stop leaked staging files" }
Assert-Equal ([IO.File]::ReadAllText($infoPath)) 'invalid daemon metadata'
# Following the reported recovery path must allow the same installation to succeed.
[IO.File]::Delete($infoPath)
Install-Binary $source $blockedTarget
Assert-Equal (Get-FileHash -LiteralPath $blockedTarget).Hash (Get-FileHash -LiteralPath $source).Hash
# A remaining lock must fail without truncation or staging debris.
$lock = [IO.File]::Open($target, [IO.FileMode]::Open, [IO.FileAccess]::Read, [IO.FileShare]::None)
try { Assert-Fails { Install-Binary $source $target } } finally { $lock.Dispose() }
Assert-Equal (Get-FileHash -LiteralPath $target).Hash $before
if (@(Get-ChildItem -LiteralPath $targetDir -Filter "*.install-*").Count) { throw "staging files leaked" }
# Exercise Main with real ZIPs/executables, isolating network and user state.
& {
$fixtureTemp = Join-Path $root 'temp [literal]'
[IO.Directory]::CreateDirectory($fixtureTemp) | Out-Null
$mainDefinition = ($ast.EndBlock.Statements | Where-Object {
$_ -is [System.Management.Automation.Language.FunctionDefinitionAst] -and $_.Name -eq 'Main'
}).Extent.Text.Replace('[System.IO.Path]::GetTempPath()', '$fixtureTemp')
Invoke-Expression $mainDefinition
function Write-Die([string]$Message) { throw $Message }
$bashProfile = (Get-Command Add-ToBashProfile).ScriptBlock
function Add-ToBashProfile([string]$Dir) {
& $bashProfile $Dir (Join-Path $root 'flow.bashrc')
}
$fixtureArchive = Join-Path $root 'release.zip'
Compress-Archive -LiteralPath $source -DestinationPath $fixtureArchive -CompressionLevel Fastest
# Serve the ZIP over loopback HTTP. Only the URL is redirected below;
# Invoke-WebRequest and the installer's disk writes remain real.
$script:DownloadServer = Start-Job -ArgumentList $fixtureArchive -ScriptBlock {
param($Archive)
$ErrorActionPreference = 'Stop'
$body = [IO.File]::ReadAllBytes($Archive)
$listener = [Net.Sockets.TcpListener]::new([Net.IPAddress]::Loopback, 0)
try {
$listener.Start()
Write-Output $listener.LocalEndpoint.Port
while ($true) {
if (-not $listener.Pending()) { Start-Sleep -Milliseconds 50; continue }
$client = $listener.AcceptTcpClient()
try {
$stream = $client.GetStream()
$stream.ReadTimeout = 5000
$stream.WriteTimeout = 5000
$reader = [IO.StreamReader]::new($stream)
while (($line = $reader.ReadLine()) -and $line.Length) { }
$header = [Text.Encoding]::ASCII.GetBytes("HTTP/1.1 200 OK`r`nContent-Type: application/octet-stream`r`nContent-Length: $($body.Length)`r`nConnection: close`r`n`r`n")
$stream.Write($header, 0, $header.Length)
$stream.Write($body, 0, $body.Length)
} finally { $client.Dispose() }
}
} finally { $listener.Stop() }
}
$serverPort = $null
$deadline = [DateTime]::UtcNow.AddSeconds(15)
while (-not $serverPort) {
$serverPort = Receive-Job $script:DownloadServer
if ($script:DownloadServer.State -ne 'Running' -or [DateTime]::UtcNow -gt $deadline) {
throw 'download fixture failed to start'
}
if (-not $serverPort) { Start-Sleep -Milliseconds 50 }
}
$asset = @{ sha256 = (Get-FileHash -LiteralPath $fixtureArchive).Hash }
$fixtureManifest = @{ version = '9.8.7'; assets = @{ 'windows-x64' = $asset } }
$downloads = New-Object 'System.Collections.Generic.List[string]'
function Invoke-RestMethod([string]$Uri) {
if (-not $fixtureManifest) { throw 'fixture manifest unavailable' }
return $fixtureManifest
}
function Invoke-WebRequest {
[CmdletBinding()]
param([string]$Uri, [string]$OutFile, [switch]$UseBasicParsing)
$downloads.Add($Uri)
$PSBoundParameters['Uri'] = "http://127.0.0.1:$serverPort/release.zip"
Microsoft.PowerShell.Utility\Invoke-WebRequest @PSBoundParameters -TimeoutSec 15
}
$GitHub = 'https://example.invalid/fixture'
$InstallDir = Join-Path $root 'installed [literal]'
$installed = Join-Path $InstallDir 'bsk.exe'
$env:BSK_VERSION = $null
$env:PROCESSOR_ARCHITECTURE = 'AMD64'
$env:PROCESSOR_ARCHITEW6432 = $null
$script:UserPath = "$targetDir;$InstallDir;$($InstallDir.ToUpperInvariant())"
$env:PATH = "$targetDir;$InstallDir;$oldPath"
Assert-Equal (Get-Command bsk).Source $target
$installOutput = Main 6>&1 | Out-String
if ($installOutput -notmatch 'current session only' -or
$installOutput -notmatch 'Machine PATH' -or $installOutput -notmatch 'Get-Command bsk -All') {
throw 'install did not explain PATH verification in a new terminal'
}
Assert-Equal $downloads.Count 1
Assert-Equal (Get-Command bsk).Source $installed
Assert-Equal $script:UserPath "$InstallDir;$targetDir"
Assert-Equal (Get-FileHash -LiteralPath $installed).Hash (Get-FileHash -LiteralPath $source).Hash
Assert-Equal @(Get-ChildItem -LiteralPath $fixtureTemp -Force).Count 0
# ARM64 must be listed in the manifest before any archive is requested.
$env:PROCESSOR_ARCHITECTURE = 'ARM64'
$downloads.Clear()
$message = $null
try { Main } catch { $message = $_.Exception.Message }
Assert-Equal $message 'version.json does not list a Windows ARM64 package for bsk 9.8.7'
Assert-Equal $downloads.Count 0
$env:BSK_VERSION = '9.8.7'
$fixtureManifest = $null
$message = $null
try { Main } catch { $message = $_.Exception.Message }
Assert-Equal $message 'version.json does not list a Windows ARM64 package for bsk 9.8.7'
Assert-Equal $downloads.Count 0
# Keep pinned x64 installs compatible with older releases without manifests.
$env:PROCESSOR_ARCHITECTURE = 'AMD64'
Main
Assert-Equal $downloads.Count 1
Assert-Equal @(Get-ChildItem -LiteralPath $fixtureTemp -Force).Count 0
# A listed ARM64 archive remains selectable (the fixture uses the host EXE).
$fixtureManifest = @{ version = '9.8.7'; assets = @{ 'windows-arm64' = $asset } }
$env:PROCESSOR_ARCHITECTURE = 'ARM64'
Main
Assert-Equal $downloads[$downloads.Count - 1] "$GitHub/releases/download/cli-v9.8.7/bsk-v9.8.7-aarch64-pc-windows-msvc.zip"
# A checksum failure must clean the bracketed temp path and preserve the install.
$before = (Get-FileHash -LiteralPath $installed).Hash
$fixtureManifest.assets['windows-arm64'] = @{ sha256 = '0' * 64 }
Assert-Fails { Main }
Assert-Equal (Get-FileHash -LiteralPath $installed).Hash $before
Assert-Equal @(Get-ChildItem -LiteralPath $fixtureTemp -Force).Count 0
}
Write-Host "Windows installer regressions passed ($($PSVersionTable.PSVersion))"
}
catch {
@@ -119,12 +331,20 @@ catch {
throw
}
finally {
if ($script:DownloadServer) {
Stop-Job $script:DownloadServer
Remove-Job $script:DownloadServer
}
if ($daemon -and -not $daemon.HasExited) {
Stop-Process -Id $daemon.Id -Force
$daemon.WaitForExit(5000) | Out-Null
}
$env:BSK_HOME = $oldBskHome
$env:BSK_AUTO_UPDATE = $oldAutoUpdate
$env:BSK_VERSION = $oldVersion
$env:PROCESSOR_ARCHITECTURE = $oldProcessArch
$env:PROCESSOR_ARCHITEW6432 = $oldNativeArch
$env:PATH = $oldPath
Remove-Item Env:BSK_TEST_RC -ErrorAction SilentlyContinue
# Resolve and verify before recursive cleanup; only this fixture is removed.
$resolvedRoot = [IO.Path]::GetFullPath($root)
+2 -12
View File
@@ -55,20 +55,10 @@ Use this default loop:
```text
bsk navigate <url> --session <id>
bsk observe --session <id>
bsk click|hover|scroll-to|focus|blur|fill|select|press ... --session <id>
bsk click|hover|fill|select|press ... --session <id>
bsk observe --session <id> # after navigation or a meaningful DOM change
```
`bsk scroll-to <ref-or-selector> --session <id>` scrolls an element and its frame owners into view.
Use a fresh element ref for iframe/shadow-root targets; CSS selectors search the main document.
The result is the visible border-box portion's bounds in top-level viewport CSS pixels after
ancestor clipping. Partial visibility is enough; hidden or fully clipped targets fail with
`permission_denied` and `data.reason=element_not_visible`. This does not test occlusion by other elements.
For a specific tab or deadline: `bsk scroll-to @e3 --session <id> --tab-id 42 --timeout 5s`.
`bsk focus <ref>` explicitly focuses a target; `bsk blur <ref>` removes focus and reports whether
it was focused. Use these for UI states triggered by focus changes.
Prefer fresh `@eN` refs over CSS selectors. Navigation invalidates refs; large DOM changes may also
make them stale. Observe again before the next interaction.
@@ -124,7 +114,7 @@ This list of names is complete. Never invent a command outside it; read
session start|stop|list browsers status doctor update logs
navigate navigate-back navigate-forward reload wait-for-navigation wait-ms
observe snapshot get-html screenshot console network
click hover scroll-to focus blur fill select press evaluate
click hover fill select press evaluate
tab list|create|close|select|borrow|return window resize emulate
upload download request-help record start|stop
```