Compare commits

..
Author SHA1 Message Date
Ljy-0827 a6de73aea3 feat(vom): add typed visual refs and target identity validation 2026-09-10 12:21:20 +08:00
Ljy-0827 fecb38b44c refactor(vom): preserve complete canvas candidates with exact deduplication 2026-09-10 12:21:19 +08:00
Ljy-0827 52a8e65b34 fix(vom): fix border radius incorrectly reject canvas 2026-09-10 12:21:19 +08:00
Ljy-0827 eb6cc0419b feat(vom): discover canvas candidates and resolve visible regions 2026-09-10 12:21:19 +08:00
Zhang GH ba6a14de7c Merge pull request #127 from NianJiuZst/codex/scroll-to-element-primitive
feat(interaction): add scroll-to-element primitive
2026-09-10 11:04:06 +08:00
drakezhang fd0e9de61b docs: document scroll-to primitive contract 2026-09-10 10:15:57 +08:00
drakezhang 77e3f5e1f3 feat(dsh-plugin): expose scroll-to action 2026-09-10 00:05:52 +08:00
drakezhang 61c97c9cb5 fix(interaction): honor scroll cancellation and visible bounds 2026-09-10 00:05:48 +08:00
drakezhang 29355e041d merge: integrate main into scroll-to PR 2026-09-09 23:47:27 +08:00
Zhang GH cff1ea62da Merge pull request #126 from NianJiuZst/codex/focus-blur-primitive
feat(interaction): add explicit focus and blur primitives
2026-09-09 23:23:50 +08:00
drakezhang 20e8770006 feat(dsh-plugin): expose focus and blur actions 2026-09-09 22:47:47 +08:00
drakezhang ac9e7948bf fix(interaction): verify focus changes and honor cancellation 2026-09-09 22:47:42 +08:00
drakezhang aadc1376ff merge: integrate main into focus and blur PR 2026-09-09 22:33:48 +08:00
Zhang GH 0fe36bc0b1 Merge pull request #206 from Tencent/fix/oopif-scrollbar-geometry
fix(geometry): preserve OOPIF scale with occupied scrollbars
2026-09-09 17:45:21 +08:00
drakezhang a62a28d8cd fix(geometry): preserve OOPIF scale with occupied scrollbars 2026-09-09 16:58:25 +08:00
Hannah Liu 896c7944d8 Merge pull request #203 from Tencent/fix/capture-document-consistency
fix(observation): validate DOM identity across capture phases | 校验跨采集阶段的 DOM 身份一致性
2026-09-09 16:26:24 +08:00
Hannah Liu 7e5f3bae26 Merge pull request #200 from Tencent/fix/observation-document-consistency
refactor(observation):  unify cross-frame capture coordination and data ownership | 统一跨 frame 采集协调与数据归属
2026-09-09 16:25:52 +08:00
Hannah Liu af4c2fa229 Merge pull request #198 from Tencent/refactor/document-capture-facts
refactor(observation): decouple DOM processing from capture orchestration | 解耦 DOM 处理与采集流程
2026-09-09 16:24:42 +08:00
Hannah Liu ee10da6fe9 Merge pull request #195 from Tencent/fix/frame-geometry-contracts
fix(geometry): unify cross-frame coordinates and screenshot conversion | 统一跨 frame 坐标与截图转换
2026-09-09 16:24:06 +08:00
drakezhang 4834d8b5fa test(geometry): identify the regression browser attachment 2026-09-09 16:11:39 +08:00
Ljy-0827 b4006d5b85 perf(observation): validate document identity against snapshot roots 2026-09-09 16:09:09 +08:00
Ljy-0827 2d1e5c6d9d fix(observation): validate document consistency with bounded identity checks 2026-09-09 16:09:09 +08:00
drakezhang 282e6f5623 fix(observation): carry snapshot units through capture coordination 2026-09-09 16:05:42 +08:00
Ljy-0827 bd68c293b6 perf(observation): collect frame AX trees with bounded concurrency 2026-09-09 15:58:56 +08:00
Ljy-0827 35a20582ad refactor(observation): separate capture coordination from document consistency 2026-09-09 15:58:56 +08:00
Ljy-0827 4e3365450a refactor(observation): integrate capture orchestration and document consistency 2026-09-09 15:58:56 +08:00
Ljy-0827 df5ed1a495 fix(observation): validate document consistency during capture 2026-09-09 15:58:40 +08:00
drakezhang eefbe03258 fix(observation): preserve snapshot unit normalization after document extraction 2026-09-09 15:58:28 +08:00
Ljy-0827 1d275f047c perf(observation): reduce document processing scheduling overhead 2026-09-09 15:54:41 +08:00
Ljy-0827 388b3ae2d5 refactor(observation): isolate document processing from capture orchestration 2026-09-09 15:54:41 +08:00
Ljy-0827 36d0d91419 fix(observation): preserve snapshot document provenance and URL metadata 2026-09-09 15:54:41 +08:00
Ljy-0827 bcb9beb4f8 refactor(observation): defer visual facts until visual discovery 2026-09-09 15:54:41 +08:00
Ljy-0827 34ccb96fd6 fix(observation): await capture task cleanup before propagating cancellation 2026-09-09 15:54:41 +08:00
Ljy-0827 8dfa097e55 refactor(observation): organize capture around document facts 2026-09-09 15:54:41 +08:00
drakezhang 383e3128eb fix(geometry): normalize snapshot layout bounds and scroll together 2026-09-09 15:53:35 +08:00
Ljy-0827 2d2636030d perf(geometry): batch iframe owner size measurements 2026-09-09 13:20:21 +08:00
Ljy-0827 a28b9d55e7 fix(vom): surface incomplete iframe geometry in observations 2026-09-09 13:20:21 +08:00
Ljy-0827 c39874591f perf(observation): measure sibling iframe geometry with bounded concurrency 2026-09-09 13:20:21 +08:00
Ljy-0827 1485addfd5 fix(geometry): unify frame projection and screenshot coordinates 2026-09-09 13:20:21 +08:00
Zhang GH 90b0ff301b Merge pull request #204 from Tencent/release/0.2.1
Release/0.2.1
2026-09-09 11:35:05 +08:00
drakezhang e74f354e90 fix(release): support changelogs without unreleased section 2026-09-08 20:38:17 +08:00
drakezhang 05c036518a chore(release): 0.2.1 2026-09-08 20:35:14 +08:00
Zhang GH 020fb3212e Merge pull request #201 from Tencent/fix/windows-install-and-cancellation
fix(windows): harden installation and parent cancellation
2026-09-08 20:15:12 +08:00
Zhang GH 0822c53c65 fix(windows): preserve transfer cleanup after cancellation 2026-09-08 20:04:24 +08:00
Zhang GH 4450fbe28c Merge pull request #196 from Tencent/codex/dsh-plugin-docs
docs(dsh-plugin): clarify installation, configuration, and releases
2026-09-08 19:43:24 +08:00
drakezhang a3bb81bfab docs(dsh-plugin): clarify patch editing and reload behavior 2026-09-08 19:34:25 +08:00
Zhang GH caea98cd52 fix(windows): harden installation and parent cancellation 2026-09-08 19:21:54 +08:00
drakezhang 187b088013 docs(dsh-plugin): restore configuration guidance and simplify releases 2026-09-08 19:03:40 +08:00
drakezhang 5046b5ab44 docs(dsh-plugin): clarify setup and verify npm publishing 2026-09-08 18:44:20 +08:00
NianJiuZst 7a10be21cb feat(interaction): add scroll-to-element primitive 2026-08-21 20:56:24 +08:00
NianJiuZst 0055e90f91 feat(interaction): add explicit focus and blur primitives 2026-08-21 19:14:40 +08:00
105 changed files with 9590 additions and 705 deletions
+11 -3
View File
@@ -38,7 +38,7 @@ jobs:
run: cargo test --workspace --locked
windows-daemon:
name: Windows daemon and update regression tests
name: Windows daemon, installer and cancellation regression tests
runs-on: windows-latest
steps:
@@ -55,8 +55,16 @@ jobs:
- name: Run Windows update tests
run: cargo test -p bsk --lib --locked cli::update::tests
- name: Run Windows update process tests
run: cargo test -p bsk --test windows_update --locked
- name: Run Windows update and cancellation process tests
run: cargo test -p bsk --test windows_update --test windows_parent_cancel --locked
- name: Run installer tests (Windows PowerShell 5.1)
shell: powershell
run: ./scripts/install-windows.test.ps1
- name: Run installer tests (PowerShell 7)
shell: pwsh
run: ./scripts/install-windows.test.ps1
frontend:
name: Frontend lint, typecheck, tests, build
+24
View File
@@ -7,6 +7,30 @@ Starting from 0.2.0, CLI / Extension / DSH Plugin share the same version number.
## [Unreleased]
### Added
- [Scroll-to element primitive](docs/scroll-to.md) across CLI, Extension and DSH Plugin,
with ancestor-clipped visible bounds, iframe support and cooperative cancellation
## [0.2.1] - 2026-09-09
### Changed
- DSH Plugin: clarified installation, configuration, profile restart, and upgrade instructions
### Fixed
- CLI: reliable Windows self-updates, including executable replacement and daemon restart
- CLI: Windows named-pipe connection continuity and stale daemon process detection
- Windows installer: PowerShell 5.1 compatibility, path quoting, and staged executable replacement
- CLI / DSH Plugin: cancellation propagation and upload/download cleanup when the parent process exits
- Extension: tab ownership and session cleanup when agent tabs fail to open or borrowed tabs are returned or closed
- Extension: navigation recovery when other extensions restrict Chrome DevTools Protocol access
- Extension: background input filling, automatic append behavior, and verification of field values before reporting success
- Extension: match observed form state to the correct element after page changes
- Extension: UUID generation fallback for recording and overlays on insecure origins
- Browser locale matching for regional language variants
- DSH Plugin: observation state synchronization, thumbnail capture lifecycle, and stale preview cleanup
- DSH Plugin: removed the obsolete client runtime host dependency
## [0.2.0] - 2026-09-02
### Added
Generated
+2 -2
View File
@@ -134,7 +134,7 @@ dependencies = [
[[package]]
name = "bsk"
version = "0.2.0"
version = "0.2.1"
dependencies = [
"anyhow",
"base64",
@@ -169,7 +169,7 @@ dependencies = [
[[package]]
name = "bsk-protocol"
version = "0.2.0"
version = "0.2.1"
dependencies = [
"schemars",
"semver",
+2 -2
View File
@@ -7,7 +7,7 @@ members = [
default-members = ["crates/bsk-cli"]
[workspace.package]
version = "0.2.0"
version = "0.2.1"
edition = "2024"
rust-version = "1.85"
license = "MIT"
@@ -43,7 +43,7 @@ rand = "0.8"
uuid = { version = "1", features = ["v4", "serde"] }
# Workspace crates (version required for `cargo publish` of dependents)
bsk-protocol = { version = "0.2.0", path = "crates/bsk-protocol" }
bsk-protocol = { version = "0.2.1", path = "crates/bsk-protocol" }
[profile.release]
strip = "symbols"
+16 -7
View File
@@ -147,20 +147,26 @@ Start a new Agent session and write a prompt that needs the browser, for example
Using [DeepSeek Harness](https://github.com/deepseek-ai/deepseek-harness) (`dsh`)?
BrowserSkill ships a first-class dsh plugin on npm as
[`@wxg-prc-cpg/browser-skill-dsh-plugin`](https://www.npmjs.com/package/@wxg-prc-cpg/browser-skill-dsh-plugin).
It injects native `browser_*` tools (no shelling out to `bsk`) and a live Web UI
overlay of each Agent Window.
It gives the agent native `browser_*` tools and a live view of its browser sessions
in the Web UI. The plugin runs `bsk` on the agent's behalf.
Add it to a dsh profile, then start that profile:
Install the `bsk` CLI and connect the browser extension first. Then add the plugin
to a dsh profile and start it (replace `web` with your profile name):
```sh
dsh plugin --profile web add @wxg-prc-cpg/browser-skill-dsh-plugin
dsh --profile web
```
The plugin carries its own copy of the skill, so `bsk install-skill` is not needed
for dsh — but the `bsk` CLI and the browser extension are still prerequisites. See
the [plugin README](packages/dsh-plugin-browserskill/README.md) for the tool list,
configuration, and the observation overlay.
The plugin includes the `browser-skill` skill, so `bsk install-skill` is not needed
for dsh. Installed plugins do not update automatically. To upgrade this plugin:
```sh
dsh plugin --profile web update @wxg-prc-cpg/browser-skill-dsh-plugin --latest
```
Restart the profile after upgrading. See the
[plugin README](packages/dsh-plugin-browserskill/README.md) for usage and configuration.
## How It Works
@@ -201,6 +207,9 @@ through the [plugin](#deepseek-harness-plugin): the agent calls injected
## For Developers
The [scroll-to primitive reference](docs/scroll-to.md) covers its CLI, protocol
and plugin entry points, visible bounds and interruption behavior.
The repository is a Cargo + pnpm workspace:
- `crates/bsk-cli` — `bsk` CLI and local daemon
+11 -3
View File
@@ -125,16 +125,22 @@ bsk install-skill
## DeepSeek Harness 插件
在用 [DeepSeek Harness](https://github.com/deepseek-ai/deepseek-harness)(`dsh`)?BrowserSkill 提供了官方 dsh 插件,已发布到 npm:[`@wxg-prc-cpg/browser-skill-dsh-plugin`](https://www.npmjs.com/package/@wxg-prc-cpg/browser-skill-dsh-plugin)。它会注入原生 `browser_*` 工具(无需再通过 Shell 调用 `bsk`),并在 Web UI 中实时观察每个 Agent Window。
在用 [DeepSeek Harness](https://github.com/deepseek-ai/deepseek-harness)(`dsh`)?BrowserSkill 提供了官方 dsh 插件,已发布到 npm:[`@wxg-prc-cpg/browser-skill-dsh-plugin`](https://www.npmjs.com/package/@wxg-prc-cpg/browser-skill-dsh-plugin)。它为 Agent 提供原生 `browser_*` 工具,由插件代为调用 `bsk`,并在 Web UI 中实时展示浏览器会话。
把它装进某个 dsh profile,然后启动该 profile:
先安装 `bsk` CLI 并连接浏览器扩展,再将插件装进 dsh profile 并启动(将 `web` 替换为你的 profile 名称):
```sh
dsh plugin --profile web add @wxg-prc-cpg/browser-skill-dsh-plugin
dsh --profile web
```
插件自带 skill,所以在 dsh 下无需执行 `bsk install-skill`;但 `bsk` CLI 和浏览器扩展仍是前置条件。工具清单、配置项与观察浮层见[插件 README](packages/dsh-plugin-browserskill/README.md)。
插件自带 `browser-skill` skill,所以在 dsh 下无需执行 `bsk install-skill`。已安装的插件不会自动更新;升级此插件请运行:
```sh
dsh plugin --profile web update @wxg-prc-cpg/browser-skill-dsh-plugin --latest
```
升级后重启该 profile。用法与配置见[插件 README](packages/dsh-plugin-browserskill/README.md)。
## 工作原理
@@ -171,6 +177,8 @@ Agent 不直接与浏览器通信。它通过 `bsk` CLI 下发浏览器任务;
## 面向开发者
[scroll-to 原语说明](docs/scroll-to.md)介绍 CLI、协议和插件入口,以及可见区域、错误和中断语义。
本仓库是 Cargo + pnpm workspace:
- `crates/bsk-cli` — `bsk` CLI 与本地 daemon
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@browser-skill/extension",
"version": "0.2.0",
"version": "0.2.1",
"private": true,
"type": "module",
"scripts": {
@@ -29,6 +29,24 @@ function fakeApi() {
}
describe("ChromiumCdp", () => {
it("changes attachment identity only across real debugger attachments", async () => {
const { api, onEvent, onDetach } = fakeApi();
const cdp = new ChromiumCdp(api);
expect(cdp.getAttachmentId(4)).toBeUndefined();
await cdp.ensureAttached(4);
const first = cdp.getAttachmentId(4);
expect(first).toBeTruthy();
await cdp.ensureAttached(4);
onEvent.fire({ tabId: 4 }, "Target.attachedToTarget", { sessionId: "child" });
expect(cdp.getAttachmentId(4)).toBe(first);
onDetach.fire({ tabId: 4 }, "target_closed");
expect(cdp.getAttachmentId(4)).toBeUndefined();
await cdp.ensureAttached(4);
expect(cdp.getAttachmentId(4)).not.toBe(first);
await cdp.detach(4);
expect(cdp.getAttachmentId(4)).toBeUndefined();
});
it("discovers multiple iframe targets and recursively routes nested OOPIF commands", async () => {
const { api, onEvent } = fakeApi();
(api.sendCommand as ReturnType<typeof vi.fn>).mockImplementation(
@@ -164,6 +164,7 @@ async function settleBeforeDeadline(promises: Promise<void>[], deadline: number)
export class ChromiumCdp {
private readonly api: CdpDebuggerApi;
private readonly attachedTabs = new Set<number>();
private readonly attachmentIds = new Map<number, string>();
private readonly attachInFlight = new Map<number, Promise<void>>();
private readonly detachInFlight = new Map<number, Promise<void>>();
private readonly tabOwners = new Map<number, Set<string>>();
@@ -198,6 +199,11 @@ export class ChromiumCdp {
this.bindFrameTargetHandler();
}
/** Identity of this actual debugger attachment, independent of frame topology. */
getAttachmentId(tabId: number): string | undefined {
return this.attachmentIds.get(tabId);
}
/** Attach to `tabId` if we haven't already in this driver. */
async ensureAttached(tabId: number): Promise<void> {
// Returning a tab clears the cache before Chrome finishes detaching.
@@ -217,6 +223,7 @@ export class ChromiumCdp {
await this.enableConsoleDomains(tabId);
await this.enableNetworkDomainBestEffort(tabId);
this.attachedTabs.add(tabId);
this.attachmentIds.set(tabId, crypto.randomUUID());
await this.enableFrameDiscovery({ tabId }).catch((err) => {
console.debug("[bsk cdp] frame discovery unavailable", { tabId, err });
});
@@ -444,6 +451,7 @@ export class ChromiumCdp {
this.attachInFlight.delete(tabId);
if (!this.attachedTabs.has(tabId)) return;
this.attachedTabs.delete(tabId);
this.attachmentIds.delete(tabId);
this.clearDialogState(tabId);
this.clearConsoleState(tabId);
this.clearNetworkState(tabId);
@@ -502,6 +510,7 @@ export class ChromiumCdp {
this.attachInFlight.clear();
this.tabOwners.clear();
this.attachedTabs.clear();
this.attachmentIds.clear();
this.dialogBuffers.clear();
this.dialogSequences.clear();
this.consoleBuffers.clear();
@@ -848,6 +857,7 @@ export class ChromiumCdp {
const listener = (source: chrome.debugger.Debuggee, _reason: string) => {
if (typeof source.tabId === "number") {
this.attachedTabs.delete(source.tabId);
this.attachmentIds.delete(source.tabId);
this.attachInFlight.delete(source.tabId);
this.tabOwners.delete(source.tabId);
this.clearDialogState(source.tabId);
+42 -22
View File
@@ -1,19 +1,11 @@
/**
* Per-session map from `@e<N>` snapshot refs to a CDP node address.
*
* Each fresh `tool.snapshot` resets the store: M6 will call
* `replace(...)` with the new ref → node address pairs. A node address
* includes the owning flat CDP session/frame when the element lives in
* an OOPIF; tools resolve live geometry from that identity at call time.
*
* Refs are session-scoped (§7): looking up a ref in the wrong session
* returns `null`, never silently leaks. Storing values in different
* sessions is fine; they live in independent `RefStore` instances
* inside the `SessionContext`.
*/
import type { VisualCandidate } from "@/tools/vom/visual-discovery";
/** Session-local refs describe the latest observation. Reusing eN does not identify
* which observation a caller read; generation is internal bookkeeping only. */
export type BackendNodeId = number;
export interface RefEntry {
export interface DomRefEntry {
readonly kind: "dom";
backendNodeId: BackendNodeId;
tabId: number | null;
frameId?: string;
@@ -21,7 +13,15 @@ export interface RefEntry {
generation: number;
}
export interface VisualRefInput {
readonly kind: "visual-region";
readonly candidate: VisualCandidate;
}
export type RefEntry = DomRefEntry | (VisualRefInput & { readonly generation: number });
export type RefInput =
| VisualRefInput
| BackendNodeId
| {
backendNodeId: BackendNodeId;
@@ -31,7 +31,7 @@ export type RefInput =
};
export class RefStore {
private readonly map = new Map<string, RefEntry>();
private map = new Map<string, RefEntry>();
private generation = 0;
size(): number {
@@ -44,7 +44,7 @@ export class RefStore {
resolve(ref: string, opts: { tabId?: number } = {}): BackendNodeId | null {
const entry = this.map.get(normaliseRef(ref));
if (!entry) return null;
if (!entry || entry.kind !== "dom") return null;
if (opts.tabId !== undefined && entry.tabId !== opts.tabId) return null;
return entry.backendNodeId;
}
@@ -58,9 +58,11 @@ export class RefStore {
* Used after every fresh `tool.snapshot`.
*/
replace(entries: Iterable<readonly [string, RefInput]>): void {
this.map.clear();
this.generation += 1;
for (const [ref, input] of entries) this.map.set(normaliseRef(ref), this.entry(input));
const generation = this.generation + 1;
const next = new Map<string, RefEntry>();
for (const [ref, input] of entries) next.set(normaliseRef(ref), this.entry(input, generation));
this.map = next;
this.generation = generation;
}
set(
@@ -73,6 +75,7 @@ export class RefStore {
} = {},
): void {
this.map.set(normaliseRef(ref), {
kind: "dom",
backendNodeId: id,
tabId: opts.tabId ?? null,
...(opts.frameId ? { frameId: opts.frameId } : {}),
@@ -89,20 +92,37 @@ export class RefStore {
return this.map.entries();
}
private entry(input: RefInput): RefEntry {
private entry(input: RefInput, generation: number): RefEntry {
if (typeof input !== "number" && "kind" in input) {
const { document } = input.candidate;
if (
!document?.attachmentId ||
!document.frameId ||
!Number.isSafeInteger(document.target?.tabId) ||
!Number.isSafeInteger(document.documentElementBackendNodeId) ||
document.documentElementBackendNodeId <= 0 ||
!Number.isSafeInteger(input.candidate.backendNodeId) ||
input.candidate.backendNodeId <= 0
)
throw new TypeError("visual ref requires a verified DOM identity and anchor");
// Preserve the read-only evidence and shared clipping chain; do not clone ancestors per ref.
return { kind: "visual-region", candidate: input.candidate, generation };
}
if (typeof input === "number") {
return {
kind: "dom",
backendNodeId: input,
tabId: null,
generation: this.generation,
generation,
};
}
return {
kind: "dom",
backendNodeId: input.backendNodeId,
tabId: input.tabId,
...(input.frameId ? { frameId: input.frameId } : {}),
...(input.cdpSessionId ? { cdpSessionId: input.cdpSessionId } : {}),
generation: this.generation,
generation,
};
}
}
@@ -599,6 +599,79 @@ describe("ToolDispatcher", () => {
expect(onSessionsChanged).toHaveBeenCalledTimes(2);
});
it.each([
"focus",
"blur",
"scroll_to",
] as const)("routes %s with hover cleanup and cooperative cancellation", async (action) => {
const tab = { id: 7, windowId: 4242, active: true };
vi.stubGlobal("chrome", {
tabs: {
get: vi.fn(async () => tab),
query: vi.fn(async () => [tab]),
sendMessage: vi.fn(async () => undefined),
},
});
const sessions = new SessionManager({
agentWindow: {
create: async () => 4242,
remove: async () => {},
ensureActiveTab: async () => 7,
},
});
const ctx = await sessions.start("aa11");
ctx.refStore.set("e1", 12, { tabId: 7 });
const { transport, sent, deliver } = fakeTransport();
const onBrowserControlResumed = vi.fn();
let resolveNode: ((value: object) => void) | undefined;
const cdp = {
send: vi.fn(async (_tabId: number, method: string) => {
if (method === "DOM.resolveNode")
return new Promise((resolve) => {
resolveNode = resolve;
});
if (method === "Runtime.callFunctionOn") return { result: { value: true } };
return {};
}),
} as unknown as TestDispatcherCdp;
const dispatcher = new ToolDispatcher({ transport, sessions, cdp, onBrowserControlResumed });
const hover = dispatcher as unknown as {
rememberHover: (sessionId: string, result: object) => void;
setHoverBypass: (sessionId: string, tabId: number, enabled: boolean) => Promise<void>;
};
hover.rememberHover("aa11", { tab_id: 7, x: 10, y: 20 });
await hover.setHoverBypass("aa11", 7, true);
dispatcher.start();
deliver(makeRequest(`tool.${action}`, { session_id: "aa11", ref: "e1" }));
await vi.waitFor(() => expect(resolveNode).toBeDefined());
expect(onBrowserControlResumed).toHaveBeenCalledWith("aa11");
deliver({ id: "cancel-focus", method: "cancel", params: { rpc_id: "r-1" } });
resolveNode!({ object: { objectId: "focus-target" } });
await vi.waitFor(() => expect(sent).toHaveLength(2));
expect(sent).toContainEqual({ id: "cancel-focus", result: { cancelled: true } });
expect(sent).toContainEqual({
id: "r-1",
error: expect.objectContaining({ code: "cancelled" }),
});
expect(cdp.send).not.toHaveBeenCalledWith(7, "DOM.focus", expect.anything());
expect(cdp.send).not.toHaveBeenCalledWith(7, "Runtime.callFunctionOn", expect.anything());
if (action === "scroll_to") {
expect(cdp.send).toHaveBeenCalledWith(7, "Runtime.releaseObjectGroup", {
objectGroup: expect.any(String),
});
} else {
expect(cdp.send).toHaveBeenCalledWith(7, "Runtime.releaseObject", {
objectId: "focus-target",
});
}
expect(chrome.tabs.sendMessage).toHaveBeenCalledWith(
7,
expect.objectContaining({ enabled: false }),
);
expect(dispatcher.inflightAbortControllers.size).toBe(0);
dispatcher.stop();
});
it("invokes onBrowserControlResumed for browser-control tools but not passive reads", async () => {
vi.stubGlobal("chrome", {
tabs: {
@@ -436,6 +436,8 @@ describe("file transfer tools", () => {
if (method === "DOM.getContentQuads") {
return { quads: [[10, 20, 210, 20, 210, 120, 10, 120]] };
}
if (method === "Runtime.evaluate")
return { result: { value: { width: 400, height: 300 } } };
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 400, clientHeight: 300 } };
}
@@ -0,0 +1,166 @@
// @vitest-environment node
// Opt in with BSK_FOCUS_CHROME=/path/to/chrome; each test owns its browser/profile.
import { describe, expect, it } from "vitest";
import { SessionManager } from "@/session-manager/manager";
import { handleBlur, handleFocus } from "../interaction";
import type { CdpRunner } from "../shared";
type Send = <T = Record<string, unknown>>(
method: string,
params?: object,
sessionId?: string,
) => Promise<T>;
async function withFocusBrowser(
run: (h: {
evaluate: (expression: string) => Promise<unknown>;
ref: () => Promise<void>;
focus: (signal?: AbortSignal) => ReturnType<typeof handleFocus>;
blur: (signal?: AbortSignal) => ReturnType<typeof handleBlur>;
calls: string[];
afterCommand: { current: (method: string) => void };
}) => Promise<void>,
) {
// Reuse the existing isolated Chrome launcher; no new browser dependency.
const { withChrome } = await import(
new URL(
"../../../../../evals/browser/cases/regression/snapshot-coordinates/chrome.mjs",
import.meta.url,
).href
);
await withChrome(
{ executable: process.env.BSK_FOCUS_CHROME, deviceScale: 1, zoom: 1 },
async (send: Send) => {
const { targetId } = await send<{ targetId: string }>("Target.createTarget", {
url: "about:blank",
});
const { sessionId } = await send<{ sessionId: string }>("Target.attachToTarget", {
targetId,
flatten: true,
});
await send("Page.bringToFront", {}, sessionId);
const evaluate = async (expression: string) => {
const reply = await send<{ result: { value: unknown }; exceptionDetails?: unknown }>(
"Runtime.evaluate",
{ expression, returnByValue: true },
sessionId,
);
expect(reply.exceptionDetails).toBeUndefined();
return reply.result.value;
};
await evaluate(
`document.body.innerHTML = '<input id="target"><input id="other">'; window.probe = document.querySelector('#target');`,
);
const manager = new SessionManager({
agentWindow: {
create: async () => 100,
remove: async () => {},
ensureActiveTab: async () => 4,
},
});
const ctx = await manager.start("aa11");
const calls: string[] = [];
const afterCommand = { current: (_method: string) => {} };
const cdp: CdpRunner = {
send: async (_tabId, method, params) => {
calls.push(method);
const result = await send(method, params, sessionId);
afterCommand.current(method);
return result as never;
},
};
const deps = {
cdp,
tabsApi: {
get: async (id: number) => ({ id, windowId: 100, active: true }) as chrome.tabs.Tab,
query: async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab],
},
};
await run({
evaluate,
calls,
afterCommand,
ref: async () => {
const { result } = await send<{ result: { objectId: string } }>(
"Runtime.evaluate",
{ expression: "window.probe" },
sessionId,
);
const { node } = await send<{ node: { backendNodeId: number } }>(
"DOM.describeNode",
{ objectId: result.objectId },
sessionId,
);
await send("Runtime.releaseObject", { objectId: result.objectId }, sessionId);
ctx.refStore.set("e1", node.backendNodeId, { tabId: 4 });
},
focus: (signal) =>
handleFocus(manager, { session_id: "aa11", ref: "e1" }, { ...deps, signal }),
blur: (signal) =>
handleBlur(manager, { session_id: "aa11", ref: "e1" }, { ...deps, signal }),
});
},
);
}
describe.skipIf(!process.env.BSK_FOCUS_CHROME)("real browser focus and blur", () => {
it.each(["light", "open", "closed"])("verifies focus and blur in %s DOM", async (mode) => {
await withFocusBrowser(async (h) => {
if (mode !== "light")
await h.evaluate(`(() => {
const host = document.createElement('div'); document.body.append(host);
const nested = document.createElement('div'); host.attachShadow({mode:'${mode}'}).append(nested);
const input = document.createElement('input'); nested.attachShadow({mode:'${mode}'}).append(input);
window.probe = input;
})()`);
await h.ref();
expect(await h.focus()).toMatchObject({ focused: true });
expect(await h.evaluate("window.probe.matches(':focus')")).toBe(true);
expect(await h.blur()).toMatchObject({ was_focused: true, focused: false });
expect(await h.evaluate("window.probe.matches(':focus')")).toBe(false);
expect(h.calls.filter((method) => method === "Runtime.releaseObject")).toHaveLength(2);
});
});
it("detects focus changes from event-handler microtasks", async () => {
await withFocusBrowser(async (h) => {
await h.ref();
await h.evaluate(
"window.probe.addEventListener('focus', () => queueMicrotask(() => document.querySelector('#other').focus()), {once:true})",
);
expect(await h.focus()).toMatchObject({ code: "cdp_failed" });
expect(await h.focus()).toMatchObject({ focused: true });
await h.evaluate(
"window.probe.addEventListener('blur', () => queueMicrotask(() => window.probe.focus()), {once:true})",
);
expect(await h.blur()).toMatchObject({ code: "cdp_failed" });
expect(await h.evaluate("window.probe.matches(':focus')")).toBe(true);
});
});
it("cancels before blur while still releasing the resolved object", async () => {
await withFocusBrowser(async (h) => {
await h.ref();
await h.focus();
const abort = new AbortController();
h.afterCommand.current = (method) => {
if (method === "DOM.resolveNode") abort.abort();
};
expect(await h.blur(abort.signal)).toMatchObject({ code: "cancelled" });
expect(await h.evaluate("window.probe.matches(':focus')")).toBe(true);
expect(h.calls.at(-1)).toBe("Runtime.releaseObject");
});
});
it("reports a page exception with its cause", async () => {
await withFocusBrowser(async (h) => {
await h.ref();
await h.evaluate("window.probe.blur = () => { throw new Error('page blur failed'); }");
expect(await h.blur()).toMatchObject({
code: "cdp_failed",
message: expect.stringContaining("page blur failed"),
});
expect(h.calls.at(-1)).toBe("Runtime.releaseObject");
});
});
});
@@ -0,0 +1,293 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import type { CdpTarget } from "@/browser-driver/frame-graph";
import { SessionManager } from "@/session-manager/manager";
import { handleBlur, handleFocus } from "../interaction";
import type { CdpRunner } from "../shared";
interface ScriptParams {
functionDeclaration: string;
}
interface Call {
target: CdpTarget;
method: string;
}
async function setup(element?: HTMLElement, childSession = false) {
if (!element) {
document.body.innerHTML = '<input id="target"><input id="other">';
element = document.querySelector<HTMLInputElement>("#target")!;
}
const targetElement = element;
const manager = new SessionManager({
agentWindow: {
create: async () => 100,
remove: async () => {},
ensureActiveTab: async () => 4,
},
});
const ctx = await manager.start("aa11");
ctx.refStore.set("e1", 12, {
tabId: 4,
...(childSession ? { frameId: "child", cdpSessionId: "child-session" } : {}),
});
const calls: Call[] = [];
const afterCommand = vi.fn<(call: Call) => void>();
const script = vi.fn(async (params: ScriptParams) => {
try {
const fn = new Function(`return (${params.functionDeclaration})`)();
return { result: { value: await fn.call(targetElement) } };
} catch (error) {
return { exceptionDetails: { text: "Uncaught", exception: { description: String(error) } } };
}
});
const focus = vi.fn(() => targetElement.focus());
const release = vi.fn(async () => ({}));
const send = async (target: CdpTarget, method: string, params?: object) => {
const call = { target, method };
calls.push(call);
try {
switch (method) {
case "DOM.getDocument":
return { root: { nodeId: 1 } };
case "DOM.querySelector":
return { nodeId: 2 };
case "DOM.describeNode":
return { node: { backendNodeId: 12 } };
case "DOM.scrollIntoViewIfNeeded":
return {};
case "DOM.resolveNode":
return { object: { objectId: "focus-target" } };
case "DOM.focus":
focus();
return {};
case "Runtime.callFunctionOn":
return await script(params as ScriptParams);
case "Runtime.releaseObject":
return await release();
default:
throw new Error(`unexpected ${method}`);
}
} finally {
afterCommand(call);
}
};
const cdp: CdpRunner = {
send: (tabId, method, params) => send({ tabId }, method, params) as never,
sendToTarget: send as CdpRunner["sendToTarget"],
getFrameGraph: vi.fn(async () => ({
rootFrameId: "main",
frames: [
{ frameId: "main", target: { tabId: 4 } },
...(childSession
? [
{
frameId: "child",
parentFrameId: "main",
ownerBackendNodeId: 99,
target: { tabId: 4, sessionId: "child-session" },
},
]
: []),
],
})),
};
const tabsApi = {
get: vi.fn(async (id: number) => ({ id, windowId: 100, active: true }) as chrome.tabs.Tab),
query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
};
return {
element: targetElement,
calls,
afterCommand,
script,
focus,
release,
cdp,
tabsApi,
run: (
action: "focus" | "blur",
signal?: AbortSignal,
params: { ref?: string; selector?: string; tab_id?: number } = { ref: "e1" },
) =>
(action === "focus" ? handleFocus : handleBlur)(
manager,
{ session_id: "aa11", ...params },
{ cdp, tabsApi, signal },
),
};
}
afterEach(() => {
vi.restoreAllMocks();
document.body.innerHTML = "";
});
describe("focus and blur state verification", () => {
it.each(["open", "closed"] as const)("handles nested %s shadow roots", async (mode) => {
const host = document.createElement("div");
document.body.append(host);
const outer = host.attachShadow({ mode });
const innerHost = document.createElement("div");
outer.append(innerHost);
const inner = innerHost.attachShadow({ mode });
const element = document.createElement("input");
inner.append(element);
const h = await setup(element);
expect(await h.run("focus")).toMatchObject({ focused: true });
expect(inner.activeElement).toBe(element);
expect(await h.run("blur")).toMatchObject({ was_focused: true, focused: false });
expect(inner.activeElement).toBeNull();
expect(h.release).toHaveBeenCalledTimes(2);
});
it("keeps DOM focus usable when the document is in the background", async () => {
const h = await setup();
vi.spyOn(document, "hasFocus").mockReturnValue(false);
expect(await h.run("focus")).toMatchObject({ focused: true });
expect(await h.run("blur")).toMatchObject({ was_focused: true, focused: false });
});
it("blurring an unfocused target preserves the other element's focus", async () => {
const h = await setup();
const other = document.querySelector<HTMLInputElement>("#other")!;
other.focus();
expect(await h.run("blur")).toMatchObject({ was_focused: false, focused: false });
expect(document.activeElement).toBe(other);
});
it("detects focus redirected by a focus handler's microtask", async () => {
const h = await setup();
h.element.addEventListener("focus", () =>
queueMicrotask(() => document.querySelector<HTMLInputElement>("#other")!.focus()),
);
expect(await h.run("focus")).toMatchObject({
code: "cdp_failed",
message: "target element did not become focused",
});
expect(h.release).toHaveBeenCalledOnce();
});
it("does not report blur success when a microtask restores focus", async () => {
const h = await setup();
h.element.focus();
h.element.addEventListener("blur", () => queueMicrotask(() => h.element.focus()));
expect(await h.run("blur")).toMatchObject({
code: "cdp_failed",
message: "target element remained focused after blur()",
});
expect(document.activeElement).toBe(h.element);
expect(h.release).toHaveBeenCalledOnce();
});
it.each(["focus", "blur"] as const)("rejects a detached %s target", async (action) => {
const h = await setup();
h.element.remove();
expect(await h.run(action)).toMatchObject({ code: "cdp_failed" });
expect(h.release).toHaveBeenCalledOnce();
});
it("preserves a script exception's cause", async () => {
const h = await setup();
vi.spyOn(h.element, "blur").mockImplementation(() => {
throw new Error("page blur failed");
});
expect(await h.run("blur")).toMatchObject({
code: "cdp_failed",
message: expect.stringContaining("page blur failed"),
});
expect(h.release).toHaveBeenCalledOnce();
});
it.each(["focus", "blur"] as const)("rejects malformed %s script results", async (action) => {
const h = await setup();
h.script.mockResolvedValueOnce({ result: { value: null } });
expect(await h.run(action)).toMatchObject({
code: "cdp_failed",
message: expect.stringContaining("unexpected result"),
});
expect(h.release).toHaveBeenCalledOnce();
});
it.each([
"focus",
"blur",
] as const)("keeps %s, verification and cleanup in the OOPIF session", async (action) => {
const h = await setup(undefined, true);
if (action === "blur") h.element.focus();
expect(await h.run(action)).toMatchObject({ focused: action === "focus" });
const targetCalls = h.calls.filter((call) => call.method !== "DOM.scrollIntoViewIfNeeded");
expect(targetCalls.every((call) => call.target.sessionId === "child-session")).toBe(true);
expect(targetCalls.at(-1)?.method).toBe("Runtime.releaseObject");
});
it.each(["focus", "blur"] as const)("rejects %s on a user tab before CDP", async (action) => {
const h = await setup();
h.tabsApi.get.mockResolvedValue({ id: 9, windowId: 200, active: true } as chrome.tabs.Tab);
expect(await h.run(action, undefined, { ref: "e1", tab_id: 9 })).toMatchObject({
code: "permission_denied",
});
expect(h.calls).toEqual([]);
});
it("does not replace a successful result when navigation disposed the remote object", async () => {
const h = await setup();
h.release.mockRejectedValue(new Error("Cannot find context"));
expect(await h.run("focus")).toMatchObject({ focused: true });
});
});
describe("focus and blur cancellation", () => {
const boundaries = [
"DOM.getDocument",
"DOM.querySelector",
"DOM.describeNode",
"DOM.resolveNode",
"Runtime.callFunctionOn",
];
for (const action of ["focus", "blur"] as const) {
it.each(
action === "focus" ? [...boundaries, "DOM.scrollIntoViewIfNeeded", "DOM.focus"] : boundaries,
)(`${action} stops after cancellation during %s`, async (method) => {
const h = await setup();
const abort = new AbortController();
let cancelledAfter = 0;
h.afterCommand.mockImplementation((call) => {
if (call.method === method && !abort.signal.aborted) {
cancelledAfter = h.calls.length;
abort.abort();
}
});
expect(await h.run(action, abort.signal, { selector: "#target" })).toMatchObject({
code: "cancelled",
});
expect(abort.signal.aborted).toBe(true);
expect(
h.calls.slice(cancelledAfter).every((call) => call.method === "Runtime.releaseObject"),
).toBe(true);
if (h.calls.some((call) => call.method === "DOM.resolveNode"))
expect(h.release).toHaveBeenCalledOnce();
});
it(`${action} stops if cancellation arrives while resolving the tab`, async () => {
const h = await setup();
const abort = new AbortController();
h.tabsApi.query.mockImplementation(async () => {
abort.abort();
return [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab];
});
expect(await h.run(action, abort.signal)).toMatchObject({ code: "cancelled" });
expect(h.calls).toEqual([]);
});
}
it("does not scroll frame owners after cancellation in an OOPIF", async () => {
const h = await setup(undefined, true);
const abort = new AbortController();
h.afterCommand.mockImplementation((call) => {
if (call.method === "DOM.scrollIntoViewIfNeeded") abort.abort();
});
expect(await h.run("focus", abort.signal)).toMatchObject({ code: "cancelled" });
expect(h.calls.filter((call) => call.method === "DOM.scrollIntoViewIfNeeded")).toHaveLength(1);
expect(h.focus).not.toHaveBeenCalled();
});
});
@@ -13,7 +13,138 @@ import {
import { GeometryContext } from "../geometry/frame-context";
import type { CdpRunner } from "../shared";
function scrollbarDriver(
visible = { width: 185, height: 89 },
frameSize: unknown = { width: 200, height: 100 },
) {
const send = vi.fn(async (target: { sessionId?: string }, method: string) => {
if (method === "Runtime.evaluate") return { result: { value: frameSize } };
if (method === "Page.getLayoutMetrics")
return {
cssLayoutViewport: {
clientWidth: target.sessionId ? visible.width : 800,
clientHeight: target.sessionId ? visible.height : 600,
},
};
if (method === "DOM.getBoxModel")
return {
model: {
content: target.sessionId
? [0, 0, 200, 0, 200, 100, 0, 100]
: [100, 100, 500, 100, 500, 300, 100, 300],
},
};
if (method === "DOM.getContentQuads")
return { quads: [[180, 80, 230, 80, 230, 120, 180, 120]] };
throw new Error(`unexpected ${method}`);
});
const cdp: CdpRunner = {
send: (tabId, method) => send({ tabId } as never, method) as never,
sendToTarget: send as CdpRunner["sendToTarget"],
getFrameGraph: async () => ({
rootFrameId: "main",
frames: [
{ frameId: "main", target: { tabId: 4 } },
{
frameId: "child",
parentFrameId: "main",
ownerBackendNodeId: 10,
target: { tabId: 4, sessionId: "child" },
},
{
frameId: "same-child",
parentFrameId: "child",
ownerBackendNodeId: 20,
target: { tabId: 4, sessionId: "child" },
},
],
}),
};
return { cdp, send };
}
describe("frame geometry projection", () => {
it.each([
{ width: 200, height: 100 },
{ width: 185, height: 100 },
{ width: 200, height: 89 },
{ width: 185, height: 89 },
])("preserves scale and clips scrollbar strips for visible viewport $width × $height", async (visible) => {
const { cdp, send } = scrollbarDriver(visible);
const geometry = await resolveNodeGeometry(cdp, 4, {
target: { tabId: 4, sessionId: "child" },
frameId: "child",
backendNodeId: 101,
});
expect(geometry).toMatchObject({
topBounds: {
x: 460,
y: 260,
width: (visible.width - 180) * 2,
height: (visible.height - 80) * 2,
},
actionPoint: { x: 460 + visible.width - 180, y: 260 + visible.height - 80 },
targetActionPoint: { x: (180 + visible.width) / 2, y: (80 + visible.height) / 2 },
});
expect(send.mock.calls.filter(([, method]) => method === "Runtime.evaluate")).toHaveLength(1);
expect(send.mock.calls.filter(([, method]) => method === "Page.getLayoutMetrics")).toHaveLength(
2,
);
});
it("shares the full viewport read across same-target frames, and refreshes it next operation", async () => {
const { cdp, send } = scrollbarDriver();
const context = new GeometryContext(cdp, 4);
await Promise.all([context.targetProjection("child"), context.targetProjection("same-child")]);
expect(send.mock.calls.filter(([, method]) => method === "Runtime.evaluate")).toHaveLength(1);
await new GeometryContext(cdp, 4).targetProjection("child");
expect(send.mock.calls.filter(([, method]) => method === "Runtime.evaluate")).toHaveLength(2);
});
it.each([
null,
{},
{ width: 0, height: 100 },
{ width: 200, height: NaN },
])("rejects unavailable full viewport dimensions: %j", async (size) => {
const { cdp } = scrollbarDriver(undefined, size);
expect(await new GeometryContext(cdp, 4).targetProjection("child")).toBeNull();
});
it("clips an inner OOPIF against an intermediate OOPIF's occupied scrollbar strips", async () => {
const { cdp } = scrollbarDriver();
const outer = cdp.sendToTarget!;
cdp.sendToTarget = async (target, method, params) => {
if (method === "DOM.getBoxModel")
return { model: { content: [150, 60, 250, 60, 250, 120, 150, 120] } } as never;
if (target.sessionId !== "inner") return outer(target, method, params);
if (method === "Runtime.evaluate")
return { result: { value: { width: 100, height: 60 } } } as never;
if (method === "Page.getLayoutMetrics")
return { cssLayoutViewport: { clientWidth: 90, clientHeight: 50 } } as never;
if (method === "DOM.getContentQuads")
return { quads: [[0, 0, 100, 0, 100, 60, 0, 60]] } as never;
throw new Error(`unexpected ${method}`);
};
const graph = await cdp.getFrameGraph!(4);
graph.frames[2] = {
frameId: "inner",
parentFrameId: "child",
ownerBackendNodeId: 20,
target: { tabId: 4, sessionId: "inner" },
};
cdp.getFrameGraph = async () => graph;
expect(
await resolveNodeGeometry(cdp, 4, {
target: { tabId: 4, sessionId: "inner" },
frameId: "inner",
backendNodeId: 101,
}),
).toMatchObject({
topBounds: { x: 400, y: 220, width: 70, height: 58 },
});
});
it("keeps region bounds separate from polygon area", () => {
const region = [
rectPolygon({ x: 0, y: 0, w: 10, h: 10 }),
@@ -77,6 +208,8 @@ describe("frame geometry projection", () => {
const cdp: CdpRunner = {
send: send as CdpRunner["send"],
sendToTarget: vi.fn(async (_target, method) => {
if (method === "Runtime.evaluate")
return { result: { value: { width: 200, height: 100 } } };
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
}
@@ -135,6 +268,8 @@ describe("frame geometry projection", () => {
throw new Error(`unexpected root command ${method}`);
}) as CdpRunner["send"],
sendToTarget: vi.fn(async (_target, method) => {
if (method === "Runtime.evaluate")
return { result: { value: { width: 200, height: 100 } } };
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
}
@@ -196,6 +331,8 @@ describe("frame geometry projection", () => {
}) as CdpRunner["send"],
sendToTarget: vi.fn(async (_target, method) => {
if (method === "DOM.scrollIntoViewIfNeeded") return {};
if (method === "Runtime.evaluate")
return { result: { value: { width: 200, height: 100 } } };
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
}
@@ -5,9 +5,10 @@ import { rectPolygon } from "../geometry";
import {
projectSnapshotRect,
screenshotPageRect,
snapshotCoordinates,
snapshotViewportRect,
} from "../geometry/coordinate-types";
import { GeometryContext } from "../geometry/frame-context";
import { GeometryContext, snapshotLayoutScale } from "../geometry/frame-context";
import type { CdpRunner } from "../shared";
const target = { tabId: 4 };
@@ -22,7 +23,11 @@ const graph: CdpFrameGraph = {
function driver() {
const send = vi.fn(async (_tab: number, method: string): Promise<object> => {
if (method === "Page.getLayoutMetrics")
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 1000 } };
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 1000 },
};
if (method === "DOM.getBoxModel")
return { model: { content: [52.5, 612.5, 427.5, 612.5, 427.5, 862.5, 52.5, 862.5] } };
if (method === "DOM.resolveNode") return { object: { objectId: "owner" } };
@@ -35,6 +40,63 @@ function driver() {
}
describe("measurement geometry context", () => {
it.each([0.8, 1, 1.25, 2])("normalizes snapshot layout units at scale %s", (scale) => {
const measured = snapshotLayoutScale({
visualViewport: { clientWidth: 997.25, zoom: 1.1 },
cssVisualViewport: { clientWidth: 997.25 / scale, zoom: 1.1 },
// Rounded layout dimensions must not supply the scale.
layoutViewport: { clientWidth: 997 },
cssLayoutViewport: { clientWidth: Math.floor(997.25 / scale) },
});
expect(measured).toBeCloseTo(scale, 12);
const coordinates = snapshotCoordinates(
{ scrollOffsetX: 80 * scale, scrollOffsetY: 200 * scale },
measured,
{ x: 999, y: 999 },
);
expect(
snapshotViewportRect(
[160 * scale, 320 * scale, 120 * scale, 40 * scale],
{ target },
coordinates,
)?.rect,
).toEqual({ x: 80, y: 120, width: 120, height: 40 });
});
it("uses CSS root scroll only for missing snapshot offsets, without scaling it twice", () => {
const coordinates = snapshotCoordinates({ scrollOffsetX: 80 }, 2, { x: 999, y: 100 });
expect(snapshotViewportRect([200, 800, 240, 80], { target }, coordinates)?.rect).toEqual({
x: 60,
y: 300,
width: 120,
height: 40,
});
expect(snapshotCoordinates({ scrollOffsetX: 0 }, 2)).toBeNull();
expect(
snapshotCoordinates({ scrollOffsetX: NaN, scrollOffsetY: 0 }, 2, { x: 0, y: 0 }),
).toBeNull();
});
it("does not infer snapshot units from missing, invalid, or CSS-only metrics", () => {
expect(snapshotLayoutScale({ cssLayoutViewport: { clientWidth: 800 } })).toBeNull();
for (const raw of [0, -1, NaN, Infinity]) {
expect(
snapshotLayoutScale({
visualViewport: { clientWidth: raw },
cssVisualViewport: { clientWidth: 800 },
}),
).toBeNull();
}
expect(
snapshotLayoutScale({
visualViewport: { clientWidth: 0, clientHeight: 600 },
cssVisualViewport: { clientWidth: 0, clientHeight: 300 },
}),
).toBe(2);
expect(snapshotCoordinates({ scrollOffsetX: 0, scrollOffsetY: 0 }, null)).toBeNull();
expect(snapshotViewportRect([10, 20, 120, 40], { target }, null)).toBeNull();
});
it("shares in-flight frame measurements, but never shares them with the next context", async () => {
const cdp = driver();
const context = new GeometryContext(cdp, 4);
@@ -61,14 +123,20 @@ describe("measurement geometry context", () => {
height: 1000,
});
if (projection.status !== "available") throw new Error("expected available projection");
const input = snapshotViewportRect([17, 23, 120, 40], source, { x: 0, y: 0 });
const input = snapshotViewportRect([17, 23, 120, 40], source, {
layoutUnitsPerCssPixel: 1,
scrollCss: { x: 0, y: 0 },
});
expect(projectSnapshotRect(input!, projection.projection)).toEqual({
x: 73.75,
y: 641.25,
width: 150,
height: 50,
});
const scrolled = snapshotViewportRect([17, 23, 120, 40], source, { x: 0, y: 10 });
const scrolled = snapshotViewportRect([17, 23, 120, 40], source, {
layoutUnitsPerCssPixel: 1,
scrollCss: { x: 0, y: 10 },
});
expect(projectSnapshotRect(scrolled!, projection.projection)?.y).toBe(628.75);
await context.snapshotProjection(source, 10, [], { width: 1000, height: 1000 });
expect(cdp.calls.mock.calls.filter(([, method]) => method === "DOM.resolveNode")).toHaveLength(
@@ -79,11 +147,101 @@ describe("measurement geometry context", () => {
).toHaveLength(1);
});
it("falls back only for missing or invalid batch entries and does not reuse sizes across contexts", async () => {
const cdp = driver();
const original = cdp.send;
const calls: string[] = [];
cdp.send = async (tabId, method, params) => {
calls.push(method);
if (method === "Runtime.evaluate")
return {
result: {
deepSerializedValue: {
type: "array",
value: [
{
type: "array",
value: [
{ type: "node", value: { backendNodeId: 10 } },
{ type: "string", value: '{"width":300,"height":200}' },
],
},
{
type: "array",
value: [
{ type: "node", value: { backendNodeId: 11 } },
{ type: "string", value: '{"width":0,"height":200}' },
],
},
],
},
},
} as never;
if (method === "Runtime.releaseObjectGroup") return {} as never;
return original(tabId, method, params);
};
const context = new GeometryContext(cdp, 4);
context.registerSnapshotOwners(target, new Set([10, 11]));
const values = await Promise.all(
[10, 11].map((id) =>
context.snapshotProjection({ target, frameId: String(id) }, id, [], {
width: 1000,
height: 1000,
}),
),
);
expect(values.every((value) => value.status === "available")).toBe(true);
expect(calls.filter((method) => method === "DOM.resolveNode")).toHaveLength(1);
expect(calls.filter((method) => method === "Runtime.evaluate")).toHaveLength(1);
await new GeometryContext(cdp, 4).snapshotProjection({ target, frameId: "child" }, 10, [], {
width: 1000,
height: 1000,
});
expect(calls.filter((method) => method === "DOM.resolveNode")).toHaveLength(2);
});
it("waits for batch object cleanup and propagates cancellation", async () => {
const cdp = driver();
const original = cdp.send;
const controller = new AbortController();
let release!: () => void;
let cleanup!: () => void;
const started = new Promise<void>((resolve) => {
cleanup = resolve;
});
cdp.send = async (tabId, method, params) => {
if (method === "Runtime.evaluate") return {} as never;
if (method === "Runtime.releaseObjectGroup") {
cleanup();
await new Promise<void>((resolve) => {
release = resolve;
});
return {} as never;
}
return original(tabId, method, params);
};
const context = new GeometryContext(cdp, 4, undefined, controller.signal);
context.registerSnapshotOwners(target, new Set([10, 11]));
let settled = false;
const operation = context
.snapshotProjection({ target, frameId: "child" }, 10, [], { width: 1000, height: 1000 })
.finally(() => {
settled = true;
});
const rejected = expect(operation).rejects.toMatchObject({ name: "AbortError" });
await started;
controller.abort();
expect(settled).toBe(false);
release();
await rejected;
expect(cdp.calls.mock.calls.some(([, method]) => method === "DOM.resolveNode")).toBe(false);
});
it("rejects owner mismatches and invalid numeric bounds", () => {
const input = snapshotViewportRect(
[30, 50, 120, 40],
{ target, frameId: "main" },
{ x: 10, y: 20 },
{ layoutUnitsPerCssPixel: 1, scrollCss: { x: 10, y: 20 } },
)!;
const projection = {
source: { target, frameId: "main" },
@@ -104,8 +262,20 @@ describe("measurement geometry context", () => {
source: { target: { tabId: 5 }, frameId: "main" },
}),
).toBeNull();
expect(snapshotViewportRect([0, 0, NaN, 40], { target }, { x: 0, y: 0 })).toBeNull();
expect(snapshotViewportRect([0, 0, 10, 40], { target }, { x: Infinity, y: 0 })).toBeNull();
expect(
snapshotViewportRect(
[0, 0, NaN, 40],
{ target },
{ layoutUnitsPerCssPixel: 1, scrollCss: { x: 0, y: 0 } },
),
).toBeNull();
expect(
snapshotViewportRect(
[0, 0, 10, 40],
{ target },
{ layoutUnitsPerCssPixel: 1, scrollCss: { x: Infinity, y: 0 } },
),
).toBeNull();
});
it("does not turn a border quad or a failed owner read into a content projection", async () => {
@@ -138,7 +308,7 @@ describe("measurement geometry context", () => {
const input = snapshotViewportRect(
[0, 0, 300, 200],
{ target, frameId: "nested" },
{ x: 0, y: 0 },
{ layoutUnitsPerCssPixel: 1, scrollCss: { x: 0, y: 0 } },
);
expect(projectSnapshotRect(input!, projection.projection)).toEqual({
x: 60,
@@ -188,7 +358,11 @@ describe("measurement geometry context", () => {
peak = Math.max(peak, active);
await new Promise<void>((resolve) => release.push(resolve));
active--;
return { cssLayoutViewport: { clientWidth: 100, clientHeight: 100 } };
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 100, clientHeight: 100 },
};
});
const controller = new AbortController();
const context = new GeometryContext(
@@ -405,6 +405,7 @@ describe("handleRequestHelp", () => {
agentWindowId: 99,
refStore: {
resolveEntry: () => ({
kind: "dom",
backendNodeId: 42,
tabId: 5,
frameId: "child",
@@ -433,6 +434,7 @@ describe("handleRequestHelp", () => {
if (method === "DOM.getContentQuads") {
return { quads: [[10, 20, 110, 20, 110, 60, 10, 60]] };
}
if (method === "Runtime.evaluate") return { result: { value: { width: 200, height: 100 } } };
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
}
@@ -2,8 +2,10 @@ import { describe, expect, it, vi } from "vitest";
import { SessionManager } from "@/session-manager/manager";
import type { CdpRunner } from "@/tools/shared";
import {
handleBlur,
handleClick,
handleFill,
handleFocus,
handleHover,
handlePress,
handleSelect,
@@ -201,6 +203,7 @@ describe("handleClick", () => {
if (method === "DOM.getContentQuads") {
return { quads: [[10, 20, 110, 20, 110, 60, 10, 60]] };
}
if (method === "Runtime.evaluate") return { result: { value: { width: 200, height: 100 } } };
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
}
@@ -220,6 +223,7 @@ describe("handleClick", () => {
{ sessionId: "child-session", method: "DOM.scrollIntoViewIfNeeded" },
{ sessionId: "child-session", method: "DOM.getContentQuads" },
{ sessionId: "child-session", method: "Page.getLayoutMetrics" },
{ sessionId: "child-session", method: "Runtime.evaluate" },
]);
expect(fake.sent.filter((call) => call.method === "Input.dispatchMouseEvent")).toHaveLength(3);
});
@@ -540,6 +544,161 @@ describe("handleHover", () => {
});
});
describe("handleFocus and handleBlur", () => {
it("focuses a ref and verifies the deep active element", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
ctx.refStore.set("e3", 1234, { tabId: 4 });
const fake = makeFakeCdp({
"DOM.scrollIntoViewIfNeeded": () => ({}),
"DOM.focus": () => ({}),
"DOM.resolveNode": () => ({ object: { objectId: "focus-target" } }),
"Runtime.callFunctionOn": () => ({ result: { value: true } }),
});
const res = await handleFocus(
sm,
{ session_id: "aa11", ref: "@e3" },
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
);
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
expect(res).toMatchObject({ tab_id: 4, used_ref: "e3", focused: true });
expect(fake.sent.map((call) => call.method)).toEqual([
"DOM.scrollIntoViewIfNeeded",
"DOM.resolveNode",
"DOM.focus",
"Runtime.callFunctionOn",
"Runtime.releaseObject",
]);
});
it("focuses an OOPIF ref in its CDP session", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
ctx.refStore.set("e3", 1234, {
tabId: 4,
frameId: "child-frame",
cdpSessionId: "child-session",
});
const fake = makeFakeCdp({
"DOM.scrollIntoViewIfNeeded": () => ({}),
});
fake.cdp.getFrameGraph = vi.fn(async () => ({
rootFrameId: "main",
frames: [
{ frameId: "main", target: { tabId: 4 } },
{
frameId: "child-frame",
parentFrameId: "main",
ownerBackendNodeId: 99,
target: { tabId: 4, sessionId: "child-session" },
},
],
}));
const targetCalls: Array<{ sessionId?: string; method: string }> = [];
fake.cdp.sendToTarget = vi.fn(async (target, method) => {
targetCalls.push({ sessionId: target.sessionId, method });
if (method === "DOM.scrollIntoViewIfNeeded") return {};
if (method === "DOM.focus" || method === "Runtime.releaseObject") return {};
if (method === "DOM.resolveNode") return { object: { objectId: "focus-target" } };
if (method === "Runtime.callFunctionOn") return { result: { value: true } };
throw new Error(`unexpected child CDP call ${method}`);
}) as CdpRunner["sendToTarget"];
const res = await handleFocus(
sm,
{ session_id: "aa11", ref: "@e3" },
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
);
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
expect(res.focused).toBe(true);
expect(targetCalls).toEqual([
{ sessionId: "child-session", method: "DOM.scrollIntoViewIfNeeded" },
{ sessionId: "child-session", method: "DOM.resolveNode" },
{ sessionId: "child-session", method: "DOM.focus" },
{ sessionId: "child-session", method: "Runtime.callFunctionOn" },
{ sessionId: "child-session", method: "Runtime.releaseObject" },
]);
});
it("does not focus after cancellation during scrolling", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
ctx.refStore.set("e3", 1234, { tabId: 4 });
const abort = new AbortController();
const fake = makeFakeCdp({
"DOM.scrollIntoViewIfNeeded": () => {
abort.abort();
return {};
},
});
const res = await handleFocus(
sm,
{ session_id: "aa11", ref: "@e3" },
{ cdp: fake.cdp, tabsApi: fake.tabsApi, signal: abort.signal },
);
expect(res).toMatchObject({ code: "cancelled" });
expect(fake.sent.some((call) => call.method === "DOM.focus")).toBe(false);
});
it("blurs a ref and returns its previous focus state", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
ctx.refStore.set("e3", 1234, { tabId: 4 });
const fake = makeFakeCdp({
"DOM.resolveNode": () => ({ object: { objectId: "focus-target" } }),
"Runtime.callFunctionOn": vi
.fn()
.mockResolvedValueOnce({ result: { value: { ok: true, was_focused: true } } })
.mockResolvedValueOnce({ result: { value: false } }),
});
const res = await handleBlur(
sm,
{ session_id: "aa11", ref: "@e3" },
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
);
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
expect(res).toMatchObject({
tab_id: 4,
used_ref: "e3",
was_focused: true,
focused: false,
});
expect(fake.sent.map((call) => call.method)).toEqual([
"DOM.resolveNode",
"Runtime.callFunctionOn",
"Runtime.callFunctionOn",
"Runtime.releaseObject",
]);
});
it("rejects a target that does not implement blur", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
ctx.refStore.set("e3", 1234, { tabId: 4 });
const fake = makeFakeCdp({
"DOM.resolveNode": () => ({ object: { objectId: "focus-target" } }),
"Runtime.callFunctionOn": () => ({
result: { value: { ok: false, was_focused: false, focused: false } },
}),
});
const res = await handleBlur(
sm,
{ session_id: "aa11", ref: "@e3" },
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
);
expect(res).toMatchObject({ code: "invalid_params", message: /does not support blur/ });
});
});
function successfulFillScript(params: unknown) {
const script = params as { arguments?: Array<{ value: unknown }>; functionDeclaration: string };
const args = script.arguments ?? [];
@@ -93,7 +93,11 @@ function makeFakeCdp(handlers: Record<string, (params?: object) => unknown>) {
sent.push({ method, params });
const handler = handlers[method];
if (!handler && method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 },
};
}
if (!handler) throw new Error(`unexpected CDP call ${method}`);
return handler(params);
@@ -681,6 +685,59 @@ describe("buildFrameVomScene", () => {
expect(nestedInput?.parentId).toBe(result.nodes.find((item) => item.backendNodeId === 210)?.id);
});
it("excludes an overlay only in its owning document when target backend ids collide", () => {
const owner = (id: number): CapturedNode => ({
backendNodeId: id,
parentBackendNodeId: null,
frameId: "main",
tag: "iframe",
attrs: {},
rect: { x: id * 10, y: 0, w: 200, h: 100 },
paintOrder: 0,
position: "static",
pointerEvents: "auto",
});
const owners = [owner(10), owner(20)];
const result = buildFrameVomScene(
[
{
frameId: "main",
contextScopeId: "main",
target: { tabId: 7 },
domNodes: owners,
axNodes: [],
},
...["left", "right"].map((frameId, i) => ({
frameId,
contextScopeId: frameId,
parentFrameId: "main",
ownerBackendNodeId: owners[i].backendNodeId,
target: { tabId: 7, sessionId: frameId },
excludedBackendNodeIds: new Set(i === 0 ? [7] : []),
domNodes: [],
axNodes: [
{
nodeId: "button",
frameId,
backendDOMNodeId: 7,
role: { type: "role", value: "button" },
name: { type: "computedString", value: frameId + " action" },
},
],
})),
],
{
nodes: owners,
viewport: { width: 800, height: 600 },
excludedBackendNodeIds: new Set(),
},
);
const rendered = renderVom(result);
expect(rendered.text).not.toContain("left action");
expect(rendered.text).toContain("right action");
expect(rendered.refs.find((ref) => ref.backendNodeId === 7)?.frameId).toBe("right");
});
it("does not place a child document at the page root when its frame boundary is unresolved", () => {
const childNode: CapturedNode = {
backendNodeId: 101,
@@ -2530,7 +2587,11 @@ describe("handleSnapshot", () => {
if (method === "Accessibility.enable") return {};
if (method === "Accessibility.getFullAXTree") return { nodes };
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 } };
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
};
}
if (method === "DOMSnapshot.enable") return {};
if (method === "DOMSnapshot.captureSnapshot") throw new Error("snapshot unsupported");
@@ -2617,7 +2678,11 @@ describe("handleSnapshot", () => {
if (method === "Accessibility.enable") return {};
if (method === "Accessibility.getFullAXTree") return { nodes: [root, button] };
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 } };
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
};
}
if (method === "DOMSnapshot.enable") return {};
if (method === "DOMSnapshot.captureSnapshot") {
@@ -2625,6 +2690,8 @@ describe("handleSnapshot", () => {
strings,
documents: [
{
scrollOffsetX: 0,
scrollOffsetY: 0,
frameId: "root",
nodes: {
parentIndex: [-1, 0],
@@ -2709,7 +2776,11 @@ describe("handleSnapshot", () => {
if (method === "Accessibility.enable") return {};
if (method === "Accessibility.getFullAXTree") return { nodes: [root, button] };
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 } };
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
};
}
if (method === "DOMSnapshot.enable") return {};
if (method === "DOMSnapshot.captureSnapshot") {
@@ -2717,6 +2788,8 @@ describe("handleSnapshot", () => {
strings,
documents: [
{
scrollOffsetX: 0,
scrollOffsetY: 0,
frameId: "root",
nodes: {
parentIndex: [-1, 0],
@@ -2804,7 +2877,11 @@ describe("handleSnapshot", () => {
if (method === "Accessibility.enable") return {};
if (method === "Accessibility.getFullAXTree") return { nodes: [root, button] };
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 } };
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
};
}
if (method === "DOMSnapshot.enable") return {};
if (method === "DOMSnapshot.captureSnapshot") {
@@ -2812,6 +2889,8 @@ describe("handleSnapshot", () => {
strings,
documents: [
{
scrollOffsetX: 0,
scrollOffsetY: 0,
frameId: "root",
nodes: {
parentIndex: [-1, 0],
@@ -2970,7 +3049,11 @@ describe("handleSnapshot", () => {
};
}
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } };
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 },
};
}
if (method === "DOMSnapshot.enable") return {};
if (method === "DOMSnapshot.captureSnapshot") {
@@ -3077,6 +3160,8 @@ describe("handleSnapshot", () => {
strings: S,
documents: [
{
scrollOffsetX: 0,
scrollOffsetY: 0,
frameId: "root",
nodes: {
parentIndex: [-1, 0, 1, 2],
@@ -3135,6 +3220,8 @@ describe("handleSnapshot", () => {
strings: S,
documents: [
{
scrollOffsetX: 0,
scrollOffsetY: 0,
frameId: "root",
nodes: {
parentIndex: [-1, 0, 1, 1, 1, 1],
@@ -3185,6 +3272,8 @@ describe("handleSnapshot", () => {
}
const VP_METRICS = {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
};
@@ -3198,6 +3287,8 @@ describe("handleSnapshot", () => {
strings,
documents: [
{
scrollOffsetX: 0,
scrollOffsetY: 0,
frameId: "main",
nodes: {
parentIndex: [-1, 0, 1],
@@ -3218,6 +3309,8 @@ describe("handleSnapshot", () => {
},
},
{
scrollOffsetX: 0,
scrollOffsetY: 0,
frameId: "child",
nodes: {
parentIndex: [-1, 0, 1],
@@ -3287,8 +3380,13 @@ describe("handleSnapshot", () => {
throw new Error(`unexpected root CDP method ${method}`);
});
const sendToTarget = vi.fn(async (_target, method: string) => {
if (method === "Runtime.evaluate") return { result: { value: { width: 400, height: 300 } } };
if (method === "Page.getLayoutMetrics" && ownerGeometry !== "unavailable")
return { cssLayoutViewport: { clientWidth: 400, clientHeight: 300 } };
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 400, clientHeight: 300 },
};
if (method === "Accessibility.enable" || method === "DOMSnapshot.enable") return {};
if (method === "DOMSnapshot.captureSnapshot")
return { ...snapshot, documents: [snapshot.documents[1]] };
@@ -3331,6 +3429,8 @@ describe("handleSnapshot", () => {
strings: ["body", "a", "href", "//outside.test/docs", "https://page.test/start"],
documents: [
{
scrollOffsetX: 0,
scrollOffsetY: 0,
frameId: "main",
...(hasSnapshotUrl ? { documentURL: 4 } : {}),
nodes: {
@@ -3452,6 +3552,8 @@ describe("handleSnapshot", () => {
strings,
documents: [
{
scrollOffsetX: 0,
scrollOffsetY: 0,
frameId: "root",
nodes: {
parentIndex: [-1, 0, 1, 1, 3],
@@ -3594,20 +3696,74 @@ describe("handleSnapshot", () => {
const result = await captureVomObservation(makeFrameAwareDeps().cdp, 4, "https://example.com", {
maxTokens,
});
expect(result.text.match(/@warning/g)).toHaveLength(1);
expect(result.text).toContain("iframe geometry incomplete");
expect(result.text.match(/@warning/g)).toHaveLength(2);
expect(result.text).toContain("document identity unverified");
expect(result.text).toContain("geometry incomplete");
expect(result.text.startsWith("@vom 1\n")).toBe(true);
if (maxTokens === undefined) {
expect(result.truncated).toBe(false);
expect(result.text).toContain("Frame action");
expect(result.matchNodes.find((node) => node.backendNodeId === 22)).toMatchObject({
rect: null,
localRect: { x: 20, y: 30, w: 120, h: 40 },
localRect: null,
});
}
if (maxTokens === 100) expect(Math.ceil(result.text.length / 4)).toBeLessThanOrEqual(maxTokens);
});
it.each([
["DOMSnapshot.captureSnapshot", "dom"],
["Accessibility.getFullAXTree", "ax"],
])("reports partial capture when child %s fails", async (failedMethod, stage) => {
const { cdp } = makeFrameAwareDeps("available");
const send = cdp.sendToTarget;
cdp.sendToTarget = async (target, method, params) => {
if (method === failedMethod) throw new Error("unavailable child data");
return send(target, method, params);
};
for (const maxTokens of [undefined, 100, 0]) {
const result = await captureVomObservation(cdp, 4, "https://example.com", { maxTokens });
expect(result.text).toContain(`@warning observation incomplete: some ${stage} data`);
expect(result.text.match(/@warning/g)).toHaveLength(2);
expect(result.text).toContain("document identity unverified");
expect(result.text.startsWith("@vom 1\n")).toBe(true);
if (maxTokens === undefined) {
expect(result.frames.map((frame) => frame.frameId)).toEqual(["main", "child"]);
expect(result.truncated).toBe(false);
}
if (maxTokens === 100)
expect(Math.ceil(result.text.length / 4)).toBeLessThanOrEqual(maxTokens);
}
});
it("reports omitted child ownership without exposing capture internals", async () => {
const { cdp } = makeFrameAwareDeps("available");
const send = cdp.send;
cdp.send = async (tabId, method, params) => {
const result = await send(tabId, method, params);
if (method === "DOMSnapshot.captureSnapshot") {
const child = await cdp.sendToTarget({ tabId, sessionId: "child-session" }, method, params);
(result as { documents: unknown[] }).documents.push(
...(child as { documents: unknown[] }).documents,
);
}
return result as never;
};
const result = await captureVomObservation(cdp, 4, "https://example.com");
expect(result.frames.map((frame) => frame.frameId)).toEqual(["main"]);
expect(result.text).toContain("observation incomplete: some ownership data");
expect(result.text).not.toContain("Frame action");
expect(result).not.toHaveProperty("issues");
});
it("reports unavailable form state while retaining the observed document", async () => {
const { cdp } = makeOverlayDeps([], loginSnapshotReply(), VP_METRICS);
const result = await captureVomObservation(cdp, 4, "https://example.com");
expect(result.text).toContain("observation incomplete: some forms data");
expect(result.frames).toHaveLength(1);
expect(result.text).not.toContain("unexpected CDP method");
});
it.each([
"available",
"clipped",
@@ -3617,7 +3773,8 @@ describe("handleSnapshot", () => {
4,
"https://example.com",
);
expect(result.text).not.toContain("@warning");
expect(result.text).not.toContain("iframe geometry incomplete");
expect(result.text).toContain("document identity unverified");
const node = result.matchNodes.find((item) => item.backendNodeId === 22);
if (ownerGeometry === "clipped") expect(node?.rect).toBeNull();
else expect(node?.rect).toEqual({ x: 120, y: 130, w: 120, h: 40 });
@@ -3639,7 +3796,7 @@ describe("handleSnapshot", () => {
expect(result.matchNodes.find((node) => node.backendNodeId === 22)).toMatchObject({
frameId: "child",
tag: "button",
localRect: { x: 20, y: 30, w: 120, h: 40 },
localRect: null,
});
expect(result.refs.find((ref) => ref.backendNodeId === 22)).toMatchObject({
frameId: "child",
@@ -3656,7 +3813,9 @@ describe("handleSnapshot", () => {
if ("code" in result) throw new Error(`unexpected error: ${JSON.stringify(result)}`);
expect(result.text).toContain('@e1 button "Frame action"');
const frameRef = [...ctx.refStore.entries()].find(([, entry]) => entry.backendNodeId === 22);
const frameRef = [...ctx.refStore.entries()].find(
([, entry]) => entry.kind === "dom" && entry.backendNodeId === 22,
);
expect(frameRef?.[1]).toMatchObject({
tabId: 4,
frameId: "child",
@@ -3983,6 +4142,8 @@ describe("handleSnapshot", () => {
strings: S,
documents: [
{
scrollOffsetX: 0,
scrollOffsetY: 0,
frameId: "root",
nodes: {
parentIndex: [-1, 0, 1, 0, 3, 4, 5],
@@ -0,0 +1,348 @@
// @vitest-environment node
// Opt in with BSK_SCROLL_CHROME=/path/to/chrome. Uses an isolated browser/profile.
import { describe, expect, it } from "vitest";
import type { CdpFrame, CdpFrameGraph, CdpTarget } from "@/browser-driver/frame-graph";
import { SessionManager } from "@/session-manager/manager";
import type { ViewportRect } from "../geometry";
import { handleScrollTo } from "../scroll";
import type { CdpRunner } from "../shared";
type Send = <T = Record<string, unknown>>(
method: string,
params?: object,
sessionId?: string,
) => Promise<T>;
type Tree = { frame: { id: string; parentId?: string; name?: string }; childFrames?: Tree[] };
async function withBrowser(
run: (h: Awaited<ReturnType<typeof harness>>) => Promise<void>,
zoom = 1,
) {
const { withChrome } = await import(
new URL(
"../../../../../evals/browser/cases/regression/snapshot-coordinates/chrome.mjs",
import.meta.url,
).href
);
await withChrome(
{ executable: process.env.BSK_SCROLL_CHROME, deviceScale: 1, zoom },
async (send: Send) => run(await harness(send)),
);
}
async function harness(send: Send) {
const { targetId } = await send<{ targetId: string }>("Target.createTarget", {
url: "about:blank",
});
const { sessionId: rootSession } = await send<{ sessionId: string }>("Target.attachToTarget", {
targetId,
flatten: true,
});
await send("Page.bringToFront", {}, rootSession);
const manager = new SessionManager({
agentWindow: {
create: async () => 100,
remove: async () => {},
ensureActiveTab: async () => 4,
},
});
const ctx = await manager.start("aa11");
const calls: { method: string; sessionId?: string }[] = [];
const after = { command: (_method: string) => {} };
const sessionFor = (target: CdpTarget) => target.sessionId ?? rootSession;
const forward = async <T>(target: CdpTarget, method: string, params?: object): Promise<T> => {
calls.push({ method, sessionId: target.sessionId });
const result = await send<T>(method, params, sessionFor(target));
after.command(method);
return result;
};
const cdp: CdpRunner = {
send: (_tab, method, params) => forward({ tabId: 4 }, method, params),
sendToTarget: forward,
};
const evaluate = async <T>(expression: string, frame?: CdpFrame): Promise<T> => {
const session = frame ? sessionFor(frame.target) : rootSession;
const world = frame
? await send<{ executionContextId: number }>(
"Page.createIsolatedWorld",
{
frameId: frame.frameId,
worldName: "scroll-oracle",
},
session,
)
: undefined;
const reply = await send<{ result: { value: T }; exceptionDetails?: unknown }>(
"Runtime.evaluate",
{
expression,
contextId: world?.executionContextId,
returnByValue: true,
awaitPromise: true,
},
session,
);
expect(reply.exceptionDetails).toBeUndefined();
return reply.result.value;
};
const rememberRef = (frame: CdpFrame, backendNodeId: number) =>
ctx.refStore.set("e1", backendNodeId, {
tabId: 4,
frameId: frame.frameId,
cdpSessionId: frame.target.sessionId,
});
const ref = async (frame: CdpFrame, selector = "#probe") => {
const { executionContextId } = await send<{ executionContextId: number }>(
"Page.createIsolatedWorld",
{
frameId: frame.frameId,
worldName: "scroll-oracle",
},
sessionFor(frame.target),
);
const { result } = await send<{ result: { objectId: string } }>(
"Runtime.evaluate",
{
expression: `document.querySelector(${JSON.stringify(selector)})`,
contextId: executionContextId,
},
sessionFor(frame.target),
);
const { node } = await send<{ node: { backendNodeId: number } }>(
"DOM.describeNode",
{ objectId: result.objectId },
sessionFor(frame.target),
);
await send("Runtime.releaseObject", { objectId: result.objectId }, sessionFor(frame.target));
rememberRef(frame, node.backendNodeId);
};
const scroll = (target: { selector: string } | { ref: string }, signal?: AbortSignal) =>
handleScrollTo(
manager,
{ session_id: "aa11", ...target },
{
cdp,
signal,
tabsApi: {
get: async (id) => ({ id, windowId: 100, active: true }) as chrome.tabs.Tab,
query: async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab],
},
},
);
const loadFrames = async () => {
const { targetInfos } = await send<{ targetInfos: { targetId: string; type: string }[] }>(
"Target.getTargets",
);
const sessions = [rootSession];
for (const target of targetInfos.filter((t) => t.type === "iframe")) {
const { sessionId } = await send<{ sessionId: string }>("Target.attachToTarget", {
targetId: target.targetId,
flatten: true,
});
sessions.push(sessionId);
}
const frames: (CdpFrame & { name: string })[] = [];
for (const session of sessions) {
const { frameTree } = await send<{ frameTree: Tree }>("Page.getFrameTree", {}, session);
const target = { tabId: 4, ...(session !== rootSession ? { sessionId: session } : {}) };
const visit = (tree: Tree, parentFrameId = tree.frame.parentId) => {
frames.push({ frameId: tree.frame.id, parentFrameId, target, name: tree.frame.name ?? "" });
for (const child of tree.childFrames ?? []) visit(child, tree.frame.id);
};
visit(frameTree);
}
for (const frame of frames) {
if (!frame.parentFrameId) continue;
const parent = frames.find((f) => f.frameId === frame.parentFrameId)!;
const owner = await send<{ backendNodeId: number }>(
"DOM.getFrameOwner",
{ frameId: frame.frameId },
sessionFor(parent.target),
);
frame.ownerBackendNodeId = owner.backendNodeId;
}
const graph: CdpFrameGraph = { rootFrameId: frames[0].frameId, frames };
cdp.getFrameGraph = async () => graph;
return frames;
};
return { send, rootSession, evaluate, ref, rememberRef, scroll, loadFrames, calls, after };
}
function intersection(a: ViewportRect, b: ViewportRect): ViewportRect | null {
const x = Math.max(a.x, b.x),
y = Math.max(a.y, b.y);
const right = Math.min(a.x + a.width, b.x + b.width),
bottom = Math.min(a.y + a.height, b.y + b.height);
return right > x && bottom > y ? { x, y, width: right - x, height: bottom - y } : null;
}
function expectRect(actual: unknown, expected: ViewportRect) {
expect(actual).not.toHaveProperty("code");
for (const key of ["x", "y", "width", "height"] as const)
expect(Math.abs((actual as ViewportRect)[key] - expected[key]), key).toBeLessThan(2);
}
describe.skipIf(!process.env.BSK_SCROLL_CHROME)("real browser scroll-to", () => {
it("scrolls ordinary elements and reports the clipped portion of nested scrollers", async () => {
await withBrowser(async (h) => {
for (const html of [
'<div style="height:2200px"></div><div id="target" style="width:100px;height:40px"></div>',
'<div style="height:200px"></div><div id="clip" style="height:80px;width:200px;overflow:auto"><div id="target" style="width:100px;height:400px"></div></div>',
'<div style="height:200px"></div><div id="clip" style="height:80px;width:200px;overflow:auto;transform:scale(1.25);transform-origin:0 0"><div id="target" style="width:100px;height:400px"></div></div>',
]) {
await h.evaluate(`document.body.innerHTML = ${JSON.stringify(html)}`);
const result = await h.scroll({ selector: "#target" });
const oracle = await h.evaluate<{ rect: ViewportRect; clip?: ViewportRect }>(`({
rect: document.querySelector('#target').getBoundingClientRect().toJSON(),
clip: document.querySelector('#clip')?.getBoundingClientRect().toJSON()
})`);
expectRect(result, oracle.clip ? intersection(oracle.rect, oracle.clip)! : oracle.rect);
}
});
});
it.each([
"visibility:hidden",
"opacity:0",
"display:none",
"content-visibility:hidden",
])("rejects %s targets", async (style) => {
await withBrowser(async (h) => {
await h.evaluate(
`document.body.innerHTML = '<div style="${style}"><button id="target">Target</button></div>'`,
);
expect(await h.scroll({ selector: "#target" })).toMatchObject({
code: "permission_denied",
data: { reason: "element_not_visible" },
});
});
});
it("rejects targets fully outside an overflow:clip ancestor", async () => {
await withBrowser(async (h) => {
await h.evaluate(
`document.body.innerHTML = '<div style="height:80px;overflow:clip"><div style="height:150px"></div><button id="target">Target</button></div>'`,
);
expect(await h.scroll({ selector: "#target" })).toMatchObject({ code: "permission_denied" });
});
});
it.each(["open", "closed"])("clips %s shadow-root refs", async (mode) => {
await withBrowser(async (h) => {
await h.evaluate(`(() => {
const host = document.createElement('div'); document.body.append(host);
const root = host.attachShadow({mode:'${mode}'});
root.innerHTML = '<div style="height:60px;overflow:auto"><button id="probe" style="height:200px;width:100px">Target</button></div>';
window.probe = root.querySelector('#probe');
})()`);
const frames = await h.loadFrames();
const { result } = await h.send<{ result: { objectId: string } }>(
"Runtime.evaluate",
{ expression: "window.probe" },
h.rootSession,
);
// Build the ref from the actual closed-root node, as an observation does.
const { node } = await h.send<{ node: { backendNodeId: number } }>(
"DOM.describeNode",
{ objectId: result.objectId },
h.rootSession,
);
await h.send("Runtime.releaseObject", { objectId: result.objectId }, h.rootSession);
h.rememberRef(frames[0], node.backendNodeId);
expect(await h.scroll({ ref: "e1" })).toMatchObject({ height: 60 });
});
});
it("cancels after scrolling and releases the measurement object without follow-up reads", async () => {
await withBrowser(async (h) => {
await h.evaluate("document.body.innerHTML = '<button id=target>Target</button>'");
const abort = new AbortController();
h.after.command = (method) => {
if (method === "DOM.resolveNode") abort.abort();
};
expect(await h.scroll({ selector: "#target" }, abort.signal)).toMatchObject({
code: "cancelled",
});
expect(h.calls.at(-1)!.method).toBe("Runtime.releaseObjectGroup");
expect(h.calls.some((call) => call.method === "Runtime.callFunctionOn")).toBe(false);
});
});
it.each([
{ fixture: "snapshot-coordinates", zoom: 1 },
{ fixture: "snapshot-coordinates", zoom: 1.25 },
{ fixture: "oopif-scrollbars", zoom: 1 },
{ fixture: "oopif-scrollbars", zoom: 0.8 },
])("scrolls nested frames: $fixture at zoom $zoom", async ({ fixture, zoom }) => {
const { createEvalServer } = await import(
new URL("../../../../../evals/browser/lib/server.mjs", import.meta.url).href
);
const server = createEvalServer();
const { baseUrl } = await server.start();
try {
await withBrowser(async (h) => {
await h.send("Page.navigate", { url: `${baseUrl}/${fixture}?run=scroll` }, h.rootSession);
await expect
.poll(
() =>
server
.snapshot("scroll")
.events.some(
(e: { type: string; data: { root?: boolean } }) =>
e.type === "geometry.ready" && e.data.root,
),
{ timeout: 10_000 },
)
.toBe(true);
const frames = await h.loadFrames();
expect(frames.length).toBe(fixture === "snapshot-coordinates" ? 5 : 3);
for (const frame of frames) {
await h.ref(frame);
const result = await h.scroll({ ref: "e1" });
// Independent DOM oracle: CSS border boxes plus this fixture's known
// axis-aligned iframe scales. Re-read after each scroll changes layout.
let rect = await h.evaluate<ViewportRect>(
"document.querySelector('#probe').getBoundingClientRect().toJSON()",
frame,
);
let current = frame;
while (true) {
const viewport = await h.evaluate<ViewportRect>(
"({ x:0, y:0, width:document.documentElement.clientWidth, height:document.documentElement.clientHeight })",
current,
);
rect = intersection(rect, viewport)!;
if (!current.parentFrameId) break;
const parent = frames.find((f) => f.frameId === current.parentFrameId)!;
const owner = await h.evaluate<{ x: number; y: number; scale: number }>(
`(() => {
const iframe = document.querySelector('iframe[name="${current.name}"]');
const r = iframe.getBoundingClientRect(); const s = getComputedStyle(iframe);
const scale = r.width / iframe.offsetWidth;
return { x:r.x + (iframe.clientLeft + parseFloat(s.paddingLeft))*scale,
y:r.y + (iframe.clientTop + parseFloat(s.paddingTop))*scale, scale };
})()`,
parent,
);
rect = {
x: owner.x + rect.x * owner.scale,
y: owner.y + rect.y * owner.scale,
width: rect.width * owner.scale,
height: rect.height * owner.scale,
};
current = parent;
}
expectRect(result, rect);
}
const child = frames.find((frame) => frame.parentFrameId === frames[0].frameId)!;
await h.ref(child);
await h.evaluate(
`document.querySelector('iframe[name="${child.name}"]').style.opacity = '0'`,
);
expect(await h.scroll({ ref: "e1" })).toMatchObject({ code: "permission_denied" });
}, zoom);
} finally {
await server.stop();
}
}, 30_000);
});
@@ -0,0 +1,310 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import type { CdpFrameGraph, CdpTarget } from "@/browser-driver/frame-graph";
import { SessionManager } from "@/session-manager/manager";
import type { JavaScriptDialogInfo, ScrollToParams } from "@/transport/types";
import type { ViewportRect } from "../geometry";
import { handleScrollTo } from "../scroll";
import type { CdpRunner } from "../shared";
interface Call {
target: CdpTarget;
method: string;
params?: Record<string, unknown>;
aborted: boolean;
}
async function fixture(frame: "top" | "same-target" | "oopif" = "top") {
const manager = new SessionManager({
agentWindow: {
create: async () => 100,
remove: async () => {},
ensureActiveTab: async () => 4,
},
});
const ctx = await manager.start("aa11");
ctx.refStore.set("e3", 1234, {
tabId: 4,
...(frame !== "top" ? { frameId: "child" } : {}),
...(frame === "oopif" ? { cdpSessionId: "child-session" } : {}),
});
const abort = new AbortController();
const graph: CdpFrameGraph = {
rootFrameId: "main",
frames: [
{ frameId: "main", target: { tabId: 4 } },
{
frameId: "child",
parentFrameId: "main",
ownerBackendNodeId: 99,
target: { tabId: 4, ...(frame === "oopif" ? { sessionId: "child-session" } : {}) },
},
],
};
const rectangles = new Map<number, ViewportRect | null>([
[1234, { x: 10, y: 20, width: 100, height: 40 }],
[99, { x: 204, y: 306, width: 400, height: 200 }],
]);
const calls: Call[] = [];
const hooks = {
after: (_call: Call) => {},
reply: (_call: Call): object | undefined => undefined,
};
const send = async <T>(target: CdpTarget, method: string, params?: object): Promise<T> => {
const call = {
target,
method,
params: params as Call["params"],
aborted: abort.signal.aborted,
};
calls.push(call);
const override = hooks.reply(call);
hooks.after(call);
if (override) return override as T;
const args = call.params ?? {};
const replies: Record<string, unknown> = {
"DOM.getDocument": { root: { nodeId: 1 } },
"DOM.querySelector": { nodeId: 2 },
"DOM.describeNode": { node: { backendNodeId: 1234 } },
"DOM.scrollIntoViewIfNeeded": {},
"DOM.resolveNode": { object: { objectId: String(args.backendNodeId) } },
"Runtime.callFunctionOn": {
result: {
value: String(args.functionDeclaration).includes("IntersectionObserver")
? rectangles.get(Number(args.objectId))
: { width: 200, height: 100 },
},
},
"Runtime.releaseObject": {},
"Runtime.releaseObjectGroup": {},
"Runtime.evaluate": { result: { value: { width: 200, height: 100 } } },
"Page.getLayoutMetrics": {
cssLayoutViewport: {
clientWidth: target.sessionId ? 185 : 1280,
clientHeight: target.sessionId ? 89 : 720,
},
},
"DOM.getBoxModel": { model: { content: [204, 306, 604, 306, 604, 506, 204, 506] } },
};
if (!(method in replies)) throw new Error(`unexpected CDP call ${method}`);
return replies[method] as T;
};
const cdp: CdpRunner = {
send: (tabId, method, params) => send({ tabId }, method, params),
sendToTarget: send,
getFrameGraph: vi.fn(async () => {
hooks.after({ target: { tabId: 4 }, method: "getFrameGraph", aborted: abort.signal.aborted });
return graph;
}),
trackSessionTab: vi.fn(),
};
const tabsApi = {
get: vi.fn(async (id: number) => ({ id, windowId: 100, active: true }) as chrome.tabs.Tab),
query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
};
const run = (params: Partial<ScrollToParams> = {}) =>
handleScrollTo(
manager,
{ session_id: "aa11", ref: "@e3", ...params },
{ cdp, tabsApi, signal: abort.signal },
);
return { ctx, cdp, graph, rectangles, calls, hooks, abort, tabsApi, run };
}
afterEach(() => vi.restoreAllMocks());
describe("handleScrollTo", () => {
it("scrolls a ref and returns renderer-clipped bounds", async () => {
const f = await fixture();
expect(await f.run()).toMatchObject({
tab_id: 4,
used_ref: "e3",
x: 10,
y: 20,
width: 100,
height: 40,
});
expect(f.calls[0]).toMatchObject({
method: "DOM.scrollIntoViewIfNeeded",
params: { backendNodeId: 1234 },
});
const group = f.calls.find((call) => call.method === "DOM.resolveNode")!.params!.objectGroup;
expect(f.calls.at(-1)).toMatchObject({
method: "Runtime.releaseObjectGroup",
params: { objectGroup: group },
});
});
it("resolves a selector and tracks the attached tab even when it is missing", async () => {
const f = await fixture();
expect(await f.run({ ref: undefined, selector: "#target" })).toMatchObject({
used_selector: "#target",
});
f.hooks.reply = ({ method }) => (method === "DOM.querySelector" ? { nodeId: 0 } : undefined);
expect(await f.run({ ref: undefined, selector: "#missing" })).toMatchObject({
code: "not_found",
data: { reason: "selector_not_found" },
});
expect(f.cdp.trackSessionTab).toHaveBeenCalledWith("aa11", 4);
});
it.each([
"same-target",
"oopif",
] as const)("projects %s refs and scrolls their owners", async (frame) => {
const f = await fixture(frame);
expect(await f.run()).toMatchObject({ x: 224, y: 346, width: 200, height: 80 });
expect(
f.calls
.filter((call) => call.method === "DOM.scrollIntoViewIfNeeded")
.map((call) => call.params!.backendNodeId),
).toEqual([99, 1234]);
expect(f.cdp.getFrameGraph).toHaveBeenCalledTimes(1);
if (frame === "oopif")
expect(f.calls.find((call) => call.params?.backendNodeId === 1234)!.target.sessionId).toBe(
"child-session",
);
});
it("clips OOPIF bounds to occupied scrollbars without changing the frame scale", async () => {
const f = await fixture("oopif");
f.rectangles.set(1234, { x: 180, y: 80, width: 5, height: 9 });
expect(await f.run()).toMatchObject({ x: 564, y: 466, width: 10, height: 18 });
});
it("clips against the iframe owner's ordinary DOM ancestors", async () => {
const f = await fixture("oopif");
f.rectangles.set(99, { x: 204, y: 306, width: 400, height: 80 });
expect(await f.run()).toMatchObject({ x: 224, y: 346, width: 200, height: 40 });
});
it.each([1234, 99])("rejects a hidden or fully clipped node %i", async (node) => {
const f = await fixture("oopif");
f.rectangles.set(node, null);
expect(await f.run()).toMatchObject({
code: "permission_denied",
data: { reason: "element_not_visible" },
});
expect(f.calls.at(-1)!.method).toBe("Runtime.releaseObjectGroup");
});
it.each([
{ ref: undefined },
{ selector: "#target" },
{ timeout_ms: 0 },
])("rejects invalid params %j", async (params) => {
const f = await fixture();
expect(await f.run(params)).toMatchObject({ code: "invalid_params" });
expect(f.calls).toEqual([]);
});
it.each(["stale", "other-tab"])("rejects %s refs before scrolling", async (mode) => {
const f = await fixture();
if (mode === "stale") f.ctx.refStore.clear();
else f.ctx.refStore.set("e3", 1234, { tabId: 8 });
expect(await f.run()).toMatchObject({ code: "not_found", data: { reason: "ref_not_found" } });
expect(f.calls).toEqual([]);
});
it("enforces the Agent Window boundary", async () => {
const f = await fixture();
f.tabsApi.query.mockResolvedValue([{ id: 4, windowId: 200, active: true } as chrome.tabs.Tab]);
f.tabsApi.get.mockResolvedValue({ id: 4, windowId: 200, active: true } as chrome.tabs.Tab);
expect(await f.run({ tab_id: 4 })).toMatchObject({ code: "permission_denied" });
expect(f.calls).toEqual([]);
});
it.each([
"getFrameGraph",
"DOM.scrollIntoViewIfNeeded",
"DOM.resolveNode",
"Runtime.callFunctionOn",
"Page.getLayoutMetrics",
"Runtime.evaluate",
"DOM.getBoxModel",
])("honors cancellation during %s and only cleans up afterwards", async (method) => {
const f = await fixture("oopif");
f.hooks.after = (call) => {
if (call.method === method) f.abort.abort();
};
expect(await f.run()).toMatchObject({ code: "cancelled" });
expect(
f.calls.filter((call) => call.aborted && call.method !== "Runtime.releaseObjectGroup"),
).toEqual([]);
});
it.each([
"DOM.getDocument",
"DOM.querySelector",
"DOM.describeNode",
])("cancels selector resolution during %s", async (method) => {
const f = await fixture();
f.hooks.after = (call) => {
if (call.method === method) f.abort.abort();
};
expect(await f.run({ ref: undefined, selector: "#target" })).toMatchObject({
code: "cancelled",
});
expect(f.calls.filter((call) => call.aborted)).toEqual([]);
});
it("does not start after early cancellation", async () => {
const f = await fixture();
f.abort.abort();
expect(await f.run()).toMatchObject({ code: "cancelled" });
expect(f.calls).toEqual([]);
expect(f.tabsApi.query).not.toHaveBeenCalled();
});
it("does not issue fallback scrolling after cancellation", async () => {
const f = await fixture();
f.hooks.reply = ({ method }) => {
if (method === "DOM.scrollIntoViewIfNeeded") {
f.abort.abort();
throw new Error("scroll failed");
}
return undefined;
};
expect(await f.run()).toMatchObject({ code: "cancelled" });
expect(f.calls.map((call) => call.method)).toEqual(["DOM.scrollIntoViewIfNeeded"]);
});
it("stops target scrolling after a deadline expires while scrolling its owner", async () => {
const f = await fixture("oopif");
const now = vi.spyOn(Date, "now").mockReturnValue(100);
f.hooks.after = ({ method }) => {
if (method === "DOM.scrollIntoViewIfNeeded") now.mockReturnValue(201);
};
expect(await f.run({ timeout_ms: 100 })).toMatchObject({ code: "timeout" });
expect(f.calls.filter((call) => call.method === "DOM.scrollIntoViewIfNeeded")).toHaveLength(1);
});
it("reports measurement exceptions and releases objects even if cleanup fails", async () => {
const f = await fixture();
f.hooks.reply = ({ method }) => {
if (method === "Runtime.callFunctionOn")
return { exceptionDetails: { text: "measurement failed" } };
if (method === "Runtime.releaseObjectGroup") throw new Error("target closed");
return undefined;
};
expect(await f.run()).toMatchObject({ code: "cdp_failed", message: "measurement failed" });
expect(f.calls.at(-1)!.method).toBe("Runtime.releaseObjectGroup");
});
it("attaches dialogs observed during scrolling", async () => {
const f = await fixture();
f.cdp.dialogCursor = () => 5;
const dialogs: JavaScriptDialogInfo[] = [
{
sequence: 6,
tab_id: 4,
type: "alert",
message: "scrolled",
handled: "accepted",
url: "https://example.test",
},
];
f.cdp.dialogsSince = vi.fn(() => dialogs);
expect(await f.run()).toHaveProperty("dialogs", dialogs);
expect(f.cdp.dialogsSince).toHaveBeenCalledWith(4, 5);
});
});
@@ -0,0 +1,355 @@
// @vitest-environment node
import { describe, expect, it } from "vitest";
import type { CdpFrame, CdpFrameGraph, CdpTarget } from "@/browser-driver/frame-graph";
import { resolveNodeGeometry } from "../frame-geometry";
import type { CdpRunner } from "../shared";
import { captureObservationFacts } from "../vom/capture-coordinator";
type Send = <T = Record<string, unknown>>(
method: string,
params?: object,
sessionId?: string,
) => Promise<T>;
type Tree = { frame: { id: string; parentId?: string; name?: string }; childFrames?: Tree[] };
type Rect = { x: number; y: number; w: number; h: number };
type Oracle = {
probes: Record<string, Rect>;
viewport: { width: number; height: number };
owners: Record<string, { x: number; y: number; scale: number }>;
};
function clip(rect: Rect, viewport: Oracle["viewport"]): Rect | null {
const x = Math.max(0, rect.x),
y = Math.max(0, rect.y);
const right = Math.min(viewport.width, rect.x + rect.w);
const bottom = Math.min(viewport.height, rect.y + rect.h);
return right > x && bottom > y ? { x, y, w: right - x, h: bottom - y } : null;
}
// The independent oracle uses DOM border boxes and this fixture's axis-aligned
// iframe transforms. No production snapshot conversion/projection builds expectations.
const oracleExpression = `(() => {
const probes = {};
for (const node of document.querySelectorAll('[data-geometry-probe]')) {
const box = node.getBoundingClientRect();
probes[node.id] = { x: box.x, y: box.y, w: box.width, h: box.height };
}
const owners = {};
for (const frame of document.querySelectorAll('iframe')) {
const rect = frame.getBoundingClientRect();
const style = getComputedStyle(frame);
const scale = rect.width / frame.offsetWidth;
owners[frame.name] = {
x: rect.x + (frame.clientLeft + parseFloat(style.paddingLeft)) * scale,
y: rect.y + (frame.clientTop + parseFloat(style.paddingTop)) * scale,
scale,
};
}
return { probes, viewport: { width: document.documentElement.clientWidth, height: document.documentElement.clientHeight }, owners };
})()`;
describe.skipIf(!process.env.BSK_GEOMETRY_CHROME)("real DOMSnapshot coordinate contract", () => {
it.each(
[
{ deviceScale: 1, zoom: 1 },
{ deviceScale: 0.8, zoom: 1 },
{ deviceScale: 1, zoom: 1.25 },
{ deviceScale: 2, zoom: 1 },
{ deviceScale: 2, zoom: 0.8 },
]
.flatMap((configuration) => [
{ ...configuration, fixture: "snapshot-coordinates", scrollbars: "none" },
{ ...configuration, fixture: "oopif-scrollbars", scrollbars: "both" },
])
.concat(
["vertical", "horizontal", "none"].map((scrollbars) => ({
deviceScale: 1,
zoom: 1,
fixture: "oopif-scrollbars",
scrollbars,
})),
),
)("$fixture: device scale $deviceScale, zoom $zoom, scrollbars $scrollbars", async (configuration) => {
const evalRoot = new URL("../../../../../evals/browser/", import.meta.url);
const { createEvalServer } = await import(new URL("lib/server.mjs", evalRoot).href);
const { withChrome } = await import(
new URL("cases/regression/snapshot-coordinates/chrome.mjs", evalRoot).href
);
const server = createEvalServer();
const { baseUrl } = await server.start();
try {
await withChrome(
{ executable: process.env.BSK_GEOMETRY_CHROME, ...configuration },
async (send: Send) => {
const { targetId } = await send<{ targetId: string }>("Target.createTarget", {
url: "about:blank",
});
const { sessionId: rootSession } = await send<{ sessionId: string }>(
"Target.attachToTarget",
{ targetId, flatten: true },
);
await send(
"Page.navigate",
{
url: `${baseUrl}/${configuration.fixture}?run=coordinates&scrollbars=${configuration.scrollbars}`,
},
rootSession,
);
await expect
.poll(
() =>
server
.snapshot("coordinates")
.events.some(
(event: { type: string; data: { root?: boolean } }) =>
event.type === "geometry.ready" && event.data.root,
),
{ timeout: 10_000 },
)
.toBe(true);
const targets = await send<{ targetInfos: { targetId: string; type: string }[] }>(
"Target.getTargets",
);
const sessions = [rootSession];
for (const target of targets.targetInfos.filter((target) => target.type === "iframe")) {
const { sessionId } = await send<{ sessionId: string }>("Target.attachToTarget", {
targetId: target.targetId,
flatten: true,
});
sessions.push(sessionId);
}
expect(sessions.length).toBe(configuration.fixture === "oopif-scrollbars" ? 3 : 2);
const frames: CdpFrame[] = [];
const names = new Map<string, string>();
const sessionFor = (target: CdpTarget) => target.sessionId ?? rootSession;
for (const sessionId of sessions) {
const { frameTree } = await send<{ frameTree: Tree }>(
"Page.getFrameTree",
{},
sessionId,
);
const target: CdpTarget = {
tabId: 1,
...(sessionId === rootSession ? {} : { sessionId }),
};
const visit = (tree: Tree, parentFrameId = tree.frame.parentId) => {
frames.push({ frameId: tree.frame.id, parentFrameId, target });
names.set(tree.frame.id, tree.frame.name ?? "");
for (const child of tree.childFrames ?? []) visit(child, tree.frame.id);
};
visit(frameTree);
}
expect(frames).toHaveLength(configuration.fixture === "oopif-scrollbars" ? 3 : 5);
for (const frame of frames) {
if (!frame.parentFrameId) continue;
const parent = frames.find((item) => item.frameId === frame.parentFrameId)!;
const owner = await send<{ backendNodeId: number }>(
"DOM.getFrameOwner",
{ frameId: frame.frameId },
sessionFor(parent.target),
);
frame.ownerBackendNodeId = owner.backendNodeId;
}
const graph: CdpFrameGraph = { rootFrameId: frames[0].frameId, frames };
const calls: string[] = [];
const cdp: CdpRunner = {
send: (tabId, method, params) => {
calls.push(`${tabId}:${method}`);
return send(method, params, rootSession);
},
sendToTarget: (target, method, params) => {
calls.push(`${target.sessionId ?? target.tabId}:${method}`);
if (
method === "Runtime.evaluate" &&
(params as { expression?: string })?.expression?.includes("window.innerWidth")
)
calls.push(`${target.sessionId ?? target.tabId}:viewport-size`);
return send(method, params, sessionFor(target));
},
getFrameGraph: async () => graph,
getAttachmentId: () => rootSession,
};
const oracles = new Map<string, Oracle>();
for (const frame of frames) {
const session = sessionFor(frame.target);
const { executionContextId } = await send<{ executionContextId: number }>(
"Page.createIsolatedWorld",
{ frameId: frame.frameId, worldName: "coordinate-oracle" },
session,
);
await expect
.poll(
async () => {
const ready = await send<{ result: { value: boolean } }>(
"Runtime.evaluate",
{
expression: 'document.documentElement.dataset.geometryReady === "true"',
contextId: executionContextId,
returnByValue: true,
},
session,
);
return ready.result.value;
},
{ timeout: 5_000 },
)
.toBe(true);
// Establish scroll after cross-process target creation/layout settles.
const scroll = frame.parentFrameId
? names.get(frame.frameId) === "nested"
? [10, 20]
: [40, 100]
: [80, 240];
await send(
"Runtime.evaluate",
{
expression: `(async () => { scrollTo(${scroll.join(",")}); await new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve))); })()`,
contextId: executionContextId,
awaitPromise: true,
},
session,
);
const reply = await send<{ result: { value: Oracle } }>(
"Runtime.evaluate",
{ expression: oracleExpression, contextId: executionContextId, returnByValue: true },
session,
);
oracles.set(frame.frameId, reply.result.value);
}
const childViewport = await send<{
result: {
value: { width: number; height: number; clientWidth: number; clientHeight: number };
};
}>(
"Runtime.evaluate",
{
expression:
"({ width: innerWidth, height: innerHeight, clientWidth: document.documentElement.clientWidth, clientHeight: document.documentElement.clientHeight })",
returnByValue: true,
},
sessions[1],
);
for (const dimension of ["Width", "Height"] as const) {
const occupied =
childViewport.result.value[dimension === "Width" ? "width" : "height"] -
childViewport.result.value[`client${dimension}`];
if (
configuration.scrollbars === "both" ||
configuration.scrollbars === (dimension === "Width" ? "vertical" : "horizontal")
)
expect(occupied).toBeGreaterThan(0);
else expect(occupied).toBe(0);
}
const facts = await captureObservationFacts(cdp, 1);
expect(facts.issues).toEqual([]);
expect(calls.filter((call) => call.endsWith(":Page.getLayoutMetrics"))).toHaveLength(
sessions.length,
);
// Identity verification also uses Runtime.evaluate; count the full viewport reads separately.
const viewportReadsBeforeLive = calls.filter((call) => call.endsWith(":viewport-size"));
expect(new Set(viewportReadsBeforeLive).size).toBe(viewportReadsBeforeLive.length);
expect(viewportReadsBeforeLive.length).toBeLessThanOrEqual(sessions.length - 1);
for (const frame of frames) {
for (const [id, local] of Object.entries(oracles.get(frame.frameId)!.probes)) {
const node = facts.documents
.find((doc) => doc.frame.frameId === frame.frameId)
?.domNodes.find((node) => node.attrs.id === id);
expect(node, `missing ${id} in ${names.get(frame.frameId) || "root"}`).toBeDefined();
let top = clip(local, oracles.get(frame.frameId)!.viewport);
let current = frame;
while (top && current.parentFrameId) {
const parentOracle = oracles.get(current.parentFrameId)!;
const owner = parentOracle.owners[names.get(current.frameId)!];
top = clip(
{
x: owner.x + top.x * owner.scale,
y: owner.y + top.y * owner.scale,
w: top.w * owner.scale,
h: top.h * owner.scale,
},
parentOracle.viewport,
);
current = frames.find((item) => item.frameId === current.parentFrameId)!;
}
for (const key of ["x", "y", "w", "h"] as const)
expect(
Math.abs(node!.localRect![key] - local[key]),
`${names.get(frame.frameId)} ${id} local ${key}`,
).toBeLessThan(2);
if (!top) {
expect(node!.rect, `${id} must be clipped out`).toBeNull();
if (configuration.fixture === "oopif-scrollbars")
expect(
await resolveNodeGeometry(cdp, 1, {
target: frame.target,
frameId: frame.frameId,
backendNodeId: node!.backendNodeId,
}),
).toMatchObject({ code: "permission_denied" });
continue;
}
expect(node!.rect).not.toBeNull();
for (const key of ["x", "y", "w", "h"] as const)
expect(
Math.abs(node!.rect![key] - top[key]),
`${names.get(frame.frameId)} ${id} top ${key}`,
).toBeLessThan(2);
if (configuration.fixture !== "oopif-scrollbars") continue;
const live = await resolveNodeGeometry(cdp, 1, {
target: frame.target,
frameId: frame.frameId,
backendNodeId: node!.backendNodeId,
});
if ("code" in live) throw new Error(live.message);
for (const [key, liveKey] of [
["x", "x"],
["y", "y"],
["w", "width"],
["h", "height"],
] as const)
expect(
Math.abs(live.topBounds[liveKey] - top[key]),
`${id} live ${key}`,
).toBeLessThan(2);
const before = server.snapshot("coordinates").events.length;
for (const type of ["mousePressed", "mouseReleased"])
await send(
"Input.dispatchMouseEvent",
{ type, ...live.actionPoint, button: "left", clickCount: 1 },
rootSession,
);
await expect
.poll(
() =>
server
.snapshot("coordinates")
.events.slice(before)
.some(
(event: { type: string; path: string; data: { probe?: string } }) =>
event.type === "geometry.clicked" &&
event.data.probe === id &&
event.path ===
(frame.parentFrameId
? names.get(frame.frameId) === "nested"
? "/oopif-scrollbars/nested"
: "/oopif-scrollbars/frame"
: "/oopif-scrollbars"),
),
{ timeout: 3000 },
)
.toBe(true);
}
}
const metrics = await send<{
cssVisualViewport: { zoom: number };
visualViewport: { clientWidth: number };
}>("Page.getLayoutMetrics", {}, rootSession);
expect(metrics.cssVisualViewport.zoom).toBeCloseTo(configuration.zoom, 4);
},
);
} finally {
await server.stop();
}
}, 30_000);
});
@@ -0,0 +1,208 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import { SessionManager } from "@/session-manager/manager";
import { RefStore, type VisualRefInput } from "@/session-manager/ref-store";
import { handleDownload } from "../download";
import { handleRequestHelp, resetHelpLifecycleForTests } from "../human-loop";
import { handleClick, handleFill, handleHover, handlePress, handleSelect } from "../interaction";
import { handleGetHtml, handleScreenshot } from "../observation";
import type { CdpRunner } from "../shared";
import { lookupRefTarget, lookupSnapshotRef, resolveSnapshotRef } from "../snapshot-ref";
import { handleUpload } from "../upload";
function visual(): VisualRefInput {
return {
kind: "visual-region",
candidate: {
document: {
attachmentId: "a",
target: { tabId: 4, sessionId: "child" },
frameId: "frame",
documentElementBackendNodeId: 1,
},
backendNodeId: 42,
parentBackendNodeId: 1,
region: {
status: "available",
borderBox: { x: 0, y: 0, width: 100, height: 50 },
crop: { x: 0, y: 0, width: 100, height: 50 },
},
},
};
}
async function setup() {
const manager = new SessionManager({
agentWindow: {
create: async () => 100,
remove: async () => {},
ensureActiveTab: async () => 4,
},
});
const ctx = await manager.start("test");
ctx.refStore.replace([
["e1", visual()],
["e2", { backendNodeId: 43, tabId: 4 }],
]);
const send = vi.fn(async () => {
throw new Error("unexpected target operation");
});
const cdp = { send, sendToTarget: send, trackSessionTab: vi.fn() } as unknown as CdpRunner;
const tab = { id: 4, windowId: 100, active: true, url: "https://example.com" } as chrome.tabs.Tab;
const tabsApi = { get: vi.fn(async () => tab), query: vi.fn(async () => [tab]) };
return { manager, ctx, send, cdp, tabsApi };
}
describe("typed visual refs", () => {
afterEach(() => resetHelpLifecycleForTests());
it("keeps candidate evidence and cannot resolve a visual anchor as a bare DOM node", () => {
const store = new RefStore();
const input = visual();
store.replace([
["@e1", input],
["e2", { backendNodeId: 43, tabId: 4 }],
]);
const entry = store.resolveEntry("e1");
expect(entry?.kind).toBe("visual-region");
if (entry?.kind !== "visual-region") throw new Error("missing visual ref");
expect(entry.candidate).toBe(input.candidate);
expect(entry.generation).toBe(1);
expect(store.resolve("@e1")).toBeNull();
expect(store.resolve("e2")).toBe(43);
// The latest observation replaces both target kinds; eN strings carry no generation.
store.replace([["e1", { backendNodeId: 99, tabId: 4 }]]);
expect(store.resolve("e1")).toBe(99);
expect(store.resolveEntry("e1")).toMatchObject({ kind: "dom", generation: 2 });
expect(store.resolveEntry("e2")).toBeNull();
});
it("rejects missing visual identity without partially publishing a replacement", () => {
const store = new RefStore();
store.set("e1", 10, { tabId: 4 });
const invalid = visual();
invalid.candidate.document.attachmentId = "";
expect(() =>
store.replace([
["e2", 20],
["e3", invalid],
]),
).toThrow(TypeError);
expect(store.resolveEntry("e1")).toMatchObject({ generation: 0, backendNodeId: 10 });
expect(store.size()).toBe(1);
store.replace([["e1", visual()]]);
expect(store.resolveEntry("e1")?.generation).toBe(1);
});
it("isolates tabs/sessions and distinguishes unsupported targets from missing refs", async () => {
const { ctx } = await setup();
expect(lookupRefTarget(ctx, "@e1", 4)?.kind).toBe("visual-region");
expect(lookupRefTarget(ctx, "e1", 5)).toBeNull();
expect(lookupSnapshotRef(ctx, "e1", 4)).toBeNull();
expect(resolveSnapshotRef(ctx, "e1", 4)).toMatchObject({
code: "unsupported",
data: { reason: "ref_kind_unsupported" },
});
expect(resolveSnapshotRef(ctx, "e1", 5)).toMatchObject({
code: "not_found",
data: { reason: "ref_not_found" },
});
expect(lookupRefTarget({ ...ctx, refStore: new RefStore() }, "e1", 4)).toBeNull();
expect(lookupSnapshotRef(ctx, "e2", 4)).toMatchObject({ backendNodeId: 43 });
});
it.each([
"click",
"fill",
"hover",
"press",
"select",
"get_html",
"screenshot",
])("rejects visual refs in %s before target effects", async (tool) => {
const { manager, send, cdp, tabsApi } = await setup();
const params = { session_id: "test", tab_id: 4, ref: "@e1" };
const deps = { cdp, tabsApi };
const captureVisibleTab = vi.fn();
let result: unknown;
switch (tool) {
case "click":
result = await handleClick(manager, params, deps);
break;
case "fill":
result = await handleFill(manager, { ...params, value: "hello" }, deps);
break;
case "hover":
result = await handleHover(manager, params, deps);
break;
case "press":
result = await handlePress(manager, { ...params, key: "Enter" }, deps);
break;
case "select":
result = await handleSelect(manager, { ...params, values: ["one"] }, deps);
break;
case "get_html":
result = await handleGetHtml(manager, params, deps);
break;
case "screenshot":
result = await handleScreenshot(manager, params, {
...deps,
captureApi: { ...tabsApi, captureVisibleTab },
});
break;
}
expect(result).toMatchObject({ code: "unsupported", data: { reason: "ref_kind_unsupported" } });
expect(send).not.toHaveBeenCalled();
expect(captureVisibleTab).not.toHaveBeenCalled();
});
it.each([
"input",
"drop",
"download",
] as const)("rejects visual refs before %s file-transfer setup", async (mode) => {
const { manager, send, cdp, tabsApi } = await setup();
const params = { session_id: "test", tab_id: 4, ref: "e1" };
const result =
mode === "download"
? await handleDownload(
manager,
{ ...params, browser_relative_dir: "test" },
{ cdp, tabsApi },
)
: await handleUpload(
manager,
{
...params,
mode,
files: [{ transfer_id: "test", name: "test.txt", staged_path: "/unused/test.txt" }],
},
{ cdp, tabsApi },
);
expect(result).toMatchObject({ code: "unsupported", data: { reason: "ref_kind_unsupported" } });
expect(send).not.toHaveBeenCalled();
});
it("does not scroll or highlight a visual anchor in human help", async () => {
const { manager, cdp, tabsApi, send } = await setup();
const sendToTab = vi.fn(async (_tab: number, _message: unknown) => ({
type: "bsk-help-response",
outcome: "continued",
}));
const result = await handleRequestHelp(
manager,
{ session_id: "test", tab_id: 4, prompt: "help", targets: [{ ref: "@e1" }] },
{
cdp,
tabsApi,
sendToTab,
windows: { update: vi.fn(async () => ({}) as never) },
activateTab: vi.fn(async () => {}),
notifications: null,
autoAttachLifecycle: false,
},
);
expect(result).toMatchObject({ resolved_targets: [{ ref: "@e1", matched: false }] });
expect(send).not.toHaveBeenCalled();
expect(sendToTab.mock.calls[0][1]).toMatchObject({ rects: [], selectors: [] });
});
});
+49 -1
View File
@@ -2,12 +2,14 @@ import { OVERLAY_AUTOMATION_BYPASS } from "@/lib/overlay-bridge";
import type { SessionManager } from "@/session-manager/manager";
import type { Transport } from "@/transport/transport";
import type {
BlurParams,
ClickParams,
ConsoleParams,
DownloadParams,
EmulateParams,
EvaluateParams,
FillParams,
FocusParams,
GetHtmlParams,
HoverParams,
HoverResult,
@@ -27,6 +29,7 @@ import type {
ResponseFrame,
RpcError,
ScreenshotParams,
ScrollToParams,
SelectParams,
SnapshotParams,
UploadParams,
@@ -39,7 +42,15 @@ import { type EmulateCdpRunner, handleEmulate } from "./emulate";
import { classifyCdpError } from "./errors";
import { handleEvaluate } from "./evaluate";
import { handleRequestHelp } from "./human-loop";
import { handleClick, handleFill, handleHover, handlePress, handleSelect } from "./interaction";
import {
handleBlur,
handleClick,
handleFill,
handleFocus,
handleHover,
handlePress,
handleSelect,
} from "./interaction";
import {
handleNavigate,
handleNavigateBack,
@@ -61,6 +72,7 @@ import {
handleRecordStop,
type RecordRuntimeDeps,
} from "./record";
import { handleScrollTo } from "./scroll";
import {
handleSessionStart,
handleSessionStop,
@@ -515,6 +527,39 @@ export class ToolDispatcher {
);
return this.rememberHover((req.params as HoverParams).session_id, result);
}
case "tool.scroll_to":
return this.withHoverReleaseForRequest(
req.params as ScrollToParams,
() =>
handleScrollTo(
this.sessions,
req.params as ScrollToParams,
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
),
signal,
);
case "tool.focus":
return this.withHoverReleaseForRequest(
req.params as FocusParams,
() =>
handleFocus(
this.sessions,
req.params as FocusParams,
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
),
signal,
);
case "tool.blur":
return this.withHoverReleaseForRequest(
req.params as BlurParams,
() =>
handleBlur(
this.sessions,
req.params as BlurParams,
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
),
signal,
);
case "tool.fill":
return this.withHoverReleaseForRequest(
req.params as FillParams,
@@ -785,6 +830,9 @@ function sessionIdForBrowserControlMethod(req: RequestFrame): string | null {
case "tool.reload":
case "tool.click":
case "tool.hover":
case "tool.scroll_to":
case "tool.focus":
case "tool.blur":
case "tool.fill":
case "tool.press":
case "tool.select":
+2 -1
View File
@@ -161,7 +161,8 @@ export async function resolveNodeGeometry(
return geometryError(`could not resolve frame geometry for ${address.frameId}`);
topVisibleRegions = projectRegionToViewport(localRegion, projection);
if (address.target.sessionId) {
const localViewport = projection.edges[0]?.sourceViewport ?? projection.topViewport;
const localViewport = await context.viewport(address.target);
if (!localViewport) return geometryError("could not resolve target viewport geometry");
const localVisibleRegions = localRegion
.map((polygon) =>
clipPolygon(
@@ -25,6 +25,12 @@ export interface UnavailableFrameProjection {
ownerBackendNodeId: number;
}
export interface UnavailableSnapshotCoordinates {
status: "unavailable";
source: CoordinateOwner;
reason: "snapshot-coordinates-unavailable";
}
export type SnapshotProjectionResult =
| { status: "available"; projection: SnapshotProjection }
| UnavailableFrameProjection;
@@ -32,25 +38,74 @@ export type SnapshotProjectionResult =
/** Descendants inherit the failed boundary without attempting another owner read. */
export type FrameProjectionIssue =
| UnavailableFrameProjection
| { status: "blocked"; source: CoordinateOwner; cause: UnavailableFrameProjection };
| UnavailableSnapshotCoordinates
| {
status: "blocked";
source: CoordinateOwner;
cause: UnavailableFrameProjection | UnavailableSnapshotCoordinates;
};
export type FrameProjectionState = SnapshotProjectionResult | FrameProjectionIssue;
/** DOMSnapshot bounds are document-relative CSS pixels, independent of raster DPR. */
export interface SnapshotCoordinates {
layoutUnitsPerCssPixel: number;
scrollCss: { x: number; y: number };
}
/** Snapshot scroll offsets share the raw layout units of its bounds. Only a
* target's root document may fall back to that target's CSS layout viewport. */
export function snapshotCoordinates(
document: { scrollOffsetX?: number; scrollOffsetY?: number },
layoutUnitsPerCssPixel: number | null,
rootScrollCss?: { x?: number; y?: number },
): SnapshotCoordinates | null {
if (
layoutUnitsPerCssPixel === null ||
!Number.isFinite(layoutUnitsPerCssPixel) ||
layoutUnitsPerCssPixel <= 0
)
return null;
const x =
document.scrollOffsetX === undefined
? rootScrollCss?.x
: document.scrollOffsetX / layoutUnitsPerCssPixel;
const y =
document.scrollOffsetY === undefined
? rootScrollCss?.y
: document.scrollOffsetY / layoutUnitsPerCssPixel;
if (x === undefined || y === undefined || !Number.isFinite(x) || !Number.isFinite(y)) return null;
return { layoutUnitsPerCssPixel, scrollCss: { x, y } };
}
/** Convert raw document-relative Blink layout units exactly once, before any
* CSS viewport clipping or frame projection. This scale is not screenshot zoom. */
export function snapshotViewportRect(
bounds: number[],
owner: CoordinateOwner,
scroll: { x: number; y: number },
coordinates: SnapshotCoordinates | null,
): FrameViewportRect | null {
if (bounds.length < 4 || !bounds.slice(0, 4).every(Number.isFinite)) return null;
if (!coordinates || bounds.length < 4 || !bounds.slice(0, 4).every(Number.isFinite)) return null;
const { layoutUnitsPerCssPixel: scale, scrollCss: scroll } = coordinates;
const [x, y, width, height] = bounds;
if (width <= 0 || height <= 0 || !Number.isFinite(scroll.x) || !Number.isFinite(scroll.y)) {
if (
!Number.isFinite(scale) ||
scale <= 0 ||
width <= 0 ||
height <= 0 ||
!Number.isFinite(scroll.x) ||
!Number.isFinite(scroll.y)
) {
return null;
}
return {
space: "frame-viewport-css",
owner,
rect: { x: x - scroll.x, y: y - scroll.y, width, height },
rect: {
x: x / scale - scroll.x,
y: y / scale - scroll.y,
width: width / scale,
height: height / scale,
},
};
}
@@ -9,15 +9,24 @@ import {
type Polygon,
type ProjectiveEdge,
parseCdpQuad,
projectPolygon,
type Quad,
type Size,
viewportPolygon,
} from "../geometry";
import { type CdpRunner, sendToCdpTarget } from "../shared";
import type { CoordinateOwner, CssViewport, SnapshotProjectionResult } from "./coordinate-types";
import { readSnapshotOwnerSizes } from "./snapshot-owner-sizes";
export interface LayoutMetrics {
cssVisualViewport?: { zoom?: number };
visualViewport?: { zoom?: number };
cssVisualViewport?: {
zoom?: number;
scale?: number;
clientWidth?: number;
clientHeight?: number;
};
visualViewport?: { zoom?: number; scale?: number; clientWidth?: number; clientHeight?: number };
cssLayoutViewport?: {
clientWidth?: number;
clientHeight?: number;
@@ -27,6 +36,29 @@ export interface LayoutMetrics {
layoutViewport?: { clientWidth?: number; clientHeight?: number; pageX?: number; pageY?: number };
}
/** Chromium emits these paired floating-point dimensions before/after dividing
* by LayoutZoomFactor. Integer layout viewport dimensions lose precision at
* fractional zoom. Neither visualViewport.scale nor .zoom is this conversion.
* See InspectorPageAgent::getLayoutMetrics in Chromium. */
export function snapshotLayoutScale(metrics: LayoutMetrics): number | null {
for (const dimension of ["clientWidth", "clientHeight"] as const) {
const raw = metrics.visualViewport?.[dimension];
const css = metrics.cssVisualViewport?.[dimension];
if (
raw !== undefined &&
css !== undefined &&
Number.isFinite(raw) &&
Number.isFinite(css) &&
raw > 0 &&
css > 0
) {
const scale = raw / css;
if (Number.isFinite(scale) && scale > 0) return scale;
}
}
return null;
}
/** The legacy field is a compatibility fallback, never a source of raster DPR. */
export function cssViewport(metrics: LayoutMetrics): CssViewport {
const value = metrics.cssLayoutViewport ?? metrics.layoutViewport;
@@ -42,9 +74,11 @@ export function cssViewport(metrics: LayoutMetrics): CssViewport {
/** One read-only measurement phase. Discard after scrolling or any later operation. */
export class GeometryContext {
private readonly metrics = new Map<string, Promise<LayoutMetrics>>();
private readonly frameViewports = new Map<string, Promise<Size | null>>();
private readonly owners = new Map<string, Promise<Quad | null>>();
private readonly projections = new Map<string, Promise<GeometryProjection | null>>();
private readonly snapshotOwners = new Map<string, Promise<{ quad: Quad; size: Size } | null>>();
private readonly sizeBatches = new Map<string, () => Promise<Map<number, Size>>>();
private active = 0;
private readonly waiting: Array<() => void> = [];
private graphPromise?: Promise<CdpFrameGraph | null>;
@@ -131,6 +165,35 @@ export class GeometryContext {
: null;
}
/** The full OOPIF viewport maps to its owner's content quad. Layout metrics
* exclude occupied scrollbars and remain the clipping boundary, not the scale. */
private frameViewport(target: CdpTarget): Promise<Size | null> {
const key = cdpTargetKey(target);
let promise = this.frameViewports.get(key);
if (!promise) {
promise = this.request<{ result?: { value?: Size }; exceptionDetails?: unknown }>(
target,
"Runtime.evaluate",
{
expression: "({ width: window.innerWidth, height: window.innerHeight })",
returnByValue: true,
},
).then(({ result, exceptionDetails }) => {
const size = result?.value;
return !exceptionDetails &&
size &&
Number.isFinite(size.width) &&
Number.isFinite(size.height) &&
size.width > 0 &&
size.height > 0
? size
: null;
});
this.frameViewports.set(key, promise);
}
return promise;
}
ownerContent(target: CdpTarget, backendNodeId: number): Promise<Quad | null> {
const key = `${cdpTargetKey(target)}:${backendNodeId}`;
let promise = this.owners.get(key);
@@ -186,12 +249,34 @@ export class GeometryContext {
: { status: "unavailable", source, ownerBackendNodeId };
}
/** Register one target's snapshot owners; the batch starts only when a
* projection actually needs a size. Single owners keep the cheaper old path. */
registerSnapshotOwners(target: CdpTarget, owners: ReadonlySet<number>): void {
const key = cdpTargetKey(target);
if (owners.size < 2 || this.sizeBatches.has(key)) return;
let pending: Promise<Map<number, Size>> | undefined;
this.sizeBatches.set(key, () => {
pending ??= readSnapshotOwnerSizes(
(method, params, cleanup) => this.request(target, method, params, cleanup),
owners,
).catch((error) => {
if (error && typeof error === "object" && "name" in error && error.name === "AbortError")
throw error;
return new Map<number, Size>();
});
return pending;
});
}
private async snapshotOwner(
target: CdpTarget,
backendNodeId: number,
): Promise<{ quad: Quad; size: Size } | null> {
const quad = await this.ownerContent(target, backendNodeId);
if (!quad) return null;
const batchSize = (await this.sizeBatches.get(cdpTargetKey(target))?.())?.get(backendNodeId);
if (this.signal?.aborted) throw new DOMException("geometry aborted", "AbortError");
if (batchSize) return { quad, size: batchSize };
const resolved = await this.request<{ object?: { objectId?: string } }>(
target,
"DOM.resolveNode",
@@ -294,12 +379,19 @@ export class GeometryContext {
const parent = frames.get(root.parentFrameId);
if (!parent || root.ownerBackendNodeId === undefined) return null;
const destinationQuad = await this.ownerContent(parent.target, root.ownerBackendNodeId);
const source = await this.viewport(root.target);
if (!destinationQuad) return null;
const source = await this.frameViewport(root.target);
const visible = await this.viewport(root.target);
const parentRoot = this.targetRoot(frames, parent);
if (!destinationQuad || !source || !parentRoot) return null;
if (!source || !visible || !parentRoot) return null;
const destinationClips = await this.clips(frames, parent, parentRoot);
if (!destinationClips) return null;
edges.push({ sourceViewport: source, destinationQuad, destinationClips });
const edge = { sourceViewport: source, destinationQuad, destinationClips };
// Keep scrollbar strips clipped at every target boundary, including
// intermediate OOPIFs. Project the clip with the same full-viewport scale.
if (source.width !== visible.width || source.height !== visible.height)
destinationClips.push(projectPolygon(viewportPolygon(visible), edge));
edges.push(edge);
root = parentRoot;
}
const topViewport = await this.viewport(root.target);
@@ -0,0 +1,74 @@
import type { Size } from "../geometry";
interface SerializedValue {
type: string;
value?: unknown;
}
type Send = <T>(method: string, params: object, cleanup?: boolean) => Promise<T>;
/** Batch only accessible frame owners. Missing/shadow/cross-origin owners keep
* the existing node-scoped read; no page-wide style columns are requested. */
export async function readSnapshotOwnerSizes(
send: Send,
owners: ReadonlySet<number>,
): Promise<Map<number, Size>> {
const sizes = new Map<number, Size>();
const objectGroup = `bsk-frame-sizes-${crypto.randomUUID()}`;
try {
const reply = await send<{ result?: { deepSerializedValue?: SerializedValue } }>(
"Runtime.evaluate",
{
expression: `(() => {
const result = [];
const documents = [document];
for (let i = 0; i < documents.length && result.length < ${owners.size}; i++) {
for (const owner of documents[i].querySelectorAll('iframe,frame')) {
if (result.length >= ${owners.size}) break;
if (!owner.isConnected) continue;
const style = getComputedStyle(owner);
result.push([owner, JSON.stringify({
width: owner.clientWidth - parseFloat(style.paddingLeft) - parseFloat(style.paddingRight),
height: owner.clientHeight - parseFloat(style.paddingTop) - parseFloat(style.paddingBottom)
})]);
try { if (owner.contentDocument) documents.push(owner.contentDocument); } catch {}
}
}
return result;
})()`,
objectGroup,
serializationOptions: {
serialization: "deep",
maxDepth: 3,
additionalParameters: { maxNodeDepth: 0, includeShadowTree: "none" },
},
},
);
const result = reply.result?.deepSerializedValue;
if (result?.type !== "array" || !Array.isArray(result.value)) return sizes;
for (const entry of result.value as SerializedValue[]) {
if (entry.type !== "array" || !Array.isArray(entry.value)) continue;
const [node, data] = entry.value as SerializedValue[];
if (node?.type !== "node" || data?.type !== "string" || typeof data.value !== "string")
continue;
const id = (node.value as { backendNodeId?: number } | undefined)?.backendNodeId;
if (id === undefined || !owners.has(id)) continue;
try {
const size = JSON.parse(data.value) as Size;
if (
size &&
Number.isFinite(size.width) &&
Number.isFinite(size.height) &&
size.width > 0 &&
size.height > 0
)
sizes.set(id, size);
} catch {
/* An invalid entry uses the existing individual read. */
}
}
} finally {
await send("Runtime.releaseObjectGroup", { objectGroup }, true).catch(() => {});
}
return sizes;
}
+206 -2
View File
@@ -1,5 +1,4 @@
// DOM interaction tools — `tool.click`, `tool.fill`, `tool.press`, and
// `tool.select`.
// DOM interaction tools — click, hover, focus/blur, fill, press, and select.
//
// All interaction tools:
// 1. Resolve target tab (sandbox: must be inside Agent Window).
@@ -14,10 +13,14 @@ import { ChromiumCdp } from "@/browser-driver/chromium-cdp";
import type { CdpTarget } from "@/browser-driver/frame-graph";
import type { SessionContext, SessionManager } from "@/session-manager/manager";
import type {
BlurParams,
BlurResult,
ClickParams,
ClickResult,
FillParams,
FillResult,
FocusParams,
FocusResult,
HoverParams,
HoverResult,
KeyModifier,
@@ -227,6 +230,207 @@ export async function resolveBackendNode(
}
}
// Check the target's own root, then its hosts: closed shadow roots are not
// reachable via host.shadowRoot. DOM focus remains meaningful in background tabs.
const FOCUS_CHECK = `function() {
if (!this.isConnected) return false;
let element = this;
while (element) {
const root = element.getRootNode();
if (root.activeElement !== element) return false;
element = root.host;
}
return true;
}`;
const BLUR_TARGET = `function() {
if (!this.isConnected) throw new Error('blur target is detached');
const wasFocused = (${FOCUS_CHECK}).call(this);
if (typeof this.blur !== 'function') return { ok: false, was_focused: wasFocused };
this.blur();
return { ok: true, was_focused: wasFocused };
}`;
interface FocusScriptReply<T> {
result?: { value?: T };
exceptionDetails?: { text?: string; exception?: { description?: string } };
}
function checkFocusAbort(signal: AbortSignal | undefined): void {
if (signal?.aborted) throw new DOMException("interaction aborted", "AbortError");
}
// Only focus/blur use this view. Guard every CDP boundary, including commands
// inside selector resolution and frame scrolling, without changing other tools.
function focusCdp(cdp: CdpRunner, signal: AbortSignal | undefined): CdpRunner {
const send = <T = unknown>(target: CdpTarget, method: string, params?: object): Promise<T> => {
checkFocusAbort(signal);
return cdpRunnerForTarget(cdp, target).send<T>(target.tabId, method, params);
};
return {
send: (tabId, method, params) => send({ tabId }, method, params),
sendToTarget: send,
trackSessionTab: cdp.trackSessionTab?.bind(cdp),
getFrameGraph: cdp.getFrameGraph
? (tabId) => {
checkFocusAbort(signal);
return cdp.getFrameGraph!(tabId);
}
: undefined,
};
}
// ---------------------------------------------------------------------------
// tool.focus / tool.blur
// ---------------------------------------------------------------------------
export function handleFocus(
manager: SessionManager,
params: FocusParams,
deps: InteractionDeps = getDefaultDeps(),
): Promise<FocusResult | RpcError> {
return changeFocus(manager, params, deps, "focus");
}
export function handleBlur(
manager: SessionManager,
params: BlurParams,
deps: InteractionDeps = getDefaultDeps(),
): Promise<BlurResult | RpcError> {
return changeFocus(manager, params, deps, "blur");
}
function changeFocus(
manager: SessionManager,
params: FocusParams,
deps: InteractionDeps,
action: "focus",
): Promise<FocusResult | RpcError>;
function changeFocus(
manager: SessionManager,
params: BlurParams,
deps: InteractionDeps,
action: "blur",
): Promise<BlurResult | RpcError>;
async function changeFocus(
manager: SessionManager,
params: FocusParams | BlurParams,
deps: InteractionDeps,
action: "focus" | "blur",
): Promise<FocusResult | BlurResult | RpcError> {
const ctx = lookupSession(manager, params, action);
if (isRpcError(ctx)) return ctx;
const cdp = focusCdp(deps.cdp, deps.signal);
let objectId: string | undefined;
let cleanupCdp: CdpRunner | undefined;
let tabId: number | undefined;
try {
checkFocusAbort(deps.signal);
const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi);
checkFocusAbort(deps.signal);
if (isRpcError(target)) return target;
const denied = enforceAgentWindow(ctx, target, action);
if (denied) return denied;
tabId = target.tabId;
const dialogCursor = markDialogCursor(deps.cdp, tabId);
const node = await resolveBackendNode(cdp, ctx, target, params, action);
checkFocusAbort(deps.signal);
if (isRpcError(node)) return node;
const nodeCdp = cdpRunnerForTarget(cdp, node.cdpTarget);
cleanupCdp = cdpRunnerForTarget(deps.cdp, node.cdpTarget);
deps.cdp.trackSessionTab?.(ctx.sessionId, tabId);
if (action === "focus") {
const scrollErr = await scrollElementAndFramesIntoView(
cdp,
tabId,
node.cdpTarget,
node.backendNodeId,
node.frameId,
);
checkFocusAbort(deps.signal);
if (scrollErr) return scrollErr;
}
const resolved = await backendNodeToObject(nodeCdp, tabId, node.backendNodeId);
// Keep the handle before checking cancellation so finally can release it.
if (!isRpcError(resolved)) objectId = resolved;
checkFocusAbort(deps.signal);
if (isRpcError(resolved)) return resolved;
const runScript = async <T>(functionDeclaration: string): Promise<T | undefined> => {
const reply = await nodeCdp.send<FocusScriptReply<T>>(
target.tabId,
"Runtime.callFunctionOn",
{
objectId,
functionDeclaration,
returnByValue: true,
},
);
checkFocusAbort(deps.signal);
if (reply.exceptionDetails) {
const details = reply.exceptionDetails;
throw new Error(
`${action} script failed: ${details.exception?.description ?? details.text ?? "unknown exception"}`,
);
}
return reply.result?.value;
};
let wasFocused: boolean | undefined;
if (action === "focus") {
await nodeCdp.send(tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
checkFocusAbort(deps.signal);
} else {
const mutation = await runScript<{ ok: boolean; was_focused: boolean }>(BLUR_TARGET);
if (typeof mutation?.ok !== "boolean" || typeof mutation.was_focused !== "boolean") {
throw new Error("blur script returned an unexpected result");
}
if (!mutation.ok) {
return { code: "invalid_params", message: "target element does not support blur()" };
}
wasFocused = mutation.was_focused;
}
// Use a separate call so microtasks from focus/blur handlers finish before
// we report success. In particular, a blur handler can restore focus.
const focused = await runScript<boolean>(FOCUS_CHECK);
if (typeof focused !== "boolean") {
throw new Error(`${action} verification returned an unexpected result`);
}
if (focused !== (action === "focus")) {
throw new Error(
action === "focus"
? "target element did not become focused"
: "target element remained focused after blur()",
);
}
return attachDialogs(deps.cdp, tabId, dialogCursor, {
tab_id: tabId,
used_ref: node.usedRef,
used_selector: node.usedSelector,
focused,
...(action === "blur" ? { was_focused: wasFocused! } : {}),
});
} catch (err) {
return (
throwIfAborted(deps.signal) ?? {
code: "cdp_failed",
message: err instanceof Error ? err.message : String(err),
}
);
} finally {
if (objectId !== undefined && cleanupCdp && tabId !== undefined) {
// Cleanup must also run after cancellation. Navigation may have already
// disposed the object, which must not replace the operation's result.
try {
await cleanupCdp.send(tabId, "Runtime.releaseObject", { objectId });
} catch {
// The target or execution context may no longer exist.
}
}
}
}
export async function resolveActionTarget(
cdp: CdpRunner,
ctx: SessionContext,
+35 -7
View File
@@ -48,7 +48,7 @@ import {
normaliseRef as sharedNormaliseRef,
type ToolEffect,
} from "./shared";
import { resolveSnapshotRef } from "./snapshot-ref";
import { lookupRefTarget, resolveSnapshotRef } from "./snapshot-ref";
import { type CapturedNode, type CapturedSurfaceProbe, probeHoverSurfaces } from "./vom/capture";
import { captureObservationFacts, semanticCapture } from "./vom/capture-coordinator";
import type { FrameDocument as CapturedFrameDocument } from "./vom/frame-document";
@@ -308,6 +308,12 @@ export async function handleScreenshot(
if (!deps.cdp) {
return { code: "cdp_failed", message: "screenshot ref capture requires CDP" };
}
if (lookupRefTarget(ctx, ref, target.tabId)?.kind === "visual-region")
return rpcError(
"unsupported",
"ref_kind_unsupported",
"visual-region screenshot execution is not available in this build",
);
const node = resolveSnapshotRef(ctx, ref, target.tabId);
if (isRpcError(node)) return node;
if (signal?.aborted) return cancelled("screenshot");
@@ -926,19 +932,41 @@ export async function captureVomObservation(
);
// Reserve space using the renderer's character-based token estimate. Like VOM
// headers, this integrity notice remains visible even under a tiny token budget.
const geometryNotice = facts.issues.some((issue) => issue.stage === "geometry")
? "@warning iframe geometry incomplete: some frame content has no top-level coordinates.\n"
: "";
const notices: string[] = [];
if (facts.issues.some((issue) => issue.stage === "geometry"))
notices.push(
"@warning geometry incomplete: some page or frame content has no top-level coordinates.",
);
const incompleteStages = ["dom", "ax", "forms", "ownership"].filter((stage) =>
facts.issues.some((issue) => issue.stage === stage),
);
if (incompleteStages.length)
notices.push(
`@warning observation incomplete: some ${incompleteStages.join(", ")} data is unavailable or omitted.`,
);
if (
facts.issues.some(
(issue) => issue.stage === "identity" && issue.reason !== "identity-unverified",
)
)
notices.push(
"@warning observation incomplete: some documents were omitted because their identity changed or could not be revalidated.",
);
if (facts.issues.some((issue) => issue.reason === "identity-unverified"))
notices.push(
"@warning document identity unverified: some retained documents could not be checked for changes during capture.",
);
const captureNotice = notices.join("\n");
const rendered = renderVom(decoratedScene, {
maxDepth: options.maxDepth,
maxTokens:
!geometryNotice || options.maxTokens === undefined
!captureNotice || options.maxTokens === undefined
? options.maxTokens
: Math.max(0, options.maxTokens - Math.ceil(geometryNotice.length / 4)),
: Math.max(0, options.maxTokens - Math.ceil((captureNotice.length + 1) / 4)),
redactValues: options.redactValues,
activeRegionPolicy: options.activeRegionPolicy,
});
if (geometryNotice) rendered.text += `\n${geometryNotice.trimEnd()}`;
if (captureNotice) rendered.text += `\n${captureNotice}`;
throwIfAborted(options.signal, "observation");
return projectRecordSafeObservation({
rootFrameId: captured.rootFrameId ?? normalizedDocuments[0]?.frameId ?? "root",
@@ -0,0 +1,142 @@
import { cdpTargetKey } from "@/browser-driver/frame-graph";
import type { NodeAddress } from "./frame-geometry";
import {
clipPolygon,
projectRegionToViewport,
type Region,
rectPolygon,
regionBounds,
type ViewportRect,
} from "./geometry";
import { GeometryContext } from "./geometry/frame-context";
import { type CdpRunner, sendToCdpTarget } from "./shared";
// Ask the renderer to apply containing-block clips (including shadow DOM),
// instead of treating layout quads as visible pixels. Each document is measured
// locally; the existing frame projections supply the top-viewport coordinates.
// IntersectionObserver reports a bounding rectangle, not an occlusion/hit test.
const VISIBLE_RECT = `function(timeoutMs) {
const element = this;
const visible = () => element.isConnected && element.checkVisibility({
checkOpacity: true, checkVisibilityCSS: true, contentVisibilityAuto: true
});
if (typeof element.checkVisibility !== 'function' || !visible()) return null;
return new Promise((resolve, reject) => {
const observer = new IntersectionObserver(([entry]) => {
clearTimeout(timer);
observer.disconnect();
const rect = entry.intersectionRect;
resolve(visible() && entry.isIntersecting && rect.width > 0 && rect.height > 0
? { x: rect.x, y: rect.y, width: rect.width, height: rect.height }
: null);
}, { root: element.ownerDocument });
const timer = setTimeout(() => {
observer.disconnect();
reject(new Error('scroll-to visibility measurement timed out'));
}, timeoutMs);
observer.observe(element);
});
}`;
async function visibleRect(
cdp: CdpRunner,
address: NodeAddress,
deadline: number,
): Promise<ViewportRect | null> {
const resolved = await sendToCdpTarget<{ object?: { objectId?: string } }>(
cdp,
address.target,
"DOM.resolveNode",
{ backendNodeId: address.backendNodeId },
);
if (!resolved.object?.objectId) throw new Error("scroll-to could not resolve the target element");
const reply = await sendToCdpTarget<{
result?: { value?: ViewportRect | null };
exceptionDetails?: { text?: string; exception?: { description?: string } };
}>(cdp, address.target, "Runtime.callFunctionOn", {
objectId: resolved.object.objectId,
functionDeclaration: VISIBLE_RECT,
// Bound this read even if a background document stops rendering. Its
// observer always disconnects; cancellation cannot leave a live observer.
arguments: [{ value: Math.max(1, Math.min(1_000, deadline - Date.now())) }],
awaitPromise: true,
returnByValue: true,
});
if (reply.exceptionDetails) {
const details = reply.exceptionDetails;
throw new Error(details.exception?.description ?? details.text ?? "visibility script failed");
}
const rect = reply.result?.value;
if (rect === null) return null;
if (
!rect ||
![rect.x, rect.y, rect.width, rect.height].every(Number.isFinite) ||
rect.width <= 0 ||
rect.height <= 0
) {
throw new Error("scroll-to visibility measurement returned invalid bounds");
}
return rect;
}
async function projectRect(
context: GeometryContext,
address: NodeAddress,
rect: ViewportRect,
): Promise<Region> {
let region: Region = [rectPolygon({ x: rect.x, y: rect.y, w: rect.width, h: rect.height })];
if (!address.frameId) return region;
const frame = await context.frame(address.frameId);
if (!frame || cdpTargetKey(frame.target) !== cdpTargetKey(address.target)) {
throw new Error("scroll-to frame no longer belongs to its target");
}
const parent = frame.parentFrameId ? await context.frame(frame.parentFrameId) : undefined;
if (parent && cdpTargetKey(parent.target) === cdpTargetKey(frame.target)) {
// DOM rectangles are document-local. Same-target iframe quads are already
// target-local, so cross this one boundary before the target projection.
const viewport = await context.viewport(frame.target);
if (frame.ownerBackendNodeId === undefined || !viewport) {
throw new Error("scroll-to could not resolve the frame viewport");
}
const local = await context.snapshotProjection(address, frame.ownerBackendNodeId, [], viewport);
if (local.status !== "available") throw new Error("scroll-to could not project the frame");
region = projectRegionToViewport(region, local.projection.geometry);
}
const projection = await context.targetProjection(frame.frameId);
if (!projection) throw new Error("scroll-to could not project the target viewport");
return projectRegionToViewport(region, projection);
}
/** Only scroll-to needs renderer-clipped bounds; other interaction geometry is unchanged. */
export async function scrollVisibleBounds(
cdp: CdpRunner,
tabId: number,
address: NodeAddress,
deadline: number,
): Promise<ViewportRect | null> {
const context = new GeometryContext(cdp, tabId);
const rect = await visibleRect(cdp, address, deadline);
if (!rect) return null;
let region = await projectRect(context, address, rect);
if (address.frameId) {
const ancestry = await context.ancestry(address.frameId);
if (!ancestry) throw new Error("scroll-to could not resolve the frame ancestry");
for (const child of ancestry) {
const parent = child.parentFrameId ? await context.frame(child.parentFrameId) : undefined;
if (!parent || child.ownerBackendNodeId === undefined) {
throw new Error("scroll-to could not resolve the frame owner");
}
const owner = {
target: parent.target,
frameId: parent.frameId,
backendNodeId: child.ownerBackendNodeId,
};
// A visible child document can still have a hidden or clipped iframe.
const ownerRect = await visibleRect(cdp, owner, deadline);
if (!ownerRect) return null;
const clips = await projectRect(context, owner, ownerRect);
region = region.flatMap((polygon) => clips.map((clip) => clipPolygon(polygon, clip)));
}
}
return regionBounds(region);
}
+137
View File
@@ -0,0 +1,137 @@
import { ChromiumCdp } from "@/browser-driver/chromium-cdp";
import { type CdpTarget, cdpTargetKey } from "@/browser-driver/frame-graph";
import type { SessionManager } from "@/session-manager/manager";
import type { RpcError, ScrollToParams, ScrollToResult } from "@/transport/types";
import { attachDialogs, markDialogCursor } from "./dialogs";
import { cdpError, rpcError } from "./errors";
import { scrollElementAndFramesIntoView } from "./frame-geometry";
import { resolveBackendNode } from "./interaction";
import { scrollVisibleBounds } from "./scroll-visibility";
import {
type CdpRunner,
type ChromeTabsApi,
chromeTabsApi,
enforceAgentWindow,
isRpcError,
lookupSession,
resolveTargetTab,
sendToCdpTarget,
} from "./shared";
export interface ScrollToDeps {
cdp: CdpRunner;
tabsApi: ChromeTabsApi;
signal?: AbortSignal;
}
let defaultDeps: { cdp: ChromiumCdp; tabsApi: ChromeTabsApi } | null = null;
function getDefaultDeps(): { cdp: ChromiumCdp; tabsApi: ChromeTabsApi } {
if (!defaultDeps) defaultDeps = { cdp: new ChromiumCdp(), tabsApi: chromeTabsApi };
return defaultDeps;
}
export async function handleScrollTo(
manager: SessionManager,
params: ScrollToParams,
deps: ScrollToDeps = getDefaultDeps(),
): Promise<ScrollToResult | RpcError> {
const ctxOrErr = lookupSession(manager, params, "scroll-to");
if (isRpcError(ctxOrErr)) return ctxOrErr;
const ctx = ctxOrErr;
const timeout = params.timeout_ms ?? 30_000;
if (!Number.isInteger(timeout) || timeout <= 0) {
return { code: "invalid_params", message: "timeout_ms must be a positive integer" };
}
const deadline = Date.now() + timeout;
const checkActive = () => {
if (deps.signal?.aborted) throw new DOMException("scroll-to aborted", "AbortError");
if (Date.now() >= deadline) throw new DOMException("scroll-to timed out", "TimeoutError");
};
const objectGroup = `bsk-scroll-${crypto.randomUUID()}`;
const objectTargets = new Map<string, CdpTarget>();
const send = async <T>(target: CdpTarget, method: string, args?: object): Promise<T> => {
checkActive();
// Also own objects allocated by the shared scroll fallback. Record the
// target before awaiting so cancellation cannot skip their cleanup.
if (method === "DOM.resolveNode") objectTargets.set(cdpTargetKey(target), target);
const result = await sendToCdpTarget<T>(
deps.cdp,
target,
method,
method === "DOM.resolveNode" ? { ...args, objectGroup } : args,
);
checkActive();
return result;
};
let graph: ReturnType<NonNullable<CdpRunner["getFrameGraph"]>> | undefined;
const cdp: CdpRunner = {
send: (tabId, method, args) => send({ tabId }, method, args),
sendToTarget: send,
trackSessionTab: deps.cdp.trackSessionTab?.bind(deps.cdp),
getFrameGraph: deps.cdp.getFrameGraph
? async (tabId) => {
checkActive();
graph ??= deps.cdp.getFrameGraph!(tabId);
const result = await graph;
checkActive();
return result;
}
: undefined,
};
try {
checkActive();
const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi);
checkActive();
if (isRpcError(target)) return target;
const denied = enforceAgentWindow(ctx, target, "scroll-to");
if (denied) return denied;
const dialogCursor = markDialogCursor(deps.cdp, target.tabId);
const node = await resolveBackendNode(cdp, ctx, target, params, "scroll-to");
checkActive();
if (isRpcError(node)) return node;
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
const scrollError = await scrollElementAndFramesIntoView(
cdp,
target.tabId,
node.cdpTarget,
node.backendNodeId,
node.frameId,
);
checkActive();
if (scrollError) return scrollError;
const bounds = await scrollVisibleBounds(
cdp,
target.tabId,
{
target: node.cdpTarget,
backendNodeId: node.backendNodeId,
frameId: node.frameId,
},
deadline,
);
checkActive();
if (!bounds)
return rpcError(
"permission_denied",
"element_not_visible",
"scroll-to target has no visible area",
);
return attachDialogs(deps.cdp, target.tabId, dialogCursor, {
tab_id: target.tabId,
used_ref: node.usedRef,
used_selector: node.usedSelector,
...bounds,
});
} catch (error) {
if (deps.signal?.aborted) return { code: "cancelled", message: "scroll-to aborted" };
if (Date.now() >= deadline) return { code: "timeout", message: "scroll-to timed out" };
return cdpError(error);
} finally {
// Cleanup bypasses the cancellation guard and cannot replace the result.
for (const target of objectTargets.values()) {
await sendToCdpTarget(deps.cdp, target, "Runtime.releaseObjectGroup", { objectGroup }).catch(
() => {},
);
}
}
}
+1
View File
@@ -55,6 +55,7 @@ export interface CdpRunner {
sendToTarget?<T = unknown>(target: CdpTarget, method: string, params?: object): Promise<T>;
detach?(tabId: number): Promise<void>;
getFrameGraph?(tabId: number): Promise<CdpFrameGraph>;
getAttachmentId?(tabId: number): string | undefined;
ensureAttachedToUrl?(tabId: number, expectedUrl: string | undefined): Promise<void>;
trackSessionTab?(sessionId: string, tabId: number): void;
releaseSessionTab?(sessionId: string, tabId: number): Promise<void>;
+21 -6
View File
@@ -3,7 +3,7 @@
// `ref_not_found` errors for hard-failure tool paths.
import type { SessionContext } from "@/session-manager/manager";
import { normaliseRef } from "@/session-manager/ref-store";
import { normaliseRef, type RefEntry } from "@/session-manager/ref-store";
import type { RpcError } from "@/transport/types";
import { rpcError } from "./errors";
@@ -14,14 +14,22 @@ export interface SnapshotRefLookup {
cdpSessionId?: string;
}
function refEntryForTab(ctx: SessionContext, refKey: string, tabId: number) {
/** Typed lookup never turns a visual anchor into a DOM operation target. */
export function lookupRefTarget(
ctx: SessionContext,
refKey: string,
tabId: number,
): RefEntry | null {
const entry = ctx.refStore.resolveEntry(refKey);
return entry?.tabId === tabId ? entry : null;
if (!entry) return null;
const ownerTabId =
entry.kind === "visual-region" ? entry.candidate.document.target.tabId : entry.tabId;
return ownerTabId === tabId ? entry : null;
}
/**
* Soft lookup: returns `null` when the ref is unknown or bound to a
* different tab. Used by paths that report `matched: false` instead of
* different tab, or is a visual region. Used by paths that report `matched: false` instead of
* emitting an RPC error (e.g. `tool.request_help`).
*/
export function lookupSnapshotRef(
@@ -30,8 +38,8 @@ export function lookupSnapshotRef(
tabId: number,
): SnapshotRefLookup | null {
const refKey = normaliseRef(ref);
const entry = refEntryForTab(ctx, refKey, tabId);
if (!entry) return null;
const entry = lookupRefTarget(ctx, refKey, tabId);
if (!entry || entry.kind !== "dom") return null;
return {
backendNodeId: entry.backendNodeId,
refKey,
@@ -50,6 +58,13 @@ export function resolveSnapshotRef(
ref: string,
tabId: number,
): SnapshotRefLookup | RpcError {
const entry = lookupRefTarget(ctx, ref, tabId);
if (entry?.kind === "visual-region")
return rpcError(
"unsupported",
"ref_kind_unsupported",
`ref ${ref} is a visual region, not a DOM operation target`,
);
const looked = lookupSnapshotRef(ctx, ref, tabId);
if (looked === null) {
return rpcError(
@@ -2,13 +2,25 @@ import { describe, expect, it, vi } from "vitest";
import type { CdpFrame, CdpTarget } from "@/browser-driver/frame-graph";
import { cdpTargetKey } from "@/browser-driver/frame-graph";
import { OVERLAY_HOST_MARKER_ATTR } from "@/lib/overlay-bridge";
import { captureVomObservation } from "../../observation";
import type { CdpRunner } from "../../shared";
import { captureObservationFacts, semanticCapture } from "../capture-coordinator";
import { buildSemanticGraph } from "../semantic-graph/build";
import { REQUESTED_STYLES } from "../snapshot";
import { buildSemanticGraph, buildSemanticVomScene } from "../semantic-graph";
import { REQUESTED_STYLES, type SnapshotReply, VISUAL_STYLES } from "../snapshot";
import { deduplicateVisualCandidates } from "../visual-dedup";
import { discoverVisualCandidates } from "../visual-discovery";
function fixture(
options: { frames?: CdpFrame[]; fail?: string; omitDocument?: string; overlay?: string } = {},
options: {
frames?: CdpFrame[];
canvas?: boolean;
after?: Record<string, { element?: number; missing?: boolean; unreadable?: boolean }>;
fail?: string;
missingIdentity?: boolean;
omitDocument?: string;
attachmentChanged?: boolean;
overlay?: string;
} = {},
) {
const frames = options.frames ?? [
{ frameId: "main", target: { tabId: 4 } },
@@ -34,7 +46,10 @@ function fixture(
const elements = new Map(
frames.map((frame, i) => [frame.frameId, frame.target.sessionId ? 1 : i * 10 + 1]),
);
const contexts = new Map(frames.map((frame, i) => [i + 1, frame.frameId]));
const logs: Array<{ target: CdpTarget; method: string; params: Record<string, unknown> }> = [];
let snapshots = 0;
const targetCount = new Set(frames.map((frame) => cdpTargetKey(frame.target))).size;
const send = vi.fn(
async <T>(target: CdpTarget, method: string, params: object = {}): Promise<T> => {
logs.push({ target, method, params: params as Record<string, unknown> });
@@ -42,21 +57,51 @@ function fixture(
if (options.fail === `${target.sessionId ?? "main"}:${method}`)
throw new Error("fixture failure");
let result: unknown = {};
if (method === "Page.createIsolatedWorld")
result = {
executionContextId: frames.findIndex((frame) => frame.frameId === args.frameId) + 1,
};
if (method === "Runtime.evaluate" && args.contextId) {
const id = contexts.get(Number(args.contextId))!;
const change = options.after?.[id];
result =
options.missingIdentity || change?.unreadable
? {}
: {
result: {
deepSerializedValue: change?.missing
? { type: "null" }
: {
type: "node",
value: { backendNodeId: change?.element ?? elements.get(id) },
},
},
};
}
if (method === "Page.getLayoutMetrics")
result = { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } };
result = {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000, scale: 1 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 },
};
if (method === "DOMSnapshot.captureSnapshot") {
expect((params as { computedStyles: unknown }).computedStyles).toEqual(REQUESTED_STYLES);
snapshots++;
const requested = (params as { computedStyles: readonly string[] }).computedStyles;
expect([REQUESTED_STYLES, VISUAL_STYLES]).toContainEqual(requested);
result = {
strings: [
"#document",
"html",
"button",
options.canvas ? "canvas" : "button",
OVERLAY_HOST_MARKER_ATTR,
"",
"visible",
"1",
"static",
"auto",
"block",
"none",
"0px",
],
documents: frames
.filter(
@@ -66,14 +111,39 @@ function fixture(
)
.map((frame) => {
const element = elements.get(frame.frameId)!;
const localFrames = frames.filter(
(f) =>
cdpTargetKey(f.target) === cdpTargetKey(target) &&
f.frameId !== options.omitDocument,
);
const children = localFrames.filter((f) => f.parentFrameId === frame.frameId);
return {
scrollOffsetX: 0,
scrollOffsetY: 0,
frameId: frame.frameId,
nodes: {
backendNodeId: [element - 1, element + 10000, element, element + 1],
nodeName: [0, 1, 1, 2],
nodeType: [9, 10, 1, 1],
parentIndex: [-1, 0, 0, 2],
attributes: [[], [], [], frame.frameId === options.overlay ? [3, 4] : []],
backendNodeId: [
element - 1,
element + 10000,
element,
element + 1,
...children.map((f) => f.ownerBackendNodeId!),
],
nodeName: [0, 1, 1, 2, ...children.map(() => 2)],
nodeType: [9, 10, 1, 1, ...children.map(() => 1)],
parentIndex: [-1, 0, 0, 2, ...children.map(() => 2)],
attributes: [
[],
[],
[],
frame.frameId === options.overlay ? [3, 4] : [],
...children.map(() => []),
],
contentDocumentIndex: {
index: children.map((_, i) => 4 + i),
value: children.map((f) => localFrames.indexOf(f)),
},
},
layout: {
nodeIndex: [2, 3],
@@ -81,10 +151,18 @@ function fixture(
[0, 0, 1000, 800],
[10, 20, 100, 40],
],
styles: [
[7, 8, 8, 5, 6],
[7, 8, 8, 5, 6],
clientRects: [
[0, 0, 1000, 800],
[0, 0, 100, 40],
],
styles: Array.from({ length: 2 }, () =>
requested.map(
(name) =>
[7, 8, 8, 5, 6, 9, 5, 5, 10, 6, 10, 10, 10, 10, 10, 8, 10, 11][
VISUAL_STYLES.indexOf(name as (typeof VISUAL_STYLES)[number])
],
),
),
},
};
}),
@@ -108,6 +186,12 @@ function fixture(
},
);
const cdp: CdpRunner = {
getAttachmentId: () =>
options.missingIdentity
? undefined
: options.attachmentChanged && snapshots === targetCount
? "new-attachment"
: "attachment",
getFrameGraph: async () => ({ rootFrameId: frames[0].frameId, frames }),
send: (tabId, method, params) =>
(send as NonNullable<CdpRunner["sendToTarget"]>)({ tabId }, method, params),
@@ -117,12 +201,44 @@ function fixture(
}
describe("captureObservationFacts", () => {
it("normalizes snapshot-owned root scroll instead of mixing in stale CSS scroll", async () => {
const { cdp } = fixture({ frames: [{ frameId: "main", target: { tabId: 4 } }] });
const original = cdp.sendToTarget!;
cdp.sendToTarget = async (target, method, params) => {
if (method === "Page.getLayoutMetrics")
return {
visualViewport: { clientWidth: 2000 },
cssVisualViewport: { clientWidth: 1000, scale: 1 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 999, pageY: 999 },
} as never;
const reply = await original(target, method, params);
if (method === "DOMSnapshot.captureSnapshot") {
const doc = (reply as SnapshotReply).documents![0];
doc.scrollOffsetX = 40;
doc.scrollOffsetY = 200;
doc.layout!.bounds![1] = [200, 800, 240, 80];
}
return reply as never;
};
cdp.send = (tabId, method, params) => cdp.sendToTarget!({ tabId }, method, params);
const facts = await captureObservationFacts(cdp, 4);
expect(facts.documents[0].domNodes.find((node) => node.backendNodeId === 2)).toMatchObject({
localRect: { x: 80, y: 300, w: 120, h: 40 },
rect: { x: 80, y: 300, w: 120, h: 40 },
rendered: true,
});
expect(facts.issues).toEqual([]);
});
it("collects each target once and scopes equal backend IDs", async () => {
const { cdp, logs } = fixture();
const facts = await captureObservationFacts(cdp, 4);
expect(logs.filter((call) => call.method === "DOMSnapshot.captureSnapshot")).toHaveLength(2);
expect(logs.filter((call) => call.method === "Accessibility.enable")).toHaveLength(2);
expect(logs.filter((call) => call.method === "Accessibility.getFullAXTree")).toHaveLength(4);
expect(facts.documents.every((doc) => doc.identity)).toBe(true);
expect(logs.filter((call) => call.method === "DOM.describeNode")).toHaveLength(0);
expect(logs.filter((call) => call.method === "Page.createIsolatedWorld")).toHaveLength(4);
const main = facts.documents.find((doc) => doc.frame.frameId === "main")!;
const remote = facts.documents.find((doc) => doc.frame.frameId === "remote")!;
expect(main.index.nodes.get(2)).not.toBe(remote.index.nodes.get(2));
@@ -133,6 +249,219 @@ describe("captureObservationFacts", () => {
expect(facts.finishedAt).toBeGreaterThanOrEqual(facts.startedAt);
});
it.each([
{ element: 99 },
{ missing: true },
])("isolates changed child identity %j and dependent descendants", async (change) => {
const { cdp } = fixture({ after: { same: change } });
const facts = await captureObservationFacts(cdp, 4);
expect(facts.documents.map((doc) => doc.frame.frameId)).toEqual(["main", "remote"]);
expect(facts.issues).toContainEqual(
expect.objectContaining({ frameId: "same", reason: "document-changed" }),
);
});
it.each([
{ after: { main: { element: 99 } } },
{ attachmentChanged: true },
])("rejects a root replacement or reattachment %j", async (options) => {
await expect(captureObservationFacts(fixture(options).cdp, 4)).rejects.toThrow(
"document identity",
);
});
it.each([
"main",
"same",
])("does not publish a previously identified %s document after identity reads fail", async (frameId) => {
const { cdp } = fixture({ after: { [frameId]: { unreadable: true } } });
if (frameId === "main")
await expect(captureObservationFacts(cdp, 4)).rejects.toThrow("document identity");
else {
const facts = await captureObservationFacts(cdp, 4);
expect(facts.documents.map((doc) => doc.frame.frameId)).toEqual(["main", "remote"]);
expect(facts.issues).toContainEqual(
expect.objectContaining({ frameId, reason: "identity-unavailable" }),
);
}
});
it("validates newly discovered snapshot documents without a pre-snapshot identity read", async () => {
const { cdp, logs } = fixture();
const graph = await cdp.getFrameGraph!(4);
cdp.getFrameGraph = async () => ({ ...graph, frames: graph.frames.slice(0, 1) });
const facts = await captureObservationFacts(cdp, 4);
expect(facts.documents.map((doc) => doc.frame.frameId)).toEqual(["main", "same", "nested"]);
expect(facts.documents.every((doc) => doc.identity)).toBe(true);
expect(logs.filter((call) => call.method === "Page.createIsolatedWorld")).toHaveLength(3);
expect(logs.findIndex((call) => call.method === "Page.createIsolatedWorld")).toBeGreaterThan(
logs.findIndex((call) => call.method === "Accessibility.getFullAXTree"),
);
});
it("rejects a snapshot root that no longer matches the current frame root", async () => {
const { cdp } = fixture();
const original = cdp.sendToTarget!;
cdp.sendToTarget = async (target, method, params) => {
const result = await original(target, method, params);
if (method === "DOMSnapshot.captureSnapshot" && !target.sessionId) {
const snapshot = result as {
documents: { frameId: string; nodes: { backendNodeId: number[] } }[];
};
snapshot.documents[0].nodes.backendNodeId[2] = 999;
}
return result as never;
};
cdp.send = (tabId, method, params) => cdp.sendToTarget!({ tabId }, method, params);
await expect(captureObservationFacts(cdp, 4)).rejects.toThrow("document identity");
});
it("starts after-identity reads only after every target finishes and propagates cancellation during identity cleanup", async () => {
const { cdp, logs } = fixture();
const controller = new AbortController();
const original = cdp.sendToTarget!;
let identityCleanup = 0;
cdp.sendToTarget = async (target, method, params) => {
if (method === "Page.createIsolatedWorld") {
expect(logs.filter((call) => call.method === "Accessibility.getFullAXTree")).toHaveLength(
4,
);
}
const result = await original(target, method, params);
if (
method === "Runtime.releaseObjectGroup" &&
String((params as { objectGroup?: string })?.objectGroup).startsWith(
"bsk-document-identity-",
) &&
target.sessionId === "remote"
) {
if (++identityCleanup === 1) controller.abort();
}
return result as never;
};
cdp.send = (tabId, method, params) => cdp.sendToTarget!({ tabId }, method, params);
await expect(captureObservationFacts(cdp, 4, controller.signal)).rejects.toMatchObject({
name: "AbortError",
});
expect(identityCleanup).toBe(1);
});
it.each([
1, 10, 100,
])("bounds identity reads for %i same-target documents after snapshot and AX collection", async (count) => {
const frames: CdpFrame[] = Array.from({ length: count }, (_, i) => ({
frameId: `frame-${i}`,
target: { tabId: 4 },
...(i ? { parentFrameId: "frame-0", ownerBackendNodeId: 90000 + i } : {}),
}));
const { cdp, logs } = fixture({ frames });
const original = cdp.sendToTarget!;
let active = 0;
let peak = 0;
let released = 0;
cdp.sendToTarget = async (target, method, params) => {
if (method === "Page.createIsolatedWorld") {
peak = Math.max(peak, ++active);
await new Promise((resolve) => setTimeout(resolve, 1));
}
if (method === "DOMSnapshot.captureSnapshot") {
expect(active).toBe(0);
expect(released).toBe(0);
}
if (method === "Page.createIsolatedWorld") {
expect(logs.filter((call) => call.method === "Accessibility.getFullAXTree")).toHaveLength(
count,
);
}
const result = await original(target, method, params);
if (
method === "Runtime.releaseObjectGroup" &&
String((params as { objectGroup?: string })?.objectGroup).startsWith(
"bsk-document-identity-",
)
) {
active--;
released++;
}
return result as never;
};
cdp.send = (tabId, method, params) => cdp.sendToTarget!({ tabId }, method, params);
const facts = await captureObservationFacts(cdp, 4);
expect(facts.documents.every((doc) => doc.identity)).toBe(true);
expect(peak).toBe(Math.min(count, 4));
expect(active).toBe(0);
expect(released).toBe(count);
for (const method of [
"Page.createIsolatedWorld",
"Runtime.evaluate",
"Runtime.releaseObjectGroup",
])
expect(logs.filter((call) => call.method === method)).toHaveLength(count);
expect(logs.filter((call) => call.method === "DOMSnapshot.captureSnapshot")).toHaveLength(1);
expect(logs.filter((call) => call.method === "Page.getFrameTree")).toHaveLength(0);
});
it("joins identity cleanup on cancellation without starting queued frames", async () => {
const frames: CdpFrame[] = Array.from({ length: 5 }, (_, i) => ({
frameId: `frame-${i}`,
target: { tabId: 4 },
...(i ? { parentFrameId: "frame-0", ownerBackendNodeId: 90000 + i } : {}),
}));
const { cdp, logs } = fixture({ frames });
const original = cdp.sendToTarget!;
const controller = new AbortController();
let release!: () => void;
const gate = new Promise<void>((resolve) => {
release = resolve;
});
let ready!: () => void;
const blocked = new Promise<void>((resolve) => {
ready = resolve;
});
let held = 0;
cdp.sendToTarget = async (target, method, params) => {
const result = await original(target, method, params);
if (
method === "Runtime.releaseObjectGroup" &&
String((params as { objectGroup?: string })?.objectGroup).startsWith(
"bsk-document-identity-",
)
) {
if (++held === 4) ready();
await gate;
}
return result as never;
};
cdp.send = (tabId, method, params) => cdp.sendToTarget!({ tabId }, method, params);
let settled = false;
const pending = captureObservationFacts(cdp, 4, controller.signal).finally(() => {
settled = true;
});
const rejected = expect(pending).rejects.toMatchObject({ name: "AbortError" });
await blocked;
controller.abort();
await Promise.resolve();
expect(settled).toBe(false);
release();
await rejected;
expect(held).toBe(4);
expect(logs.filter((call) => call.method === "Page.createIsolatedWorld")).toHaveLength(4);
});
it.each([
[{ after: { same: { element: 99 } } }, "omitted", false],
[{ after: { same: { unreadable: true } } }, "omitted", false],
[{ missingIdentity: true }, "document identity unverified", true],
[{}, "", true],
] as const)("publishes identity integrity without silently losing content %j", async (options, notice, retained) => {
const { cdp } = fixture(options);
const result = await captureVomObservation(cdp, 4, "https://example.com");
expect(result.frames.some((frame) => frame.frameId === "same")).toBe(retained);
if (notice) expect(result.text).toContain(notice);
else expect(result.text).not.toContain("identity");
if (!retained) expect(result.text).not.toContain("document identity unverified");
});
it("does not retry failed or missing documents and retains valid AX-only semantics", async () => {
const { cdp, logs } = fixture({
fail: "remote:DOMSnapshot.captureSnapshot",
@@ -151,6 +480,12 @@ describe("captureObservationFacts", () => {
);
});
it("keeps missing identity explicit without manufacturing a verified document", async () => {
const facts = await captureObservationFacts(fixture({ missingIdentity: true }).cdp, 4);
expect(facts.documents.every((doc) => !doc.identity)).toBe(true);
expect(facts.issues.filter((issue) => issue.reason === "identity-unverified")).toHaveLength(4);
});
it("excludes overlay AX in its own document without excluding equal IDs elsewhere", async () => {
const facts = await captureObservationFacts(fixture({ overlay: "remote" }).cdp, 4);
const { documents } = semanticCapture(facts);
@@ -239,9 +574,7 @@ describe("captureObservationFacts", () => {
await vi.waitFor(() => {
if (mode === "worker-abort")
expect(
calls.some(
(call) => call.target === "s3" && call.method === "Accessibility.getFullAXTree",
),
calls.some((call) => call.target === "s3" && call.method === "Accessibility.enable"),
).toBe(true);
else
expect(
@@ -308,6 +641,8 @@ function childSnapshot(frameId: string, backendNodeId: number) {
strings,
documents: [
{
scrollOffsetX: 0,
scrollOffsetY: 0,
frameId,
nodes: {
parentIndex: [-1, 0],
@@ -335,8 +670,13 @@ function childSnapshot(frameId: string, backendNodeId: number) {
describe("OOPIF capture", () => {
it("captures and positions multiple OOPIF documents missing from the root snapshot", async () => {
const sendToTarget = vi.fn(async (target, method) => {
if (method === "Runtime.evaluate") return { result: { value: { width: 300, height: 200 } } };
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 300, clientHeight: 200, pageX: 0, pageY: 0 } };
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000, scale: 1 },
cssLayoutViewport: { clientWidth: 300, clientHeight: 200, pageX: 0, pageY: 0 },
};
}
if (method === "DOMSnapshot.enable" || method === "Accessibility.enable") return {};
if (method === "DOMSnapshot.captureSnapshot") {
@@ -358,7 +698,11 @@ describe("OOPIF capture", () => {
return { model: { content: [x, 100, x + 300, 100, x + 300, 300, x, 300] } };
}
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } };
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000, scale: 1 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 },
};
}
throw new Error(`unexpected root ${method}`);
}) as CdpRunner["send"],
@@ -436,7 +780,11 @@ describe("OOPIF capture", () => {
};
const reply = async (_target: unknown, method: string) => {
if (method === "Page.getLayoutMetrics")
return { cssLayoutViewport: { clientWidth: 300, clientHeight: 200 } };
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000, scale: 1 },
cssLayoutViewport: { clientWidth: 300, clientHeight: 200 },
};
if (method === "DOMSnapshot.captureSnapshot")
return typeof _target === "number" ? childSnapshot("main", 11) : snapshot;
if (method === "DOM.getBoxModel")
@@ -494,7 +842,11 @@ describe("OOPIF capture", () => {
if (method === "Accessibility.getFullAXTree") return { nodes: [] };
if (method === "DOM.getBoxModel") throw new Error("owner geometry unavailable");
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } };
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000, scale: 1 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 },
};
}
throw new Error(`unexpected root ${method}`);
}) as CdpRunner["send"],
@@ -537,7 +889,7 @@ describe("OOPIF capture", () => {
expect.objectContaining({ backendDOMNodeId: 101, frameId: "child" }),
]);
expect(childDocument?.domNodes.find((node) => node.backendNodeId === 101)).toEqual(
expect.objectContaining({ rect: null, localRect: { x: 10, y: 20, w: 100, h: 40 } }),
expect.objectContaining({ rect: null, localRect: null, rendered: true }),
);
});
});
@@ -547,6 +899,8 @@ function siblingCaptureFixture(
beforeReply: (method: string, params: Record<string, unknown>) => Promise<void> = async () => {},
) {
const document = (id: number, owners: number[], childIndexes: number[]) => ({
scrollOffsetX: 0,
scrollOffsetY: 0,
frameId: `frame-${id}`,
nodes: {
parentIndex: [-1, ...owners.map(() => 0)],
@@ -578,7 +932,11 @@ function siblingCaptureFixture(
await beforeReply(method, args);
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
if (method === "Page.getLayoutMetrics")
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } };
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000, scale: 1 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 },
};
if (method === "DOM.getBoxModel")
return { model: { content: [0, 0, 200, 0, 200, 100, 0, 100] } };
if (method === "DOM.resolveNode") return { object: { objectId: String(args.backendNodeId) } };
@@ -782,7 +1140,11 @@ describe("snapshot document provenance", () => {
(method === "DOMSnapshot.captureSnapshot"
? { ...f.snapshot, documents: [f.snapshot.documents[1]] }
: method === "Page.getLayoutMetrics"
? { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } }
? {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000, scale: 1 },
cssLayoutViewport: { clientWidth: 200, clientHeight: 100 },
}
: {}) as T,
};
const facts = await captureObservationFacts(cdp, 4);
@@ -798,6 +1160,44 @@ describe("snapshot document provenance", () => {
});
describe("sibling frame measurement scheduling", () => {
it("batches same-target owner sizes without changing projected results", async () => {
const fixture = siblingCaptureFixture();
const original = fixture.cdp.send;
const methods: string[] = [];
fixture.cdp.send = async (tabId, method, params) => {
methods.push(method);
if (method === "Runtime.evaluate")
return {
result: {
deepSerializedValue: {
type: "array",
value: [100, 101, 102, 103, 104, 105, 200].map((backendNodeId) => ({
type: "array",
value: [
{ type: "node", value: { backendNodeId } },
{ type: "string", value: JSON.stringify({ width: 200, height: 100 }) },
],
})),
},
},
} as never;
return original(tabId, method, params);
};
const captured = await captureObservationFacts(fixture.cdp, 4);
expect(captured.issues.filter((issue) => issue.stage === "geometry")).toEqual([]);
expect(captured.documents.map((doc) => doc.frame.frameId)).toEqual(
Array.from({ length: 8 }, (_, i) => `frame-${i}`),
);
expect(
captured.documents.find((doc) => doc.frame.frameId === "frame-7")?.domNodes[0].rect,
).toEqual({ x: 0, y: 0, w: 200, h: 100 });
expect(methods.filter((method) => method === "DOM.getBoxModel")).toHaveLength(7);
expect(methods.filter((method) => method === "Runtime.evaluate")).toHaveLength(1);
expect(methods.filter((method) => method === "Runtime.releaseObjectGroup")).toHaveLength(1);
expect(methods).not.toContain("DOM.resolveNode");
expect(methods).not.toContain("Runtime.callFunctionOn");
});
it("bounds concurrent reads, fills free slots and preserves breadth-first output despite reordered replies", async () => {
const pending = new Map<number, () => void>();
const fixture = siblingCaptureFixture(async (method, params) => {
@@ -912,3 +1312,167 @@ describe("sibling frame measurement scheduling", () => {
);
});
});
describe("AX frame scheduling", () => {
it.each([
false,
true,
])("shares four workers across frames and drains cancellation=%s", async (cancel) => {
const f = siblingCaptureFixture();
const original = f.cdp.send;
const pending = new Map<string, () => void>();
let active = 0,
peak = 0;
f.cdp.send = async (tabId, method, params) => {
if (method !== "Accessibility.getFullAXTree") return original(tabId, method, params);
const frameId = (params as { frameId: string }).frameId;
active++;
peak = Math.max(peak, active);
try {
await new Promise<void>((resolve) => pending.set(frameId, resolve));
if (frameId === "frame-2") throw new Error("one AX frame unavailable");
return { nodes: [{ nodeId: frameId, frameId }] } as never;
} finally {
active--;
}
};
const controller = new AbortController();
let settled = false;
const capture = captureObservationFacts(f.cdp, 4, controller.signal);
const outcome = capture.then(
(facts) => ({ facts, error: undefined }),
(error) => ({ facts: undefined, error }),
);
void outcome.then(() => {
settled = true;
});
await vi.waitFor(() => expect(pending.size).toBe(4));
if (cancel) {
controller.abort();
pending.get("frame-3")!();
await new Promise((resolve) => setTimeout(resolve, 0));
expect(settled).toBe(false);
expect(pending.size).toBe(4);
for (const release of pending.values()) release();
expect((await outcome).error).toMatchObject({ name: "AbortError" });
} else {
for (let i = 3; i < 7; i++) {
pending.get(`frame-${i}`)!();
await vi.waitFor(() => expect(pending.has(`frame-${i + 1}`)).toBe(true));
}
for (const release of pending.values()) release();
const { facts, error } = await outcome;
expect(error).toBeUndefined();
expect(facts?.documents.map((doc) => doc.frame.frameId)).toEqual(
Array.from({ length: 8 }, (_, i) => `frame-${i}`),
);
expect(facts?.documents.find((doc) => doc.frame.frameId === "frame-2")?.axNodes).toEqual([]);
expect(facts?.issues).toContainEqual(
expect.objectContaining({ stage: "ax", frameId: "frame-2" }),
);
expect(facts?.documents.find((doc) => doc.frame.frameId === "frame-7")?.axNodes).toHaveLength(
1,
);
}
expect(peak).toBe(4);
expect(active).toBe(0);
});
});
describe("visual facts integration", () => {
it("defaults to base facts across targets and shares semantic behavior with visual capture", async () => {
const base = fixture({ canvas: true });
const visual = fixture({ canvas: true });
const baseFacts = await captureObservationFacts(base.cdp, 4);
const visualFacts = await captureObservationFacts(visual.cdp, 4, undefined, undefined, {
includeVisualFacts: true,
});
expect(baseFacts.visualFactsCollected).toBe(false);
expect(visualFacts.visualFactsCollected).toBe(true);
expect(baseFacts.documents).toHaveLength(4);
expect(visualFacts.documents).toHaveLength(4);
for (const doc of baseFacts.documents) {
expect(doc.geometry).toBeUndefined();
expect(doc.index.ancestryComplete).toBeUndefined();
for (const node of doc.index.nodes.values()) {
expect(node.parentMissing).toBeUndefined();
expect(node.layout?.clientRect).toBeUndefined();
if (node.layout) expect(Object.keys(node.layout.styles)).toEqual([...REQUESTED_STYLES]);
}
}
// This fixture has no child frame projection measurements. Enabling visual
// collection must preserve that missing evidence, not invent geometry.
expect(
visualFacts.documents.find((doc) => doc.frame.frameId === "main")?.geometry,
).toBeDefined();
expect(visualFacts.documents.filter((doc) => doc.geometry)).toHaveLength(1);
for (const doc of visualFacts.documents) {
expect(doc.index.ancestryComplete).toBeDefined();
for (const node of doc.index.nodes.values())
if (node.layout) {
expect(Object.keys(node.layout.styles)).toEqual([...VISUAL_STYLES]);
expect(Object.keys(node.layout.styles).some((key) => key.includes("radius"))).toBe(false);
}
}
for (const [capture, expected] of [
[base, REQUESTED_STYLES],
[visual, VISUAL_STYLES],
] as const) {
const calls = capture.logs.filter((c) => c.method === "DOMSnapshot.captureSnapshot");
expect(calls).toHaveLength(2);
for (const call of calls) expect(call.params.computedStyles).toEqual(expected);
}
expect(base.logs.map((c) => [c.target, c.method])).toEqual(
visual.logs.map((c) => [c.target, c.method]),
);
const scene = (input: typeof baseFacts) =>
buildSemanticVomScene({
documents: semanticCapture(input).documents,
viewport: input.viewport,
rootFrameId: input.rootFrameId,
excludedBackendNodeIds: semanticCapture(input).captured.excludedBackendNodeIds,
});
expect(scene(baseFacts)).toEqual(scene(visualFacts));
const before = base.logs.length;
expect(await discoverVisualCandidates(baseFacts)).toMatchObject({
complete: false,
candidates: [],
issues: [{ reason: "visual-facts-not-collected" }],
});
expect(base.logs).toHaveLength(before);
});
it("passes actual snapshot styles, client units, projection and identity to pure discovery without additional CDP", async () => {
const { cdp, logs } = fixture({
canvas: true,
frames: [{ frameId: "main", target: { tabId: 4 } }],
});
const facts = await captureObservationFacts(cdp, 4, undefined, undefined, {
includeVisualFacts: true,
});
const before = logs.length;
const result = await discoverVisualCandidates(facts);
expect(logs).toHaveLength(before);
expect(logs.filter((call) => call.method === "DOMSnapshot.captureSnapshot")).toHaveLength(1);
expect(result.complete).toBe(true);
expect(result.candidates).toHaveLength(1);
expect(result.candidates[0]).toMatchObject({
document: { frameId: "main", documentElementBackendNodeId: 1 },
region: { crop: { x: 10, y: 20, width: 100, height: 40 } },
});
const deduplicated = await deduplicateVisualCandidates(result);
expect(deduplicated.candidates).toEqual(result.candidates);
expect(deduplicated.candidates[0]).toBe(result.candidates[0]);
expect(deduplicated).toMatchObject({
candidateCount: 1,
deduplicatedCount: 0,
dedupDegraded: false,
});
expect(logs).toHaveLength(before);
expect(facts.documents[0].index.nodes.get(2)?.layout?.clientRect).toEqual([0, 0, 100, 40]);
expect(facts.documents[0].geometry?.pageScale).toBe(1);
const baseline = fixture({ frames: [{ frameId: "main", target: { tabId: 4 } }] });
await captureObservationFacts(baseline.cdp, 4);
expect(logs.map((call) => call.method)).toEqual(baseline.logs.map((call) => call.method));
});
});
@@ -215,7 +215,11 @@ function makeCdp(snapshot: unknown) {
if (method === "DOMSnapshot.enable") return {};
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 } };
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
};
}
if (method === "Runtime.evaluate") {
return {
@@ -277,6 +281,8 @@ describe("snapshot capture and hover", () => {
if (method === "DOMSnapshot.captureSnapshot") return hoverTriggerSnapshotReply();
if (method === "Page.getLayoutMetrics") {
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
};
}
@@ -340,6 +346,8 @@ describe("snapshot capture and hover", () => {
if (method === "DOMSnapshot.captureSnapshot") return twoHoverTriggerSnapshotReply();
if (method === "Page.getLayoutMetrics") {
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
};
}
@@ -388,6 +396,8 @@ describe("snapshot capture and hover", () => {
if (method === "DOMSnapshot.captureSnapshot") return nestedHoverTriggerSnapshotReply();
if (method === "Page.getLayoutMetrics") {
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
};
}
@@ -733,6 +743,8 @@ describe("snapshot capture and hover", () => {
if (method === "DOMSnapshot.captureSnapshot") return fakeSnapshotReply();
if (method === "Page.getLayoutMetrics") {
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 200 },
};
}
@@ -745,7 +757,7 @@ describe("snapshot capture and hover", () => {
expect(div?.rect).toMatchObject({ y: 0, h: 600 });
});
it("keeps CSS snapshot bounds independent of legacy metrics ratio", async () => {
it("normalizes raw snapshot bounds before viewport clipping", async () => {
const S = ["html", "body", "div", "position", "fixed", "static", "pointer-events", "auto"];
const i = (s: string) => S.indexOf(s);
const snapshot = {
@@ -780,6 +792,8 @@ describe("snapshot capture and hover", () => {
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
if (method === "Page.getLayoutMetrics") {
return {
visualViewport: { clientWidth: 2000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
layoutViewport: { clientWidth: 2000, clientHeight: 1600 },
};
@@ -797,8 +811,8 @@ describe("snapshot capture and hover", () => {
expect(nodes.find((n) => n.backendNodeId === 12)?.localRect).toEqual({
x: 0,
y: 0,
w: 2000,
h: 1600,
w: 1000,
h: 800,
});
});
@@ -856,6 +870,8 @@ describe("snapshot capture and hover", () => {
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
if (method === "Page.getLayoutMetrics") {
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
};
}
@@ -946,6 +962,8 @@ describe("snapshot capture and hover", () => {
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
if (method === "Page.getLayoutMetrics") {
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
};
}
@@ -1035,6 +1053,8 @@ describe("snapshot capture and hover", () => {
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
if (method === "Page.getLayoutMetrics") {
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
};
}
@@ -1097,6 +1117,8 @@ describe("snapshot capture and hover", () => {
},
},
{
scrollOffsetX: 0,
scrollOffsetY: 0,
frameId: "child",
nodes: {
parentIndex: [-1, 0, 1],
@@ -1119,6 +1141,8 @@ describe("snapshot capture and hover", () => {
},
},
{
scrollOffsetX: 0,
scrollOffsetY: 0,
frameId: "nested",
nodes: {
parentIndex: [-1, 0],
@@ -1146,6 +1170,8 @@ describe("snapshot capture and hover", () => {
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
if (method === "Page.getLayoutMetrics") {
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
};
}
@@ -1225,6 +1251,8 @@ describe("snapshot capture and hover", () => {
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
if (method === "Page.getLayoutMetrics") {
return {
visualViewport: { clientWidth: 2000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 100 },
layoutViewport: { clientWidth: 2000, clientHeight: 1600 },
};
@@ -1233,7 +1261,7 @@ describe("snapshot capture and hover", () => {
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
};
const { nodes } = semanticCapture(await captureObservationFacts(cdp, 4)).captured;
expect(nodes.find((n) => n.backendNodeId === 12)?.rect?.y).toBe(400 - 100);
expect(nodes.find((n) => n.backendNodeId === 12)?.rect?.y).toBe(400 / 2 - 100);
});
it("collectOverlayExcludedBackendIds walks the pierced overlay host subtree", async () => {
@@ -0,0 +1,151 @@
import { describe, expect, it, vi } from "vitest";
import type { CdpRunner } from "../../shared";
import { verifyDocumentIdentity, verifyVisualTargetIdentity } from "../document-identity";
import type { VisualCandidate } from "../visual-discovery";
const candidate: VisualCandidate = {
document: {
attachmentId: "attached",
target: { tabId: 4, sessionId: "child-session" },
frameId: "child-frame",
documentElementBackendNodeId: 10,
},
backendNodeId: 20,
parentBackendNodeId: 10,
region: {
status: "available",
borderBox: { x: 0, y: 0, width: 10, height: 10 },
crop: { x: 0, y: 0, width: 10, height: 10 },
},
};
function fixture(
options: {
connected?: boolean;
sameDocument?: boolean;
root?: number;
missing?: boolean;
fail?: string;
abortAt?: string;
detachAt?: string;
} = {},
) {
const controller = new AbortController();
let attachment = "attached";
const send = vi.fn(
async (_tab: number, _session: string, method: string, params: Record<string, unknown>) => {
if (options.abortAt === method) controller.abort();
if (options.detachAt === method) attachment = "new-attachment";
if (options.fail === method) throw new Error("CDP failed");
if (method === "Page.createIsolatedWorld") return { executionContextId: 7 };
if (method === "DOM.resolveNode")
return options.missing ? {} : { object: { objectId: "anchor" } };
if (method === "Runtime.evaluate" || method === "Runtime.callFunctionOn") {
let root: unknown = {};
if (method === "Runtime.callFunctionOn") {
const document = { documentElement: {} };
// Exercise the actual predicate, including connection and owner checks.
root = new Function("document", `return (${params.functionDeclaration}).call(this)`).call(
{
isConnected: options.connected ?? true,
ownerDocument: options.sameDocument === false ? {} : document,
},
document,
);
}
return {
result: {
deepSerializedValue:
root === null
? { type: "null" }
: { type: "node", value: { backendNodeId: options.root ?? 10 } },
},
};
}
if (method === "Runtime.releaseObjectGroup") return {};
throw new Error(`unexpected ${method}`);
},
);
const cdp = {
send: vi.fn(),
sendToTarget: (
target: { tabId: number; sessionId?: string },
method: string,
params: Record<string, unknown>,
) => send(target.tabId, target.sessionId!, method, params),
getAttachmentId: () => attachment,
} as unknown as CdpRunner;
return { cdp, send, controller };
}
describe("visual target identity", () => {
it("uses the exact frame and target with bounded local reads and releases objects", async () => {
const { cdp, send } = fixture();
expect(await verifyVisualTargetIdentity(cdp, candidate)).toBe("current");
expect(send.mock.calls.map((c) => c[2])).toEqual([
"Page.createIsolatedWorld",
"DOM.resolveNode",
"Runtime.callFunctionOn",
"Runtime.releaseObjectGroup",
]);
expect(
send.mock.calls.every(([tab, session]) => tab === 4 && session === "child-session"),
).toBe(true);
expect(send.mock.calls[0][3]).toMatchObject({ frameId: "child-frame" });
expect(send.mock.calls[1][3]).toMatchObject({ backendNodeId: 20, executionContextId: 7 });
expect(send.mock.calls[2][3]).toMatchObject({ objectId: "anchor" });
expect(send.mock.calls[3][3].objectGroup).toBe(send.mock.calls[1][3].objectGroup);
});
it.each([
{ connected: false },
{ sameDocument: false },
{ root: 11 },
])("rejects detached/adopted/replaced DOM: %j", async (options) => {
const { cdp } = fixture(options);
expect(await verifyVisualTargetIdentity(cdp, candidate)).toBe("changed");
});
it("does not add anchor reads to existing capture identity verification", async () => {
const { cdp, send } = fixture();
expect(await verifyDocumentIdentity(cdp, candidate.document)).toBe("current");
expect(send.mock.calls.map((c) => c[2])).toEqual([
"Page.createIsolatedWorld",
"Runtime.evaluate",
"Runtime.releaseObjectGroup",
]);
});
it.each([
"Page.createIsolatedWorld",
"DOM.resolveNode",
"Runtime.callFunctionOn",
])("fails closed and cleans up when %s fails", async (fail) => {
const { cdp, send } = fixture({ fail });
expect(await verifyVisualTargetIdentity(cdp, candidate)).toBe("unavailable");
expect(send.mock.calls.at(-1)?.[2]).toBe("Runtime.releaseObjectGroup");
});
it("rejects attachment changes before and during verification", async () => {
const first = fixture();
expect(
await verifyVisualTargetIdentity({ ...first.cdp, getAttachmentId: () => "other" }, candidate),
).toBe("changed");
expect(first.send).not.toHaveBeenCalled();
const second = fixture({ detachAt: "Runtime.callFunctionOn" });
expect(await verifyVisualTargetIdentity(second.cdp, candidate)).toBe("changed");
const missing = fixture({ missing: true });
expect(await verifyVisualTargetIdentity(missing.cdp, candidate)).toBe("unavailable");
});
it.each([
"DOM.resolveNode",
"Runtime.callFunctionOn",
])("propagates cancellation at %s after releasing the object group", async (abortAt) => {
const { cdp, send, controller } = fixture({ abortAt });
await expect(
verifyVisualTargetIdentity(cdp, candidate, controller.signal),
).rejects.toMatchObject({ name: "AbortError" });
expect(send.mock.calls.at(-1)?.[2]).toBe("Runtime.releaseObjectGroup");
});
});
@@ -1,7 +1,8 @@
import { describe, expect, it } from "vitest";
import { describe, expect, it, vi } from "vitest";
import { OVERLAY_HOST_MARKER_ATTR } from "@/lib/overlay-bridge";
import { createCaptureCheckpoint } from "../capture-abort";
import { buildDocumentIndex, type DecodedNode } from "../facts";
import { decodeDocument, REQUESTED_STYLES } from "../snapshot";
import { decodeDocument, REQUESTED_STYLES, VISUAL_SNAPSHOT } from "../snapshot";
function node(id: number, parent: number | null): DecodedNode {
return {
@@ -76,7 +77,7 @@ describe("document facts", () => {
);
expect(nodes[0].attrs).toEqual({ "aria-hidden": "true", inert: "" });
expect(nodes[0].layout).toEqual({
boundsSpace: "snapshot-document-css",
boundsSpace: "snapshot-document-layout",
bounds: [10, 20, 120, 40],
styles: {
position: "static",
@@ -90,7 +91,23 @@ describe("document facts", () => {
expect(nodes[0]).not.toHaveProperty("excluded");
});
it("yields during large inputs so cancellation interrupts indexing", async () => {
it("keeps short blocks synchronous and checks cancellation after yielding", async () => {
const clock = vi.spyOn(performance, "now").mockReturnValue(0);
try {
const controller = new AbortController();
const checkpoint = createCaptureCheckpoint(controller.signal);
for (let i = 0; i < 20; i++) expect(checkpoint()).toBeUndefined();
clock.mockReturnValue(9);
const pending = checkpoint();
expect(pending).toBeInstanceOf(Promise);
controller.abort();
await expect(pending).rejects.toMatchObject({ name: "AbortError" });
} finally {
clock.mockRestore();
}
});
it("checks cancellation at bounded indexing blocks", async () => {
const controller = new AbortController();
const input = Array.from({ length: 100_000 }, (_, i) => node(i, i ? i - 1 : null));
let reads = 0;
@@ -108,3 +125,44 @@ describe("document facts", () => {
expect(reads).toBeLessThanOrEqual(356);
});
});
describe("visual ancestry facts", () => {
it("distinguishes document roots, malformed roots, missing parents and cycles", async () => {
const input = [
{ ...node(0, null), nodeType: 9 },
{ ...node(1, 0), nodeType: 1 },
{ ...node(2, 1), nodeType: 11 },
node(3, 2),
node(4, 99),
node(5, 6),
node(6, 5),
node(7, 5),
{ ...node(8, null), nodeType: 1 },
{ ...node(9, null), nodeType: 9, parentMissing: true },
];
const index = await buildDocumentIndex(input.reverse(), undefined, true);
for (const id of [0, 1, 2, 3]) expect(index.ancestryComplete!.get(id)).toBe(true);
for (const id of [4, 5, 6, 7, 8, 9]) expect(index.ancestryComplete!.get(id)).toBe(false);
});
it("retains client offsets in their source units and does not disguise missing parent indices", async () => {
const decoded = await decodeDocument(
{
nodes: { backendNodeId: [1, 2], nodeType: [1, 1], nodeName: [0, 0], parentIndex: [99] },
layout: {
nodeIndex: [0],
bounds: [[745, 455.625, 97, 59.5]],
clientRects: [[5, 5, 68, 38]],
},
},
["div"],
undefined,
VISUAL_SNAPSHOT,
);
expect(decoded.nodes[0].layout?.clientRect).toEqual([5, 5, 68, 38]);
expect(decoded.nodes[0].layout?.bounds).toEqual([745, 455.625, 97, 59.5]);
expect(decoded.nodes.every((node) => node.parentMissing)).toBe(true);
const index = await buildDocumentIndex(decoded.nodes, undefined, true);
expect([...index.ancestryComplete!.values()]).toEqual([false, false]);
});
});
@@ -1,6 +1,7 @@
import { describe, expect, it, vi } from "vitest";
import { type CdpRunner, cdpRunnerForTarget } from "../../shared";
import { captureObservationFacts } from "../capture-coordinator";
import { enrichFormControlStates } from "../form-capture";
import { decodeDocument } from "../snapshot";
interface Control {
id: number;
@@ -77,21 +78,32 @@ function reply(entries: Array<[number, unknown]>) {
};
}
function fakeCdp(groups: Control[][], evaluate: (params: Record<string, unknown>) => unknown) {
async function fakeCdp(
groups: Control[][],
evaluate: (params: Record<string, unknown>) => unknown,
) {
const snap = snapshot(groups);
const send = vi.fn(async (_tabId: number, method: string, params?: object) => {
if (method === "DOMSnapshot.captureSnapshot") return snap;
if (method === "Runtime.evaluate") return evaluate(params as Record<string, unknown>);
return {};
});
return { send, cdp: { send: send as CdpRunner["send"] } };
const nodes = await decodeControls(snap);
return { send, cdp: { send: send as CdpRunner["send"] }, nodes };
}
async function decodeControls(snap: ReturnType<typeof snapshot>) {
return Promise.all(
snap.documents.map(async (doc) =>
(await decodeDocument(doc, snap.strings)).nodes.map((node) => ({ ...node, rect: null })),
),
);
}
describe("form state identity", () => {
it("matches reordered controls by identity and preserves controls missing from the batch", async () => {
// The snapshot includes a shadow input. The runtime query skips it,
// encounters a new input, and sees the remaining inputs in a new order.
const { cdp } = fakeCdp(
const { cdp, nodes } = await fakeCdp(
[
[
{ id: 10, value: "" },
@@ -106,23 +118,21 @@ describe("form state identity", () => {
[11, state("alice@example.com")],
]),
);
const captured = await captureObservationFacts(cdp, 7);
expect(captured.documents[0].domNodes.find((n) => n.backendNodeId === 10)).toMatchObject({
await enrichFormControlStates(cdp, 7, nodes);
expect(nodes[0].find((n) => n.backendNodeId === 10)).toMatchObject({
formValue: "",
formState: "empty",
formPlaceholder: "snapshot placeholder",
});
expect(captured.documents[0].domNodes.find((n) => n.backendNodeId === 11)).toMatchObject({
expect(nodes[0].find((n) => n.backendNodeId === 11)).toMatchObject({
formValue: "alice@example.com",
});
expect(captured.documents[0].domNodes.find((n) => n.backendNodeId === 12)).toMatchObject({
formValue: "上海",
});
expect(captured.documents[0].domNodes.some((n) => n.backendNodeId === 99)).toBe(false);
expect(nodes[0].find((n) => n.backendNodeId === 12)).toMatchObject({ formValue: "上海" });
expect(nodes[0].some((n) => n.backendNodeId === 99)).toBe(false);
});
it("does not shift state between frames when one frame is absent from the runtime batch", async () => {
const { cdp } = fakeCdp(
const { cdp, nodes } = await fakeCdp(
[
[],
[{ id: 11, value: "inaccessible frame" }],
@@ -135,10 +145,8 @@ describe("form state identity", () => {
[21, state("second frame")],
]),
);
const captured = await captureObservationFacts(cdp, 7);
const controls = captured.documents
.filter((doc) => doc.frame.parentFrameId)
.flatMap((doc) => doc.domNodes);
await enrichFormControlStates(cdp, 7, nodes);
const controls = nodes.slice(1).flat();
expect(controls.find((n) => n.backendNodeId === 11)?.formValue).toBe("inaccessible frame");
expect(controls.find((n) => n.backendNodeId === 21)?.formValue).toBe("second frame");
expect(controls.find((n) => n.backendNodeId === 31)?.formValue).toBe("third frame");
@@ -146,7 +154,6 @@ describe("form state identity", () => {
it("keeps identical backend ids in different OOPIF targets separate", async () => {
const sendToTarget = vi.fn(async (target, method) => {
if (method === "DOMSnapshot.captureSnapshot") return snapshot([[{ id: 11, value: "old" }]]);
if (method === "Runtime.evaluate") return reply([[11, state(target.sessionId)]]);
return {};
});
@@ -155,13 +162,9 @@ describe("form state identity", () => {
});
const cdp = { send, sendToTarget } as unknown as CdpRunner;
for (const sessionId of ["left-frame", "right-frame"]) {
const captured = await captureObservationFacts(
cdpRunnerForTarget(cdp, { tabId: 7, sessionId }),
7,
);
expect(captured.documents[0].domNodes.find((n) => n.backendNodeId === 11)?.formValue).toBe(
sessionId,
);
const nodes = await decodeControls(snapshot([[{ id: 11, value: "old" }]]));
await enrichFormControlStates(cdpRunnerForTarget(cdp, { tabId: 7, sessionId }), 7, nodes);
expect(nodes[0].find((n) => n.backendNodeId === 11)?.formValue).toBe(sessionId);
expect(sendToTarget).toHaveBeenCalledWith(
{ tabId: 7, sessionId },
"Runtime.releaseObjectGroup",
@@ -177,7 +180,7 @@ describe("form state identity", () => {
"malformed",
"invalid-state",
])("preserves snapshot data when enrichment is %s", async (mode) => {
const { cdp, send } = fakeCdp([[{ id: 11, value: "snapshot value" }]], () => {
const { cdp, send, nodes } = await fakeCdp([[{ id: 11, value: "snapshot value" }]], () => {
if (mode === "unsupported") throw new Error("serialization unsupported");
if (mode === "legacy") return { result: { value: { controls: [state("wrong value")] } } };
const result = reply([
@@ -187,8 +190,8 @@ describe("form state identity", () => {
result.result.deepSerializedValue.value[0].value[1].value = "not json";
return result;
});
const captured = await captureObservationFacts(cdp, 7);
expect(captured.documents[0].domNodes.find((n) => n.backendNodeId === 11)).toMatchObject({
expect(await enrichFormControlStates(cdp, 7, nodes)).toBe(mode !== "unsupported");
expect(nodes[0].find((n) => n.backendNodeId === 11)).toMatchObject({
formValue: "snapshot value",
formPlaceholder: "snapshot placeholder",
});
@@ -197,18 +200,34 @@ describe("form state identity", () => {
it("releases the batch's remote objects when capture is cancelled", async () => {
const controller = new AbortController();
const { cdp, send } = fakeCdp([[{ id: 11, value: "old" }]], () => {
const { cdp, send, nodes } = await fakeCdp([[{ id: 11, value: "old" }]], () => {
controller.abort();
return reply([[11, state("new")]]);
});
await expect(captureObservationFacts(cdp, 7, controller.signal)).rejects.toMatchObject({
await expect(enrichFormControlStates(cdp, 7, nodes, controller.signal)).rejects.toMatchObject({
name: "AbortError",
});
expect(send).toHaveBeenCalledWith(7, "Runtime.releaseObjectGroup", expect.anything());
});
it("does not report success when cancelled during object cleanup", async () => {
const controller = new AbortController();
const nodes = await decodeControls(snapshot([[{ id: 11, value: "old" }]]));
const cdp: CdpRunner = {
send: vi.fn(async (_tabId, method) => {
if (method === "Runtime.evaluate") return reply([[11, state("new")]]);
if (method === "Runtime.releaseObjectGroup") controller.abort();
return {};
}) as CdpRunner["send"],
};
await expect(enrichFormControlStates(cdp, 7, nodes, controller.signal)).rejects.toMatchObject({
name: "AbortError",
});
expect(cdp.send).toHaveBeenCalledWith(7, "Runtime.releaseObjectGroup", expect.anything());
});
it("skips entries without node identity while still applying valid entries", async () => {
const { cdp } = fakeCdp(
const { cdp, nodes } = await fakeCdp(
[
[
{ id: 11, value: "snapshot value" },
@@ -225,24 +244,20 @@ describe("form state identity", () => {
return result;
},
);
const captured = await captureObservationFacts(cdp, 7);
expect(captured.documents[0].domNodes.find((n) => n.backendNodeId === 11)?.formValue).toBe(
"snapshot value",
);
expect(captured.documents[0].domNodes.find((n) => n.backendNodeId === 12)?.formValue).toBe(
"live value",
);
await enrichFormControlStates(cdp, 7, nodes);
expect(nodes[0].find((n) => n.backendNodeId === 11)?.formValue).toBe("snapshot value");
expect(nodes[0].find((n) => n.backendNodeId === 12)?.formValue).toBe("live value");
});
it.each([
"password",
"text",
])("redacts a sensitive %s control after identity matching", async (type) => {
const { cdp } = fakeCdp([[{ id: 11, value: "secret", type }]], () =>
const { cdp, nodes } = await fakeCdp([[{ id: 11, value: "secret", type }]], () =>
reply([[11, { ...state("secret"), sensitive: true }]]),
);
const captured = await captureObservationFacts(cdp, 7);
const control = captured.documents[0].domNodes.find((n) => n.backendNodeId === 11);
await enrichFormControlStates(cdp, 7, nodes);
const control = nodes[0].find((n) => n.backendNodeId === 11);
expect(control?.formState).toBe("filled");
expect(control?.formValue).toBeUndefined();
expect(control?.formDefaultValue).toBeUndefined();
@@ -259,7 +274,7 @@ describe("form state identity", () => {
defaultValue: "",
placeholder: "",
}));
const { cdp, send } = fakeCdp([controls], (params) => {
const { cdp, send, nodes } = await fakeCdp([controls], (params) => {
const doc = {
querySelectorAll: (selector: string) =>
selector === "input,textarea,select" ? elements : [],
@@ -270,13 +285,9 @@ describe("form state identity", () => {
expect(entries).toHaveLength(250);
return reply(entries.map(([element, json]) => [element.backendNodeId, JSON.parse(json)]));
});
const captured = await captureObservationFacts(cdp, 7);
expect(captured.documents[0].domNodes.find((n) => n.backendNodeId === 250)?.formValue).toBe(
"live 250",
);
expect(captured.documents[0].domNodes.find((n) => n.backendNodeId === 251)?.formValue).toBe(
"old",
);
await enrichFormControlStates(cdp, 7, nodes);
expect(nodes[0].find((n) => n.backendNodeId === 250)?.formValue).toBe("live 250");
expect(nodes[0].find((n) => n.backendNodeId === 251)?.formValue).toBe("old");
expect(send.mock.calls.filter(([, method]) => method === "Runtime.evaluate")).toHaveLength(1);
expect(
send.mock.calls.filter(([, method]) => method === "Runtime.releaseObjectGroup"),
@@ -0,0 +1,147 @@
import { describe, expect, it } from "vitest";
import { deduplicateVisualCandidates, MAX_VISUAL_DEDUP_KEYS } from "../visual-dedup";
import type { VisualCandidate, VisualDiscoveryResult } from "../visual-discovery";
function candidate(id: number, width = 100, parent = 1): VisualCandidate {
const crop = { x: 0, y: 0, width, height: 100 };
return {
document: {
attachmentId: "a",
target: { tabId: 1 },
frameId: "main",
documentElementBackendNodeId: 1,
},
backendNodeId: id,
parentBackendNodeId: parent,
region: { status: "available", borderBox: crop, crop },
};
}
function discovery(candidates: VisualCandidate[]): VisualDiscoveryResult {
return { candidates, complete: true, issues: [], captureIssues: [] };
}
describe("visual candidate deduplication", () => {
it("preserves small, unnamed and labelled candidates and their original evidence", async () => {
const small = { ...candidate(3, 1), label: ' A "label"\n😀' };
const large = candidate(2, 1000);
const input = discovery([small, large]);
Object.freeze(input.candidates);
Object.freeze(small);
Object.freeze(large);
for (let i = 0; i < 2; i++) {
const result = await deduplicateVisualCandidates(input);
expect(result.candidates).toEqual([small, large]);
expect(result.candidates[0]).toBe(small);
expect(result.candidates[1]).toBe(large);
expect(result).toMatchObject({
candidateCount: 2,
deduplicatedCount: 0,
dedupDegraded: false,
});
}
});
it("only deduplicates exact crops within the same full DOM identity and direct parent", async () => {
const base = candidate(20);
const variants = [
candidate(30, 100, 2),
{ ...candidate(40), parentBackendNodeId: null },
...["x", "y", "width", "height"].map((axis) => ({
...base,
region: {
...base.region,
crop: {
...base.region.crop,
[axis]: base.region.crop[axis as keyof typeof base.region.crop] + 0.001,
},
},
})),
...[
{ frameId: "child" },
{ target: { tabId: 2 } },
{ target: { tabId: 1, sessionId: "remote" } },
{ documentElementBackendNodeId: 9 },
{ attachmentId: "b" },
].map((identity) => ({ ...base, document: { ...base.document, ...identity } })),
];
const duplicate = { ...candidate(10), label: "Not a new target" };
const result = await deduplicateVisualCandidates(discovery([base, ...variants, duplicate]));
expect(result.candidates).toEqual([base, ...variants]);
expect(result.candidates[0]).toBe(base);
expect(result.deduplicatedCount).toBe(1);
});
it.each([
50_001, 100_000,
])("keeps all %s distinct candidates including the last tiny Canvas", async (count) => {
const input = Array.from({ length: count }, (_, i) => candidate(i + 1, 100, i + 1));
input[count - 1] = candidate(count, 1, count);
// A remembered key still deduplicates; an unremembered key passes through.
input.push({ ...input[0] }, { ...input[count - 1] });
const result = await deduplicateVisualCandidates(discovery(input));
expect(result.candidates).toEqual([...input.slice(0, count), input[count + 1]]);
expect(result.candidates[count - 1]).toBe(input[count - 1]);
expect(result).toMatchObject({
candidateCount: count + 2,
deduplicatedCount: 1,
dedupDegraded: true,
});
});
it("does not report degradation when every key fits", async () => {
const input = Array.from({ length: MAX_VISUAL_DEDUP_KEYS }, (_, i) => candidate(i, 1, i));
input.push({ ...input[0] });
const result = await deduplicateVisualCandidates(discovery(input));
expect(result.candidates).toHaveLength(MAX_VISUAL_DEDUP_KEYS);
expect(result.dedupDegraded).toBe(false);
expect(result.deduplicatedCount).toBe(1);
});
it("preserves upstream unknowns without counting issues as Canvas nodes", async () => {
const input: VisualDiscoveryResult = {
...discovery([candidate(2)]),
complete: false,
issues: [{ reason: "geometry-unavailable", target: { tabId: 1 }, frameId: "missing" }],
captureIssues: [{ stage: "ax", reason: "capture-unavailable", target: { tabId: 1 } }],
};
const result = await deduplicateVisualCandidates(input);
expect(result.complete).toBe(false);
expect(result.candidateCount).toBe(1);
expect(result.issues).toBe(input.issues);
expect(result.captureIssues).toBe(input.captureIssues);
const absent = await deduplicateVisualCandidates({
...input,
candidates: [],
issues: [{ reason: "visual-facts-not-collected" }],
});
expect(absent).toMatchObject({
complete: false,
candidateCount: 0,
issues: [{ reason: "visual-facts-not-collected" }],
});
expect(await deduplicateVisualCandidates(discovery([]))).toMatchObject({
complete: true,
candidates: [],
candidateCount: 0,
dedupDegraded: false,
});
});
it("drains cancellation through the shared checkpoint", async () => {
const controller = new AbortController();
controller.abort();
await expect(
deduplicateVisualCandidates(discovery([]), controller.signal),
).rejects.toMatchObject({ name: "AbortError" });
const during = new AbortController();
const input = discovery(Array.from({ length: 100_000 }, (_, i) => candidate(i, 100, i)));
const timer = setTimeout(() => during.abort(), 0);
try {
await expect(deduplicateVisualCandidates(input, during.signal)).rejects.toMatchObject({
name: "AbortError",
});
} finally {
clearTimeout(timer);
}
});
});
@@ -0,0 +1,502 @@
import { describe, expect, it } from "vitest";
import type { CdpFrame } from "@/browser-driver/frame-graph";
import { OVERLAY_HOST_MARKER_ATTR } from "@/lib/overlay-bridge";
import type { GeometryProjection } from "../../geometry";
import { buildDocumentIndex, type DocumentFacts, type ObservationFacts } from "../facts";
import type { FrameOwnedAxNode } from "../frame-document";
import { normalizeDocument } from "../normalize";
import { VISUAL_SNAPSHOT, VISUAL_STYLES } from "../snapshot";
import { discoverVisualCandidates } from "../visual-discovery";
const defaults: Record<string, string> = {
position: "static",
"pointer-events": "auto",
cursor: "auto",
visibility: "visible",
opacity: "1",
display: "block",
"overflow-x": "visible",
"overflow-y": "visible",
transform: "none",
zoom: "1",
"clip-path": "none",
"mask-image": "none",
rotate: "none",
scale: "none",
perspective: "none",
clip: "auto",
contain: "none",
"overflow-clip-margin": "0px",
};
const projection: GeometryProjection = {
sourceClips: [],
edges: [],
topViewport: { width: 1000, height: 800 },
};
interface Spec {
id: number;
parent?: number;
tag?: string;
styles?: Record<string, string>;
bounds?: number[] | null;
client?: number[];
attrs?: Record<string, string>;
}
async function document(
nodes: Spec[],
options: {
frame?: CdpFrame;
projection?: GeometryProjection;
pageScale?: number;
scrollY?: number;
layoutScale?: number;
} = {},
): Promise<DocumentFacts<FrameOwnedAxNode>> {
const frame = options.frame ?? { frameId: "main", target: { tabId: 1 } };
const strings: string[] = [];
const str = (value: string) => {
const index = strings.indexOf(value);
if (index >= 0) return index;
strings.push(value);
return strings.length - 1;
};
const input: Spec[] = [
{ id: 0, tag: "#document", bounds: null },
{ id: 1, parent: 0, tag: "html", bounds: [0, 0, 1000, 800] },
...nodes,
];
const layouts = input.filter((node) => node.bounds !== null);
const normalized = await normalizeDocument(
{
nodes: {
backendNodeId: input.map((n) => n.id),
nodeType: input.map((n) =>
n.tag === "#document" ? 9 : n.tag === "#document-fragment" ? 11 : 1,
),
parentIndex: input.map((n) =>
n.parent === undefined ? -1 : input.findIndex((p) => p.id === n.parent),
),
nodeName: input.map((n) => str(n.tag ?? "canvas")),
attributes: input.map((n) =>
Object.entries(n.attrs ?? {}).flatMap(([k, v]) => [str(k), str(v)]),
),
},
layout: {
nodeIndex: layouts.map((n) => input.indexOf(n)),
bounds: layouts.map((n) => n.bounds ?? [10, 20, 120, 40]),
clientRects: layouts.map((n) => n.client ?? [0, 0, 1000, 800]),
styles: layouts.map((n) =>
VISUAL_STYLES.map((key) => str({ ...defaults, ...n.styles }[key])),
),
},
},
strings,
{
frameId: frame.frameId,
ownerFrameBackendNodeId: frame.ownerBackendNodeId ?? null,
target: frame.target,
projection: {
status: "available",
projection: {
source: { target: frame.target, frameId: frame.frameId },
geometry: options.projection ?? projection,
},
},
coordinates: {
layoutUnitsPerCssPixel: options.layoutScale ?? 1,
scrollCss: { x: 0, y: options.scrollY ?? 0 },
},
pageScale: options.pageScale ?? 1,
},
undefined,
VISUAL_SNAPSHOT,
);
return {
frame,
identity: {
attachmentId: "attached",
target: frame.target,
frameId: frame.frameId,
documentElementBackendNodeId: 1,
},
index: normalized.index,
geometry: normalized.geometry,
domNodes: normalized.nodes,
axNodes: [],
};
}
function facts(
documents: DocumentFacts<FrameOwnedAxNode>[],
issues: ObservationFacts<FrameOwnedAxNode>["issues"] = [],
): ObservationFacts<FrameOwnedAxNode> {
return {
visualFactsCollected: true,
rootFrameId: "main",
documents,
viewport: { width: 1000, height: 800 },
issues,
startedAt: 1,
finishedAt: 2,
};
}
describe("Canvas discovery", () => {
it("distinguishes visual facts not collected from complete empty discovery and honors cancellation", async () => {
const empty = facts([]);
expect(await discoverVisualCandidates(empty)).toMatchObject({
complete: true,
candidates: [],
issues: [],
});
expect(await discoverVisualCandidates({ ...empty, visualFactsCollected: false })).toMatchObject(
{ complete: false, candidates: [], issues: [{ reason: "visual-facts-not-collected" }] },
);
const controller = new AbortController();
controller.abort();
await expect(
discoverVisualCandidates({ ...empty, visualFactsCollected: false }, controller.signal),
).rejects.toMatchObject({ name: "AbortError" });
});
it("discovers Canvas and frame contents without corner radius facts", async () => {
const own = await document([{ id: 2, parent: 1, styles: {} }]);
expect(await discoverVisualCandidates(facts([own]))).toMatchObject({
candidates: [{ backendNodeId: 2 }],
complete: true,
issues: [],
});
const parent = await document([{ id: 2, parent: 1, tag: "iframe", styles: {} }]);
const child = await document([{ id: 3, parent: 1 }], {
frame: {
frameId: "child",
parentFrameId: "main",
ownerBackendNodeId: 2,
target: { tabId: 1 },
},
});
expect(await discoverVisualCandidates(facts([parent, child]))).toMatchObject({
candidates: [{ backendNodeId: 3 }],
complete: true,
issues: [],
});
});
it.each([
0.8, 1, 1.25, 2,
])("combines layout-unit bounds with CSS client clips at scale %s", async (scale) => {
const doc = await document(
[
{
id: 2,
parent: 1,
tag: "div",
bounds: [100, 200, 64, 34].map((n) => n * scale),
client: [2, 2, 60, 30],
styles: { "overflow-x": "hidden", "overflow-y": "hidden" },
},
{ id: 3, parent: 2, bounds: [102, 202, 120, 40].map((n) => n * scale) },
],
{ layoutScale: scale, scrollY: 75 },
);
const result = await discoverVisualCandidates(facts([doc]));
expect(result.issues).toEqual([]);
const expected = {
borderBox: { x: 102, y: 127, width: 120, height: 40 },
crop: { x: 102, y: 127, width: 60, height: 30 },
};
for (const box of ["borderBox", "crop"] as const)
for (const key of ["x", "y", "width", "height"] as const)
expect(result.candidates[0].region[box][key]).toBeCloseTo(expected[box][key], 8);
});
it("normalizes a zero-width ancestor origin before one-axis clipping", async () => {
const doc = await document(
[
{
id: 2,
parent: 1,
tag: "div",
bounds: [200, 400, 0, 60],
client: [0, 0, 0, 30],
styles: { "overflow-y": "hidden" },
},
{ id: 3, parent: 2, bounds: [200, 400, 240, 80] },
],
{ layoutScale: 2, scrollY: 75 },
);
const result = await discoverVisualCandidates(facts([doc]));
expect(result.issues).toEqual([]);
expect(result.candidates[0].region.crop).toEqual({ x: 100, y: 125, width: 120, height: 30 });
});
it("keeps unnamed, semantic, aria-hidden, inert, pointer-none and visibility override Canvas", async () => {
const doc = await document([
{ id: 2, parent: 1 },
{ id: 3, parent: 1, attrs: { "aria-label": "Chart" } },
{
id: 4,
parent: 1,
attrs: { "aria-hidden": "true", inert: "" },
styles: { "pointer-events": "none" },
},
{ id: 5, parent: 1, tag: "div", styles: { visibility: "hidden" } },
{ id: 6, parent: 5, styles: { visibility: "visible" } },
{ id: 7, parent: 1, attrs: { hidden: "" }, styles: { display: "block" } },
]);
doc.axNodes.push({ nodeId: "canvas", backendDOMNodeId: 2 } as FrameOwnedAxNode);
const result = await discoverVisualCandidates(facts([doc]));
expect(result.candidates.map((c) => c.backendNodeId)).toEqual([2, 3, 4, 6, 7]);
expect(result.candidates[1].label).toBe("Chart");
expect(result.complete).toBe(true);
});
it("excludes CSS transparent subtrees, hidden/zero/no-layout/fully clipped nodes and overlays", async () => {
const doc = await document([
{ id: 2, parent: 1, tag: "div", styles: { opacity: "0" } },
{ id: 3, parent: 2 },
{ id: 4, parent: 1, styles: { visibility: "hidden" } },
{ id: 5, parent: 1, bounds: [0, 0, 0, 40] },
{ id: 6, parent: 1, bounds: null },
{ id: 7, parent: 1, bounds: [1200, 0, 100, 50] },
{ id: 8, parent: 1, tag: "div", attrs: { [OVERLAY_HOST_MARKER_ATTR]: "" } },
{ id: 9, parent: 8, tag: "#document-fragment", bounds: null },
{ id: 10, parent: 9 },
]);
expect(await discoverVisualCandidates(facts([doc]))).toMatchObject({
candidates: [],
issues: [],
complete: true,
});
});
it.each([
["both", "hidden", "hidden", [0, 0, 60, 30], [60, 30]],
["x", "clip", "visible", [0, 0, 60, 20], [60, 40]],
["border", "hidden", "hidden", [5, 5, 68, 38], [64, 34]],
])("uses client box for %s clipping", async (_name, x, y, client, expected) => {
const doc = await document([
{
id: 2,
parent: 1,
tag: "div",
bounds: [0, 0, 78, 48],
client: client as number[],
styles: { "overflow-x": x as string, "overflow-y": y as string },
},
{
id: 3,
parent: 2,
bounds: x === "clip" || _name === "both" ? [0, 0, 120, 40] : [9, 9, 120, 40],
},
]);
const [candidate] = (await discoverVisualCandidates(facts([doc]))).candidates;
expect([candidate.region.crop.width, candidate.region.crop.height]).toEqual(expected);
expect(candidate.region.clips?.backendNodeId).toBe(2);
});
it("keeps Canvas border-box and clips partial viewport intersections", async () => {
const doc = await document([{ id: 2, parent: 1, bounds: [950, 20, 130, 50] }]);
const [candidate] = (await discoverVisualCandidates(facts([doc]))).candidates;
expect(candidate.region.borderBox.width).toBe(130);
expect(candidate.region.crop).toEqual({ x: 950, y: 20, width: 50, height: 50 });
});
it("propagates owner ancestry across targets, without depending on semantic parents", async () => {
const parent = await document([
{
id: 2,
parent: 1,
tag: "div",
bounds: [0, 0, 200, 100],
client: [0, 0, 200, 100],
styles: { "overflow-x": "hidden", "overflow-y": "hidden" },
},
{ id: 3, parent: 2, tag: "iframe", bounds: [100, 0, 200, 200] },
]);
parent.domNodes.splice(0); // Visual discovery never reads the semantic subset.
const child = await document([{ id: 2, parent: 1, bounds: [0, 0, 200, 200] }], {
frame: {
frameId: "child",
parentFrameId: "main",
ownerBackendNodeId: 3,
target: { tabId: 1, sessionId: "remote" },
},
projection: {
...projection,
edges: [
{
sourceViewport: { width: 200, height: 200 },
destinationQuad: [
{ x: 100, y: 0 },
{ x: 300, y: 0 },
{ x: 300, y: 200 },
{ x: 100, y: 200 },
],
},
],
},
});
const result = await discoverVisualCandidates(facts([child, parent]));
expect(result.complete).toBe(true);
expect(result.candidates[0]).toMatchObject({
document: { frameId: "child", target: { sessionId: "remote" } },
region: { crop: { x: 100, y: 0, width: 100, height: 100 } },
});
const owner = parent.index.nodes.get(3)!;
owner.layout!.styles = { ...owner.layout!.styles, visibility: "hidden" };
expect((await discoverVisualCandidates(facts([parent, child]))).candidates).toHaveLength(0);
});
it.each<Record<string, string>>([
{ transform: "matrix(0,1,-1,0,0,0)" },
{ "clip-path": "circle(50%)" },
{ "mask-image": "url(mask)" },
{ zoom: "1.25", "overflow-x": "hidden" },
])("reports unsupported ancestor geometry: %o", async (styles) => {
const doc = await document([
{ id: 2, parent: 1, tag: "div", styles },
{ id: 3, parent: 2 },
]);
const result = await discoverVisualCandidates(facts([doc]));
expect(result.candidates).toHaveLength(0);
expect(result.issues[0].reason).toBe("geometry-unsupported");
});
it("supports positive scale without overflow, rejects pinch and non-rectangular frame projection", async () => {
const nodes = [
{
id: 2,
parent: 1,
styles: { transform: "matrix(1.25,0,0,1.25,0,0)" },
bounds: [10, 20, 150, 50],
},
];
expect(
(await discoverVisualCandidates(facts([await document(nodes)]))).candidates,
).toHaveLength(1);
expect(
(await discoverVisualCandidates(facts([await document(nodes, { pageScale: 1.5 })]))).issues[0]
.reason,
).toBe("geometry-unsupported");
const rotated = {
...projection,
edges: [
{
sourceViewport: { width: 200, height: 200 },
destinationQuad: [
{ x: 10, y: 0 },
{ x: 200, y: 10 },
{ x: 190, y: 200 },
{ x: 0, y: 190 },
] as [
{ x: number; y: number },
{ x: number; y: number },
{ x: number; y: number },
{ x: number; y: number },
],
},
],
};
expect(
(await discoverVisualCandidates(facts([await document(nodes, { projection: rotated })])))
.issues[0].reason,
).toBe("geometry-unsupported");
});
it("keeps partial capture separate from valid geometry, rejects unverified identities", async () => {
const doc = await document([{ id: 2, parent: 1 }]);
const partial = facts(
[doc],
[{ target: doc.frame.target, frameId: "main", stage: "ax", reason: "capture-unavailable" }],
);
const result = await discoverVisualCandidates(partial);
expect(result.candidates).toHaveLength(1);
expect(result.captureIssues).toBe(partial.issues);
expect(result.complete).toBe(false);
const unverified = { ...doc, identity: undefined };
const failed = await discoverVisualCandidates(facts([unverified]));
expect(failed.candidates).toHaveLength(0);
expect(failed.issues[0].reason).toBe("identity-unverified");
});
it("does not trust orphan, cyclic, boxless or missing frame-owner ancestry", async () => {
const doc = await document([
{ id: 2, parent: 1, tag: "div" },
{ id: 3, parent: 2 },
]);
for (const parent of [99, 3]) {
doc.index.nodes.get(2)!.parentBackendNodeId = parent;
const index = await buildDocumentIndex([...doc.index.nodes.values()], undefined, true);
const result = await discoverVisualCandidates(facts([{ ...doc, index }]));
expect(result.candidates).toHaveLength(0);
expect(result.issues[0].reason).toBe("ancestry-incomplete");
}
const boxless = await document([
{ id: 2, parent: 1, tag: "div", bounds: null },
{ id: 3, parent: 2 },
]);
expect((await discoverVisualCandidates(facts([boxless]))).issues[0].reason).toBe(
"facts-unavailable",
);
const child = {
...doc,
frame: { ...doc.frame, frameId: "child", parentFrameId: "absent", ownerBackendNodeId: 2 },
};
expect((await discoverVisualCandidates(facts([child]))).issues[0].reason).toBe(
"ownership-unresolved",
);
});
it("is deterministic, does not deduplicate or mutate inputs", async () => {
const input = facts([
await document([
{ id: 2, parent: 1 },
{ id: 3, parent: 1 },
]),
]);
const freeze = (value: unknown): void => {
if (!value || typeof value !== "object" || Object.isFrozen(value)) return;
if (value instanceof Map) for (const item of value.values()) freeze(item);
for (const item of Object.values(value)) freeze(item);
Object.freeze(value);
};
freeze(input);
const first = await discoverVisualCandidates(input);
expect(await discoverVisualCandidates(input)).toEqual(first);
expect(first.candidates).toHaveLength(2);
});
it.each([
1000, 10000, 100000,
])("memoizes deep ancestry across %i nodes and checks cancellation", async (count) => {
const doc = await document([{ id: 2, parent: 1 }]);
const template = doc.index.nodes.get(2)!;
let reads = 0;
const input = [doc.index.nodes.get(0)!, doc.index.nodes.get(1)!];
for (let id = 2; id < count + 2; id++) {
const node = {
...template,
backendNodeId: id,
tag: id < count / 2 ? "div" : "canvas",
parentBackendNodeId: id < count / 2 ? id - 1 : count / 2 - 1,
};
Object.defineProperty(node, "parentBackendNodeId", {
get: () => {
reads++;
return id < count / 2 ? id - 1 : count / 2 - 1;
},
});
input.push(node);
}
const index = await buildDocumentIndex(input, undefined, true);
reads = 0;
const result = await discoverVisualCandidates(facts([{ ...doc, index }]));
expect(result.candidates).toHaveLength(count / 2 + 2);
expect(reads).toBeLessThan(count * 5);
const controller = new AbortController();
controller.abort();
await expect(discoverVisualCandidates(facts([doc]), controller.signal)).rejects.toMatchObject({
name: "AbortError",
});
});
});
@@ -0,0 +1,250 @@
import { describe, expect, it } from "vitest";
import {
EMPTY_VISUAL_CONTEXT,
extendVisualContext,
projectVisualBox,
resolveVisualRegion,
type VisualAncestor,
} from "../visual-region";
const styles = {
position: "static",
visibility: "visible",
opacity: "1",
display: "block",
transform: "none",
zoom: "1",
"overflow-x": "visible",
"overflow-y": "visible",
"clip-path": "none",
"mask-image": "none",
rotate: "none",
scale: "none",
perspective: "none",
clip: "auto",
contain: "none",
"overflow-clip-margin": "0px",
};
const ancestor: VisualAncestor = {
document: {
attachmentId: "a",
target: { tabId: 1 },
frameId: "main",
documentElementBackendNodeId: 1,
},
backendNodeId: 2,
styles,
clientBox: { x: 5, y: 5, width: 60, height: 30 },
};
const box = { x: 0, y: 0, width: 120, height: 40 };
describe("shared visual region rules", () => {
it.each([
"visible",
"hidden",
"auto",
"scroll",
"clip",
])("keeps %s screenshot bounds independent of rounded styling", (overflow) => {
for (const clipContents of [true, false]) {
const node = {
...ancestor,
styles: { ...styles, "overflow-x": overflow, "overflow-y": overflow },
};
const baseline = extendVisualContext(EMPTY_VISUAL_CONTEXT, node, clipContents);
for (const radius of ["0px", "8px", "50%", "4px 8px"]) {
const rounded = extendVisualContext(
EMPTY_VISUAL_CONTEXT,
{
...node,
styles: {
...node.styles,
"border-top-left-radius": radius,
"border-top-right-radius": radius,
"border-bottom-left-radius": radius,
"border-bottom-right-radius": radius,
},
},
clipContents,
);
expect(rounded).toEqual(baseline);
expect(
resolveVisualRegion({
borderBox: box,
frameVisibleBox: box,
visibility: "visible",
context: rounded,
}),
).toMatchObject({ status: "available" });
expect(
resolveVisualRegion({
borderBox: box,
frameVisibleBox: box,
visibility: "hidden",
context: rounded,
}),
).toEqual({ status: "empty" });
if (clipContents && overflow !== "visible")
expect(
resolveVisualRegion({
borderBox: box,
frameVisibleBox: { x: 80, y: 0, width: 20, height: 20 },
visibility: "visible",
context: rounded,
}),
).toEqual({ status: "empty" });
}
}
});
it("accepts independently supplied normalized facts and preserves clipping policy", () => {
const context = extendVisualContext(EMPTY_VISUAL_CONTEXT, {
...ancestor,
styles: { ...styles, "overflow-x": "hidden" },
});
expect(
resolveVisualRegion({ borderBox: box, frameVisibleBox: box, visibility: "visible", context }),
).toMatchObject({
status: "available",
crop: { x: 5, y: 0, width: 60, height: 40 },
clips: { overflowX: "hidden", overflowY: "visible" },
});
const differentPolicy = extendVisualContext(EMPTY_VISUAL_CONTEXT, {
...ancestor,
styles: { ...styles, "overflow-x": "scroll" },
});
expect(differentPolicy.clip).toEqual(context.clip);
expect(differentPolicy.clips).not.toEqual(context.clips);
});
it("distinguishes fully clipped, missing client geometry and invalid rectangles", () => {
const clipping = { ...ancestor, styles: { ...styles, "overflow-x": "hidden" } };
const context = extendVisualContext(EMPTY_VISUAL_CONTEXT, {
...clipping,
clientBox: { x: 0, y: 0, width: 0, height: 20 },
});
expect(
resolveVisualRegion({ borderBox: box, frameVisibleBox: box, visibility: "visible", context })
.status,
).toBe("empty");
const unavailable = extendVisualContext(EMPTY_VISUAL_CONTEXT, { ...clipping, clientBox: null });
expect(
resolveVisualRegion({
borderBox: box,
frameVisibleBox: box,
visibility: "visible",
context: unavailable,
}),
).toEqual({ status: "unavailable", reason: "geometry-unavailable" });
expect(
resolveVisualRegion({
borderBox: { ...box, x: NaN },
frameVisibleBox: box,
visibility: "visible",
context: EMPTY_VISUAL_CONTEXT,
}).status,
).toBe("unavailable");
expect(projectVisualBox({ x: 5, y: 5, width: 0, height: 20 }, [])).toEqual({
x: 5,
y: 5,
width: 0,
height: 20,
});
});
it.each<Record<string, string>>([
{ rotate: "45deg" },
{ scale: "-1 1" },
{ perspective: "100px" },
{ clip: "rect(0px, 20px, 20px, 0px)" },
{ contain: "paint" },
{ "overflow-x": "clip", "overflow-clip-margin": "10px" },
{ transform: "matrix3d(1,0,0,0,0,1,0,0,0,0,1,0.01,0,0,0,1)" },
])("refuses unsupported CSS evidence without changing DOM geometry: %o", (change) => {
const context = extendVisualContext(EMPTY_VISUAL_CONTEXT, {
...ancestor,
styles: { ...styles, ...change },
});
expect(context.issue).toBe("geometry-unsupported");
});
it("accepts axis-preserving matrix3d translation and independent positive scale", () => {
for (const change of [
{ transform: "matrix3d(1,0,0,0,0,1,0,0,0,0,1,0,10,20,0,1)" },
{ scale: "1.25" },
])
expect(
extendVisualContext(EMPTY_VISUAL_CONTEXT, { ...ancestor, styles: { ...styles, ...change } })
.issue,
).toBeUndefined();
});
it.each([
"matrix(1,0,0,1,0,0)",
"matrix(1,0,0,1,10,20)",
])("accepts unscaled overflow transform %s", (transform) => {
const context = extendVisualContext(EMPTY_VISUAL_CONTEXT, {
...ancestor,
styles: {
...styles,
transform,
position: "relative",
"overflow-x": "hidden",
"overflow-y": "hidden",
},
});
expect(context.issue).toBeUndefined();
const child = extendVisualContext(
context,
{
...ancestor,
styles: {
...styles,
position: "absolute",
"overflow-x": "clip",
"overflow-y": "clip",
"overflow-clip-margin": "content-box",
},
},
false,
);
expect(
resolveVisualRegion({
borderBox: box,
frameVisibleBox: box,
visibility: "visible",
context: child,
}),
).toMatchObject({ status: "available", crop: { x: 5, y: 5, width: 60, height: 30 } });
});
it("supports positioned overflow containers but refuses uncertain out-of-flow clipping", () => {
const overflow = { ...ancestor, styles: { ...styles, "overflow-x": "hidden" } };
const staticParent = extendVisualContext(EMPTY_VISUAL_CONTEXT, overflow);
expect(
extendVisualContext(
staticParent,
{ ...ancestor, styles: { ...styles, position: "absolute" } },
false,
).issue,
).toBe("geometry-unsupported");
const positionedParent = extendVisualContext(EMPTY_VISUAL_CONTEXT, {
...overflow,
styles: { ...overflow.styles, position: "relative" },
});
const child = extendVisualContext(
positionedParent,
{ ...ancestor, styles: { ...styles, position: "absolute" } },
false,
);
expect(child.issue).toBeUndefined();
expect(child.clip).toEqual(positionedParent.clip);
expect(
extendVisualContext(
positionedParent,
{ ...ancestor, styles: { ...styles, position: "fixed" } },
false,
).issue,
).toBe("geometry-unsupported");
});
});
@@ -0,0 +1,40 @@
function captureAbortError(): Error {
const error = new Error("observation aborted");
error.name = "AbortError";
return error;
}
export function isAbortError(error: unknown): boolean {
return (
typeof error === "object" &&
error !== null &&
(error as { name?: string }).name === "AbortError"
);
}
export function throwIfAborted(signal: AbortSignal | undefined): void {
if (signal?.aborted) throw captureAbortError();
}
/** Check at bounded work blocks; yield only after a processing slice is spent.
* Each caller owns its deadline, so unrelated captures do not share state. */
export function createCaptureCheckpoint(signal?: AbortSignal): () => Promise<void> | undefined {
throwIfAborted(signal);
let deadline = performance.now() + 8;
return () => {
throwIfAborted(signal);
if (performance.now() < deadline) return;
return new Promise<void>((resolve) => {
const channel = new MessageChannel();
channel.port1.onmessage = () => {
channel.port1.close();
channel.port2.close();
resolve();
};
channel.port2.postMessage(null);
}).then(() => {
throwIfAborted(signal);
deadline = performance.now() + 8;
});
};
}
@@ -7,14 +7,13 @@ import {
import { cssViewport, GeometryContext } from "../geometry/frame-context";
import { type CdpRunner, cdpRunnerForTarget } from "../shared";
import { collectOverlayExcludedBackendIds } from "./capture";
import type { CapturedSceneInput } from "./facts";
import {
buildDocumentIndex,
type CaptureIssue,
isCaptureAbort,
type ObservationFacts,
throwCaptureAborted,
} from "./facts";
isAbortError as isCaptureAbort,
throwIfAborted as throwCaptureAborted,
} from "./capture-abort";
import { verifyDocumentIdentity } from "./document-identity";
import type { CapturedSceneInput, DocumentIdentity } from "./facts";
import { buildDocumentIndex, type CaptureIssue, type ObservationFacts } from "./facts";
import { enrichFormControlStates } from "./form-capture";
import {
buildFrameDocuments,
@@ -22,13 +21,19 @@ import {
type FrameDocument,
type FrameOwnedAxNode,
} from "./frame-document";
import { type NormalizedDocument, normalizeSnapshot } from "./normalize";
import { describeSnapshotFrames, REQUESTED_STYLES, type SnapshotReply } from "./snapshot";
import { type NormalizedFrameDocument, normalizeSnapshot } from "./normalize";
import {
BASIC_SNAPSHOT,
describeSnapshotFrames,
type SnapshotReply,
VISUAL_SNAPSHOT,
} from "./snapshot";
interface TargetBatch<T extends FrameOwnedAxNode> {
target: CdpTarget;
frames: CdpFrame[];
documents: NormalizedDocument[];
snapshotAttachmentId?: string;
documents: NormalizedFrameDocument[];
ax: FrameAxBatch<T>[];
fallbackExcluded: Set<number>;
snapshotFrameIds?: Set<string>;
@@ -37,7 +42,7 @@ interface TargetBatch<T extends FrameOwnedAxNode> {
/** A fixed worker pool scoped to one capture; each worker issues at most one
* collection request at a time. Geometry has its own measurement-phase bound. */
async function collectTargets<T>(
async function collectTasks<T>(
items: readonly T[],
collect: (item: T) => Promise<void>,
signal?: AbortSignal,
@@ -69,7 +74,9 @@ export async function captureObservationFacts<T extends FrameOwnedAxNode>(
tabId: number,
signal?: AbortSignal,
pageUrl?: string,
options: { includeVisualFacts?: boolean } = {},
): Promise<ObservationFacts<T>> {
const profile = options.includeVisualFacts ? VISUAL_SNAPSHOT : BASIC_SNAPSHOT;
const startedAt = Date.now();
throwCaptureAborted(signal);
let graph: CdpFrameGraph | undefined;
@@ -114,7 +121,8 @@ export async function captureObservationFacts<T extends FrameOwnedAxNode>(
const unavailable = (batch: TargetBatch<T>, stage: CaptureIssue["stage"], frameId?: string) =>
issues.push({ target: batch.target, frameId, stage, reason: "capture-unavailable" });
let firstFailure: unknown;
await collectTargets(
const axReady = new Set<TargetBatch<T>>();
await collectTasks(
batches,
async (batch) => {
const scoped = cdpRunnerForTarget(cdp, batch.target);
@@ -122,8 +130,9 @@ export async function captureObservationFacts<T extends FrameOwnedAxNode>(
throwCaptureAborted(signal);
await scoped.send(tabId, "DOMSnapshot.enable", {});
throwCaptureAborted(signal);
batch.snapshotAttachmentId = cdp.getAttachmentId?.(tabId);
const snapshot = await scoped.send<SnapshotReply>(tabId, "DOMSnapshot.captureSnapshot", {
computedStyles: REQUESTED_STYLES,
computedStyles: profile.computedStyles,
includePaintOrder: true,
includeDOMRects: true,
});
@@ -144,11 +153,12 @@ export async function captureObservationFacts<T extends FrameOwnedAxNode>(
issues,
signal,
observed.rootFrameId,
profile,
);
const formsAvailable = await enrichFormControlStates(
scoped,
tabId,
batch.documents.map((doc) => doc.domNodes),
batch.documents.map((doc) => doc.nodes),
signal,
);
throwCaptureAborted(signal);
@@ -181,26 +191,48 @@ export async function captureObservationFacts<T extends FrameOwnedAxNode>(
unavailable(batch, "ax");
return;
}
for (const frame of batch.frames) {
try {
throwCaptureAborted(signal);
const result = await scoped.send<{ nodes?: T[] }>(
tabId,
"Accessibility.getFullAXTree",
frame.frameId === "root" ? {} : { frameId: frame.frameId },
);
throwCaptureAborted(signal);
batch.ax.push({ frame, nodes: result.nodes ?? [] });
} catch (error) {
throwCaptureAborted(signal);
if (isCaptureAbort(error)) throw error;
firstFailure ??= error;
unavailable(batch, "ax", frame.frameId);
}
axReady.add(batch);
},
signal,
);
// Schedule AX by frame across all ready targets, rather than nesting pools.
// Keep result slots in source order even when CDP replies arrive out of order.
const axTasks: { batch: TargetBatch<T>; frame: CdpFrame; result?: FrameAxBatch<T> }[] =
batches.flatMap((batch) =>
axReady.has(batch) ? batch.frames.map((frame) => ({ batch, frame })) : [],
);
await collectTasks(
axTasks,
async (task) => {
const { batch, frame } = task;
try {
throwCaptureAborted(signal);
const result = await cdpRunnerForTarget(cdp, batch.target).send<{ nodes?: T[] }>(
tabId,
"Accessibility.getFullAXTree",
frame.frameId === "root" ? {} : { frameId: frame.frameId },
);
throwCaptureAborted(signal);
task.result = { frame, nodes: result.nodes ?? [] };
} catch (error) {
throwCaptureAborted(signal);
if (isCaptureAbort(error)) throw error;
firstFailure ??= error;
unavailable(batch, "ax", frame.frameId);
}
},
signal,
);
for (const task of axTasks) if (task.result) task.batch.ax.push(task.result);
let viewport = { width: 0, height: 0 };
try {
const measured = cssViewport(await geometry.layoutMetrics({ tabId }));
viewport = { width: measured.width, height: measured.height };
} catch (error) {
if (isCaptureAbort(error)) throw error;
}
// A snapshot claim outranks an old graph hint. Conflicting actual claims
// cannot be resolved by completion order or by overwriting a frameId map.
@@ -222,6 +254,56 @@ export async function captureObservationFacts<T extends FrameOwnedAxNode>(
!claims.has(frame.frameId) || claims.get(frame.frameId) === cdpTargetKey(batch.target),
),
);
const rootBatch = batches.find((batch) => !batch.target.sessionId);
const rootFrameId =
rootBatch?.rootFrameId ?? graph?.rootFrameId ?? rootBatch?.frames[0]?.frameId ?? "root";
const ownershipInvalid = new Set(invalid);
const identities = new Map<string, DocumentIdentity>();
const selected = new Map(currentFrames.map((frame) => [frame.frameId, frame]));
const checks = batches.flatMap((batch) => {
const docs = new Map(batch.documents.map((doc) => [doc.frame.frameId, doc]));
return batch.frames
.filter(
(frame) =>
!invalid.has(frame.frameId) &&
cdpTargetKey(selected.get(frame.frameId)!.target) === cdpTargetKey(batch.target),
)
.map((frame) => ({ batch, frame, doc: docs.get(frame.frameId) }));
});
await collectTasks(
checks,
async ({ batch, frame, doc }) => {
throwCaptureAborted(signal);
if (!batch.snapshotAttachmentId || doc?.documentElementBackendNodeId === undefined) {
issues.push({
target: frame.target,
frameId: frame.frameId,
stage: "identity",
reason: "identity-unverified",
});
return;
}
const identity: DocumentIdentity = {
attachmentId: batch.snapshotAttachmentId,
target: frame.target,
frameId: frame.frameId,
documentElementBackendNodeId: doc.documentElementBackendNodeId,
};
const verified = await verifyDocumentIdentity(cdp, identity, signal);
const status = cdp.getAttachmentId?.(tabId) === identity.attachmentId ? verified : "changed";
if (status === "current") identities.set(frame.frameId, identity);
else {
invalid.add(frame.frameId);
issues.push({
target: frame.target,
frameId: frame.frameId,
stage: "identity",
reason: status === "changed" ? "document-changed" : "identity-unavailable",
});
}
},
signal,
);
const children = new Map<string, string[]>();
for (const frame of currentFrames) {
if (!frame.parentFrameId) continue;
@@ -239,7 +321,7 @@ export async function captureObservationFacts<T extends FrameOwnedAxNode>(
}
}
for (const frame of currentFrames) {
if (invalid.has(frame.frameId))
if (ownershipInvalid.has(frame.frameId))
issues.push({
target: frame.target,
frameId: frame.frameId,
@@ -247,10 +329,12 @@ export async function captureObservationFacts<T extends FrameOwnedAxNode>(
reason: "frame-ownership-unresolved",
});
}
const rootBatch = batches.find((batch) => !batch.target.sessionId);
const rootFrameId =
rootBatch?.rootFrameId ?? graph?.rootFrameId ?? rootBatch?.frames[0]?.frameId ?? "root";
if (invalid.has(rootFrameId)) throw new Error("observation root document ownership is ambiguous");
if (invalid.has(rootFrameId))
throw new Error(
ownershipInvalid.has(rootFrameId)
? "observation root document ownership is ambiguous"
: "observation document identity changed or could not be verified; observe again",
);
const frames = currentFrames.filter((frame) => !invalid.has(frame.frameId));
const frameById = new Map(frames.map((frame) => [frame.frameId, frame]));
const belongs = (frame: CdpFrame) => {
@@ -270,7 +354,10 @@ export async function captureObservationFacts<T extends FrameOwnedAxNode>(
{
nodes: [],
rootFrameId,
frameNodes: new Map([...decoded].map(([id, doc]) => [id, doc.domNodes])),
frameNodes: new Map([...decoded].map(([id, doc]) => [id, doc.nodes])),
frameExcludedBackendNodeIds: new Map(
[...decoded].map(([id, doc]) => [id, doc.index.excludedBackendNodeIds]),
),
},
signal,
(frame) =>
@@ -281,21 +368,22 @@ export async function captureObservationFacts<T extends FrameOwnedAxNode>(
reason: "frame-ownership-unresolved",
}),
);
let viewport = { width: 0, height: 0 };
try {
const measured = cssViewport(await geometry.layoutMetrics({ tabId }));
viewport = { width: measured.width, height: measured.height };
} catch (error) {
if (isCaptureAbort(error)) throw error;
}
const facts: ObservationFacts<T>["documents"][number][] = [];
for (const document of documents) {
const doc = decoded.get(document.frameId);
const index = doc?.index ?? (await buildDocumentIndex([], signal));
const fallback = groups.get(cdpTargetKey(document.target))?.fallbackExcluded;
const { domNodes, axNodes, contextScopeId: _scope, ...frame } = document;
const {
domNodes,
axNodes,
contextScopeId: _scope,
excludedBackendNodeIds: _excluded,
...frame
} = document;
facts.push({
frame,
identity: identities.get(document.frameId),
...(doc?.geometry ? { geometry: doc.geometry } : {}),
index: fallback?.size ? { ...index, excludedBackendNodeIds: fallback } : index,
domNodes,
axNodes,
@@ -304,7 +392,15 @@ export async function captureObservationFacts<T extends FrameOwnedAxNode>(
throwCaptureAborted(signal);
if (firstFailure && facts.every((doc) => !doc.domNodes.length && !doc.axNodes.length))
throw firstFailure;
return { rootFrameId, viewport, documents: facts, issues, startedAt, finishedAt: Date.now() };
return {
visualFactsCollected: profile.includeVisualFacts,
rootFrameId,
viewport,
documents: facts,
issues,
startedAt,
finishedAt: Date.now(),
};
}
/** Existing semantic consumers get a narrow view; no raw snapshot
@@ -0,0 +1,37 @@
import type { Rect } from "@browser-skill/vom";
export interface CapturedNode {
backendNodeId: number;
parentBackendNodeId: number | null;
frameId?: string;
/** Owning iframe backend node id; `null` for the top-level document. */
ownerFrameBackendNodeId?: number | null;
tag: string;
attrs: Record<string, string>;
/** Top-level viewport-relative CSS px, clipped to the owning frame viewport. */
rect: Rect | null;
/** Frame-local viewport-relative CSS px before top-level projection. */
localRect?: Rect | null;
paintOrder: number;
position: string;
pointerEvents: string;
/**
* computed `cursor`. `cursor: pointer` is the strongest CDP-free signal
* that a non-semantic element (a `<div>`/`<span>` with a click handler)
* is actually an interactive control — used by the adapter to surface
* custom buttons/checkboxes the AX tree drops as `generic`. Optional like
* `textContent`: the live parser always sets it, hand-built fixtures may not.
*/
cursor?: string;
/**
* Whether the live DOM snapshot provides a painted, non-hidden box for this
* node. Semantic resolution uses this only for DOM fallback nodes; AX-backed
* nodes remain authoritative even when they are outside the viewport.
*/
rendered?: boolean;
textContent?: string;
formState?: "empty" | "filled" | "default";
formValue?: string;
formDefaultValue?: string;
formPlaceholder?: string;
}
+3 -2
View File
@@ -3,14 +3,15 @@
import { evaluateHoverTrigger } from "@/lib/hover-trigger-policy";
import { OVERLAY_HOST_SELECTOR } from "../../lib/overlay-bridge";
import type { CdpRunner } from "../shared";
import type { CapturedNode, CapturedSurfaceProbe } from "./facts";
import type { CapturedNode } from "./capture-types";
import type { CapturedSurfaceProbe } from "./facts";
export type {
CapturedNode,
CapturedSurfaceProbe,
} from "./facts";
import { isCaptureAbort as isAbortError, throwCaptureAborted as throwIfAborted } from "./facts";
import { isAbortError, throwIfAborted } from "./capture-abort";
import { clearHover, ProbeBudget, waitForHover } from "./hover-perception";
interface RuntimeEvaluateReply {
@@ -0,0 +1,91 @@
import type { CdpRunner } from "../shared";
import { sendToCdpTarget } from "../shared";
import { isAbortError, throwIfAborted } from "./capture-abort";
import type { DocumentIdentity } from "./facts";
import type { VisualCandidate } from "./visual-discovery";
/** Compare the snapshot root with the current root in that exact frame.
* Deep serialization supplies the backend ID without a separate describeNode. */
export async function verifyDocumentIdentity(
cdp: CdpRunner,
identity: DocumentIdentity,
signal?: AbortSignal,
): Promise<"current" | "changed" | "unavailable"> {
return verifyIdentity(cdp, identity, signal);
}
/** Verify the visual anchor in its original frame; geometry is checked by screenshot execution. */
export async function verifyVisualTargetIdentity(
cdp: CdpRunner,
candidate: VisualCandidate,
signal?: AbortSignal,
): Promise<"current" | "changed" | "unavailable"> {
return verifyIdentity(cdp, candidate.document, signal, candidate.backendNodeId);
}
async function verifyIdentity(
cdp: CdpRunner,
identity: DocumentIdentity,
signal?: AbortSignal,
anchorBackendNodeId?: number,
): Promise<"current" | "changed" | "unavailable"> {
const attached = () => cdp.getAttachmentId?.(identity.target.tabId) === identity.attachmentId;
throwIfAborted(signal);
if (!attached()) return "changed";
const objectGroup = `bsk-document-identity-${crypto.randomUUID()}`;
const send = <T>(method: string, params: object) => {
throwIfAborted(signal);
return sendToCdpTarget<T>(cdp, identity.target, method, params);
};
try {
const world = await send<{ executionContextId: number }>("Page.createIsolatedWorld", {
frameId: identity.frameId,
worldName: "bsk-document-identity",
});
const serializationOptions = {
serialization: "deep",
additionalParameters: { maxNodeDepth: 0, includeShadowTree: "none" },
};
type RootReply = {
result?: { deepSerializedValue?: { type: string; value?: { backendNodeId?: number } } };
};
let reply: RootReply;
if (anchorBackendNodeId === undefined) {
reply = await send<RootReply>("Runtime.evaluate", {
expression: "document.documentElement",
contextId: world.executionContextId,
objectGroup,
serializationOptions,
});
} else {
const anchor = await send<{ object?: { objectId?: string } }>("DOM.resolveNode", {
backendNodeId: anchorBackendNodeId,
executionContextId: world.executionContextId,
objectGroup,
});
if (!anchor.object?.objectId) return "unavailable";
reply = await send<RootReply>("Runtime.callFunctionOn", {
objectId: anchor.object.objectId,
functionDeclaration:
"function() { return this.isConnected && this.ownerDocument === document ? document.documentElement : null; }",
objectGroup,
serializationOptions,
});
}
if (!attached()) return "changed";
const root = reply.result?.deepSerializedValue;
if (root?.type === "null") return "changed";
const id = root?.type === "node" ? root.value?.backendNodeId : undefined;
if (id === undefined) return "unavailable";
return id === identity.documentElementBackendNodeId ? "current" : "changed";
} catch (error) {
throwIfAborted(signal);
if (isAbortError(error)) throw error;
return attached() ? "unavailable" : "changed";
} finally {
await sendToCdpTarget(cdp, identity.target, "Runtime.releaseObjectGroup", {
objectGroup,
}).catch(() => {});
throwIfAborted(signal);
}
}
+109 -110
View File
@@ -1,80 +1,36 @@
import type { Rect, Viewport } from "@browser-skill/vom";
import type { Viewport } from "@browser-skill/vom";
import type { CdpFrame, CdpTarget } from "@/browser-driver/frame-graph";
import { isOverlayHostNode } from "@/lib/overlay-bridge";
import type { FrameProjectionIssue } from "../geometry/coordinate-types";
import type { GeometryProjection } from "../geometry";
import type { FrameProjectionIssue, SnapshotCoordinates } from "../geometry/coordinate-types";
import { createCaptureCheckpoint } from "./capture-abort";
import type { CapturedNode } from "./capture-types";
import type { FrameOwnedAxNode } from "./frame-document";
export interface CapturedNode {
backendNodeId: number;
parentBackendNodeId: number | null;
frameId?: string;
/** Owning iframe backend node id; `null` for the top-level document. */
ownerFrameBackendNodeId?: number | null;
tag: string;
attrs: Record<string, string>;
/** Top-level viewport-relative CSS px, clipped to the owning frame viewport. */
rect: Rect | null;
/** Frame-local viewport-relative CSS px before top-level projection. */
localRect?: Rect | null;
paintOrder: number;
position: string;
pointerEvents: string;
/**
* computed `cursor`. `cursor: pointer` is the strongest CDP-free signal
* that a non-semantic element (a `<div>`/`<span>` with a click handler)
* is actually an interactive control — used by the adapter to surface
* custom buttons/checkboxes the AX tree drops as `generic`. Optional like
* `textContent`: the live parser always sets it, hand-built fixtures may not.
*/
cursor?: string;
/**
* Whether the live DOM snapshot provides a painted, non-hidden box for this
* node. Semantic resolution uses this only for DOM fallback nodes; AX-backed
* nodes remain authoritative even when they are outside the viewport.
*/
rendered?: boolean;
textContent?: string;
formState?: "empty" | "filled" | "default";
formValue?: string;
formDefaultValue?: string;
formPlaceholder?: string;
}
export interface CapturedSurfaceProbe {
triggerBackendNodeId: number;
triggerPoint?: { x: number; y: number };
triggerAction: "hover" | "focus" | string;
subItems: string[];
confidence?: "high" | "medium" | "low";
}
/** Narrow input shared by the existing semantic scene/hover consumers. */
export interface CapturedSceneInput {
nodes: CapturedNode[];
viewport: Viewport;
rootFrameId?: string;
excludedBackendNodeIds: ReadonlySet<number>;
}
/** Bounds retain their snapshot document CSS units until normalization. */
/** Bounds retain their raw snapshot document layout units until normalization. */
export interface SnapshotLayout {
readonly boundsSpace: "snapshot-document-css";
readonly boundsSpace: "snapshot-document-layout";
bounds?: number[];
/** [clientLeft, clientTop, clientWidth, clientHeight], untransformed local CSS units. */
clientRect?: number[];
styles: Readonly<Record<string, string>>;
}
export interface DecodedNode extends Omit<CapturedNode, "rect" | "localRect" | "rendered"> {
nodeType?: number;
parentMissing?: boolean;
layout?: SnapshotLayout;
}
export interface NodeFacts extends CapturedNode {
nodeType?: number;
parentMissing?: boolean;
layout?: SnapshotLayout;
}
export interface DocumentIndex<T extends DecodedNode = NodeFacts> {
readonly nodes: ReadonlyMap<number, T>;
readonly ancestryComplete?: ReadonlyMap<number, boolean>;
readonly excludedBackendNodeIds: ReadonlySet<number>;
}
@@ -82,78 +38,43 @@ export interface DecodedDocument {
nodes: DecodedNode[];
}
export interface CaptureIssue {
/** Owner failure or inherited blockage; absent for other geometry failures. */
projectionIssue?: FrameProjectionIssue;
target: CdpTarget;
frameId?: string;
stage: "dom" | "ax" | "ownership" | "geometry" | "forms";
reason: "capture-unavailable" | "frame-ownership-unresolved" | "geometry-unavailable";
}
export interface DocumentFacts<T extends FrameOwnedAxNode> {
readonly frame: CdpFrame;
readonly index: DocumentIndex;
readonly domNodes: CapturedNode[];
readonly axNodes: T[];
}
export interface ObservationFacts<T extends FrameOwnedAxNode> {
readonly rootFrameId: string;
readonly viewport: Viewport;
readonly documents: readonly DocumentFacts<T>[];
readonly issues: readonly CaptureIssue[];
readonly startedAt: number;
readonly finishedAt: number;
}
export function throwCaptureAborted(signal?: AbortSignal): void {
if (signal?.aborted) throw new DOMException("observation aborted", "AbortError");
}
export function isCaptureAbort(error: unknown): boolean {
return error instanceof Error && error.name === "AbortError";
}
/** Called only at bounded block boundaries, not once per node. */
export async function captureCheckpoint(signal?: AbortSignal): Promise<void> {
throwCaptureAborted(signal);
await new Promise<void>((resolve) => {
const channel = new MessageChannel();
channel.port1.onmessage = () => {
channel.port1.close();
channel.port2.close();
resolve();
};
channel.port2.postMessage(null);
});
throwCaptureAborted(signal);
}
/** All snapshot nodes participate, including document and shadow roots. This
* preserves overlay propagation when the semantic adapter omits those nodes. */
export async function buildDocumentIndex<T extends DecodedNode>(
input: readonly T[],
signal?: AbortSignal,
includeVisualFacts = false,
): Promise<DocumentIndex<T>> {
const checkpoint = createCaptureCheckpoint(signal);
const nodes = new Map<number, T>();
const overlayByNode = new Map<number, boolean>();
const ancestryComplete = includeVisualFacts ? new Map<number, boolean>() : undefined;
const excludedBackendNodeIds = new Set<number>();
for (let i = 0; i < input.length; i++) {
if (i % 256 === 0) await captureCheckpoint(signal);
if (i % 256 === 0) {
const pending = checkpoint();
if (pending) await pending;
}
const node = input[i];
nodes.set(node.backendNodeId, node);
}
let work = 0;
for (const node of input) {
if (work++ % 256 === 0) await captureCheckpoint(signal);
if (work++ % 256 === 0) {
const pending = checkpoint();
if (pending) await pending;
}
if (overlayByNode.has(node.backendNodeId)) continue;
const path: DecodedNode[] = [];
const visiting = new Set<number>();
let current: DecodedNode | undefined = node;
let overlay = false;
let complete = false;
while (current && !overlayByNode.has(current.backendNodeId)) {
if (work++ % 256 === 0) await captureCheckpoint(signal);
if (work++ % 256 === 0) {
const pending = checkpoint();
if (pending) await pending;
}
if (visiting.has(current.backendNodeId)) {
// Only an overlay inside the cycle may mark the cycle and its descendants.
overlay = path
@@ -164,20 +85,98 @@ export async function buildDocumentIndex<T extends DecodedNode>(
visiting.add(current.backendNodeId);
path.push(current);
if (current.parentBackendNodeId === null) {
if (ancestryComplete) complete = !current.parentMissing && current.nodeType === 9;
current = undefined;
break;
}
current = nodes.get(current.parentBackendNodeId);
}
if (current && overlayByNode.has(current.backendNodeId))
if (current && overlayByNode.has(current.backendNodeId)) {
overlay = overlayByNode.get(current.backendNodeId)!;
if (ancestryComplete) complete = ancestryComplete.get(current.backendNodeId) === true;
}
for (let i = path.length - 1; i >= 0; i--) {
if (work++ % 256 === 0) await captureCheckpoint(signal);
if (work++ % 256 === 0) {
const pending = checkpoint();
if (pending) await pending;
}
const item = path[i];
overlay = overlay || isOverlayHostNode(item.tag, Object.keys(item.attrs));
overlayByNode.set(item.backendNodeId, overlay);
if (ancestryComplete) {
complete = complete && !item.parentMissing;
ancestryComplete.set(item.backendNodeId, complete);
}
if (overlay) excludedBackendNodeIds.add(item.backendNodeId);
}
}
return { nodes, excludedBackendNodeIds };
return { nodes, ...(ancestryComplete ? { ancestryComplete } : {}), excludedBackendNodeIds };
}
export interface DocumentIdentity {
attachmentId: string;
target: CdpTarget;
frameId: string;
documentElementBackendNodeId: number;
}
/** Narrow input shared by the existing semantic scene/hover consumers. */
export interface CapturedSceneInput {
nodes: CapturedNode[];
viewport: Viewport;
rootFrameId?: string;
excludedBackendNodeIds: ReadonlySet<number>;
}
export interface CaptureIssue {
/** Owner failure or inherited blockage; absent for other geometry failures. */
projectionIssue?: FrameProjectionIssue;
target: CdpTarget;
frameId?: string;
stage: "dom" | "ax" | "identity" | "ownership" | "geometry" | "forms";
reason:
| "document-changed"
| "identity-unavailable"
| "identity-unverified"
| "capture-unavailable"
| "frame-ownership-unresolved"
| "geometry-unavailable";
}
/** Shared per document; never retain the live GeometryContext in published facts. */
export interface DocumentGeometry {
readonly projections: readonly GeometryProjection[];
readonly coordinates: SnapshotCoordinates;
/** Visual viewport scale (pinch), distinct from browser UI zoom. */
readonly pageScale?: number;
}
export interface DocumentFacts<T extends FrameOwnedAxNode> {
readonly geometry?: DocumentGeometry;
readonly frame: CdpFrame;
readonly identity?: DocumentIdentity;
readonly index: DocumentIndex;
readonly domNodes: CapturedNode[];
readonly axNodes: T[];
}
export interface ObservationFacts<T extends FrameOwnedAxNode> {
/** Whether visual collection was enabled; partial capture failures remain in issues. */
readonly visualFactsCollected: boolean;
readonly rootFrameId: string;
readonly viewport: Viewport;
readonly documents: readonly DocumentFacts<T>[];
readonly issues: readonly CaptureIssue[];
readonly startedAt: number;
readonly finishedAt: number;
}
export interface CapturedSurfaceProbe {
triggerBackendNodeId: number;
triggerPoint?: { x: number; y: number };
triggerAction: "hover" | "focus" | string;
subItems: string[];
confidence?: "high" | "medium" | "low";
}
export type { CapturedNode } from "./capture-types";
+2 -2
View File
@@ -1,6 +1,6 @@
import type { CdpRunner } from "../shared";
import type { CapturedNode } from "./facts";
import { isCaptureAbort as isAbortError, throwCaptureAborted as throwIfAborted } from "./facts";
import { isAbortError, throwIfAborted } from "./capture-abort";
import type { CapturedNode } from "./capture-types";
interface RuntimeEvaluateReply {
result?: { deepSerializedValue?: DeepSerializedValue };
+26 -14
View File
@@ -4,8 +4,8 @@ import {
type CdpTarget,
cdpTargetKey,
} from "@/browser-driver/frame-graph";
import type { CapturedNode } from "./facts";
import { captureCheckpoint } from "./facts";
import { createCaptureCheckpoint } from "./capture-abort";
import type { CapturedNode } from "./capture-types";
export interface FrameOwnedAxNode {
nodeId: string;
@@ -32,6 +32,7 @@ export interface FrameDomInput {
nodes: CapturedNode[];
rootFrameId?: string;
frameNodes?: ReadonlyMap<string, CapturedNode[]>;
frameExcludedBackendNodeIds?: ReadonlyMap<string, ReadonlySet<number>>;
}
interface Ownership {
@@ -72,6 +73,7 @@ export async function buildFrameDocuments<T extends FrameOwnedAxNode>(
signal?: AbortSignal,
unresolved?: (frame: CdpFrame) => void,
): Promise<FrameDocument<T>[]> {
const checkpoint = createCaptureCheckpoint(signal);
let work = 0;
const frames = frameList(graph, batches);
const frameById = new Map(frames.map((frame) => [frame.frameId, frame]));
@@ -81,7 +83,10 @@ export async function buildFrameDocuments<T extends FrameOwnedAxNode>(
const backendOwner = new Map<string, string>();
for (const frame of frames) {
for (const node of domNodesForFrame(frame.frameId)) {
if (work++ % 256 === 0) await captureCheckpoint(signal);
if (work++ % 256 === 0) {
const pending = checkpoint();
if (pending) await pending;
}
backendOwner.set(targetBackendKey(frame.target, node.backendNodeId), frame.frameId);
}
}
@@ -90,18 +95,24 @@ export async function buildFrameDocuments<T extends FrameOwnedAxNode>(
for (const batch of batches) {
const nodeById = new Map<string, T>();
for (const node of batch.nodes) {
if (work++ % 256 === 0) await captureCheckpoint(signal);
if (work++ % 256 === 0) {
const pending = checkpoint();
if (pending) await pending;
}
nodeById.set(node.nodeId, node);
}
let reported = false;
const ownershipByNodeId = new Map<string, Ownership>();
const resolveOwnership = async (node: T): Promise<Ownership> => {
for (const node of batch.nodes) {
const path: T[] = [];
const visiting = new Set<string>();
let current: T | undefined = node;
let ownership: Ownership = { frameId: batch.frame.frameId, strength: 1 };
while (current) {
if (work++ % 256 === 0) await captureCheckpoint(signal);
if (work++ % 256 === 0) {
const pending = checkpoint();
if (pending) await pending;
}
const cached = ownershipByNodeId.get(current.nodeId);
if (cached) {
ownership = cached;
@@ -137,17 +148,14 @@ export async function buildFrameDocuments<T extends FrameOwnedAxNode>(
current = current.parentId ? nodeById.get(current.parentId) : undefined;
}
for (let i = path.length - 1; i >= 0; i--) {
if (work++ % 256 === 0) await captureCheckpoint(signal);
if (work++ % 256 === 0) {
const pending = checkpoint();
if (pending) await pending;
}
ownership = { frameId: ownership.frameId, strength: Math.min(2, ownership.strength) };
ownershipByNodeId.set(path[i].nodeId, ownership);
}
if (!ownershipByNodeId.has(node.nodeId)) ownershipByNodeId.set(node.nodeId, ownership);
return ownership;
};
for (const node of batch.nodes) {
if (work++ % 256 === 0) await captureCheckpoint(signal);
const ownership = await resolveOwnership(node);
if (!ownership.frameId) continue;
const key = targetNodeKey(batch.frame.target, node.nodeId);
const existing = candidates.get(key);
@@ -162,7 +170,10 @@ export async function buildFrameDocuments<T extends FrameOwnedAxNode>(
);
const axNodesByFrame = new Map<string, T[]>();
for (const candidate of candidates.values()) {
if (work++ % 256 === 0) await captureCheckpoint(signal);
if (work++ % 256 === 0) {
const pending = checkpoint();
if (pending) await pending;
}
const { node, target, ownership } = candidate;
const parentFrameId = node.parentId
? ownershipByTargetNode.get(targetNodeKey(target, node.parentId))
@@ -188,5 +199,6 @@ export async function buildFrameDocuments<T extends FrameOwnedAxNode>(
contextScopeId: frame.frameId,
axNodes: axNodesByFrame.get(frame.frameId) ?? [],
domNodes: domNodesForFrame(frame.frameId),
excludedBackendNodeIds: captured.frameExcludedBackendNodeIds?.get(frame.frameId),
}));
}
+183 -60
View File
@@ -4,30 +4,138 @@ import { type GeometryProjection, projectRectToViewport } from "../geometry";
import {
type FrameProjectionState,
projectSnapshotRect,
type SnapshotCoordinates,
snapshotCoordinates,
snapshotViewportRect,
} from "../geometry/coordinate-types";
import { cssViewport, GeometryContext } from "../geometry/frame-context";
import type { CapturedNode } from "./facts";
import {
GeometryContext,
type LayoutMetrics,
snapshotLayoutScale,
} from "../geometry/frame-context";
import {
createCaptureCheckpoint,
isAbortError as isCaptureAbort,
throwIfAborted as throwCaptureAborted,
} from "./capture-abort";
import {
buildDocumentIndex,
type CaptureIssue,
captureCheckpoint,
type DocumentGeometry,
type DocumentIndex,
isCaptureAbort,
type NodeFacts,
throwCaptureAborted,
} from "./facts";
import {
BASIC_SNAPSHOT,
decodeDocument,
type SnapshotDocument,
type SnapshotProfile,
type SnapshotReply,
snapshotFrameId,
} from "./snapshot";
export interface FrameContext {
frameId?: string;
ownerFrameBackendNodeId: number | null;
projection: FrameProjectionState | null;
targetProjection?: GeometryProjection | null;
target: CdpTarget;
coordinates: SnapshotCoordinates | null;
pageScale?: number;
}
export interface NormalizedDocument {
geometry?: DocumentGeometry;
nodes: NodeFacts[];
index: DocumentIndex;
documentElementBackendNodeId?: number;
}
/** Interpret one document using a supplied projection. No live reads or frame scheduling. */
export async function normalizeDocument(
doc: SnapshotDocument,
strings: string[],
context: FrameContext,
signal?: AbortSignal,
profile: SnapshotProfile = BASIC_SNAPSHOT,
): Promise<NormalizedDocument> {
const decoded = await decodeDocument(doc, strings, signal, profile);
const checkpoint = createCaptureCheckpoint(signal);
const nodes: NodeFacts[] = [];
for (let i = 0; i < decoded.nodes.length; i++) {
if (i % 256 === 0) {
const pending = checkpoint();
if (pending) await pending;
}
const node = decoded.nodes[i];
const bounds = node.layout?.bounds ?? [];
const input = snapshotViewportRect(
bounds,
{ target: context.target, frameId: context.frameId },
context.coordinates,
);
const local = input?.rect;
let rect =
input && context.projection?.status === "available"
? projectSnapshotRect(input, context.projection.projection)
: null;
if (rect && context.targetProjection !== undefined)
rect = context.targetProjection
? projectRectToViewport(
{ x: rect.x, y: rect.y, w: rect.width, h: rect.height },
context.targetProjection,
)
: null;
const visibility = node.layout?.styles.visibility || "visible";
const opacity = node.layout?.styles.opacity || "1";
nodes.push({
...node,
...(context.frameId ? { frameId: context.frameId } : {}),
ownerFrameBackendNodeId: context.ownerFrameBackendNodeId,
localRect: local ? { x: local.x, y: local.y, w: local.width, h: local.height } : null,
rect: rect ? { x: rect.x, y: rect.y, w: rect.width, h: rect.height } : null,
rendered:
bounds.length >= 4 &&
bounds.slice(0, 4).every(Number.isFinite) &&
bounds[2] > 0 &&
bounds[3] > 0 &&
visibility !== "hidden" &&
visibility !== "collapse" &&
(Number.parseFloat(opacity) || 0) > 0,
});
}
const index = await buildDocumentIndex(nodes, signal, profile.includeVisualFacts);
return {
nodes: nodes.filter(
(node) => !node.tag.startsWith("#") && !index.excludedBackendNodeIds.has(node.backendNodeId),
),
index,
...(profile.includeVisualFacts &&
context.coordinates &&
context.projection?.status === "available" &&
context.targetProjection !== null
? {
geometry: {
projections: [
context.projection.projection.geometry,
...(context.targetProjection ? [context.targetProjection] : []),
],
coordinates: context.coordinates,
pageScale: context.pageScale,
},
}
: {}),
documentElementBackendNodeId: nodes.find(
(node) =>
node.nodeType === 1 &&
node.parentBackendNodeId !== null &&
index.nodes.get(node.parentBackendNodeId)?.nodeType === 9,
)?.backendNodeId,
};
}
export interface NormalizedFrameDocument extends NormalizedDocument {
frame: CdpFrame;
domNodes: CapturedNode[];
}
/** One target's snapshot. Source ownership is checked before interpreting any
@@ -40,9 +148,18 @@ export async function normalizeSnapshot(
issues: CaptureIssue[],
signal?: AbortSignal,
rootFrameId?: string,
): Promise<NormalizedDocument[]> {
profile: SnapshotProfile = BASIC_SNAPSHOT,
): Promise<NormalizedFrameDocument[]> {
const strings = snapshot.strings ?? [];
const raw = snapshot.documents ?? [];
const ownerIds = new Set<number>();
for (const document of raw) {
for (const index of document.nodes?.contentDocumentIndex?.index ?? []) {
const id = document.nodes?.backendNodeId?.[index];
if (id !== undefined) ownerIds.add(id);
}
}
geometry.registerSnapshotOwners(target, ownerIds);
const frameById = new Map(frames.map((frame) => [frame.frameId, frame]));
const sources = new Map<string, SnapshotDocument>();
for (const doc of raw) {
@@ -58,17 +175,20 @@ export async function normalizeSnapshot(
}
sources.set(id, doc);
}
let viewport: Viewport = { width: 0, height: 0 };
let scrollX = 0,
scrollY = 0;
let metrics: LayoutMetrics = {};
try {
const measured = cssViewport(await geometry.layoutMetrics(target));
viewport = { width: measured.width, height: measured.height };
scrollX = measured.scrollX;
scrollY = measured.scrollY;
metrics = await geometry.layoutMetrics(target);
} catch (error) {
if (isCaptureAbort(error)) throw error;
}
const viewport: Viewport = {
width: metrics.cssLayoutViewport?.clientWidth ?? 0,
height: metrics.cssLayoutViewport?.clientHeight ?? 0,
};
const layoutUnitsPerCssPixel = snapshotLayoutScale(metrics);
const viewportAvailable = [viewport.width, viewport.height].every(
(size) => Number.isFinite(size) && size > 0,
);
const children = new Map<string, CdpFrame[]>();
const pending: CdpFrame[] = [];
@@ -81,7 +201,7 @@ export async function normalizeSnapshot(
}
}
const projections = new Map<string, FrameProjectionState | null>();
const result: NormalizedDocument[] = [];
const result: NormalizedFrameDocument[] = [];
let targetProjection: GeometryProjection | null = null;
const targetRoot = rootFrameId ? frameById.get(rootFrameId) : undefined;
if (target.sessionId && targetRoot) {
@@ -93,6 +213,7 @@ export async function normalizeSnapshot(
if (isCaptureAbort(error)) throw error;
}
}
const checkpoint = createCaptureCheckpoint(signal);
const visited = new Set<string>();
const remaining = frames[Symbol.iterator]();
for (let cursor = 0; visited.size < frames.length; cursor++) {
@@ -102,7 +223,10 @@ export async function normalizeSnapshot(
if (next.done) break;
pending.push(next.value);
}
if (cursor % 256 === 0) await captureCheckpoint(signal);
if (cursor % 256 === 0) {
const pending = checkpoint();
if (pending) await pending;
}
const frame = pending[cursor];
if (visited.has(frame.frameId)) continue;
visited.add(frame.frameId);
@@ -110,12 +234,24 @@ export async function normalizeSnapshot(
const doc = sources.get(frame.frameId);
if (!doc) continue;
const source = { target, frameId: frame.frameId };
const state: FrameProjectionState | null =
const coordinates = snapshotCoordinates(
doc,
layoutUnitsPerCssPixel,
frame.frameId === rootFrameId
? {
status: "available",
projection: { source, geometry: { sourceClips: [], edges: [], topViewport: viewport } },
}
? { x: metrics.cssLayoutViewport?.pageX, y: metrics.cssLayoutViewport?.pageY }
: undefined,
);
let state: FrameProjectionState | null =
frame.frameId === rootFrameId
? viewportAvailable && coordinates
? {
status: "available",
projection: {
source,
geometry: { sourceClips: [], edges: [], topViewport: viewport },
},
}
: { status: "unavailable", source, reason: "snapshot-coordinates-unavailable" }
: (projections.get(frame.frameId) ?? null);
if (!state && frame.frameId !== rootFrameId)
issues.push({
@@ -124,6 +260,8 @@ export async function normalizeSnapshot(
stage: "ownership",
reason: "frame-ownership-unresolved",
});
if (!coordinates && (!state || state.status === "available"))
state = { status: "unavailable", source, reason: "snapshot-coordinates-unavailable" };
projections.set(frame.frameId, state);
const projection = state?.status === "available" ? state.projection : null;
if (!projection || (target.sessionId && !targetProjection))
@@ -168,6 +306,14 @@ export async function normalizeSnapshot(
projections.set(child.frameId, null);
continue;
}
if (!snapshotCoordinates(sources.get(child.frameId)!, layoutUnitsPerCssPixel)) {
projections.set(child.frameId, {
status: "unavailable",
source: childSource,
reason: "snapshot-coordinates-unavailable",
});
continue;
}
try {
projections.set(
child.frameId,
@@ -187,47 +333,24 @@ export async function normalizeSnapshot(
// Join active reads and their cleanup before returning cancellation.
if (failure) throw failure;
throwCaptureAborted(signal);
const decoded = await decodeDocument(doc, strings, signal);
const nodes: NodeFacts[] = [];
for (let i = 0; i < decoded.nodes.length; i++) {
if (i % 256 === 0) await captureCheckpoint(signal);
const node = decoded.nodes[i];
const input = snapshotViewportRect(node.layout?.bounds ?? [], source, {
x: doc.scrollOffsetX ?? (frame.frameId === rootFrameId ? scrollX : 0),
y: doc.scrollOffsetY ?? (frame.frameId === rootFrameId ? scrollY : 0),
});
const local = input?.rect;
let rect = input && projection ? projectSnapshotRect(input, projection) : null;
if (rect && target.sessionId)
rect = targetProjection
? projectRectToViewport(
{ x: rect.x, y: rect.y, w: rect.width, h: rect.height },
targetProjection,
)
: null;
const styles = node.layout?.styles;
const visibility = styles?.visibility || "visible";
const opacity = styles?.opacity || "1";
nodes.push({
...node,
const normalized = await normalizeDocument(
doc,
strings,
{
frameId: frame.frameId,
ownerFrameBackendNodeId: frame.ownerBackendNodeId ?? null,
localRect: local ? { x: local.x, y: local.y, w: local.width, h: local.height } : null,
rect: rect ? { x: rect.x, y: rect.y, w: rect.width, h: rect.height } : null,
rendered:
!!local &&
visibility !== "hidden" &&
visibility !== "collapse" &&
(Number.parseFloat(opacity) || 0) > 0,
});
}
const index = await buildDocumentIndex(nodes, signal);
const domNodes = nodes.filter((node) => !node.tag.startsWith("#"));
result.push({
index,
frame,
domNodes,
});
projection: state,
...(profile.includeVisualFacts
? { pageScale: metrics.cssVisualViewport?.scale ?? metrics.visualViewport?.scale }
: {}),
target,
...(target.sessionId ? { targetProjection } : {}),
coordinates,
},
signal,
profile,
);
result.push({ ...normalized, frame });
}
return result;
}
+53 -11
View File
@@ -1,6 +1,7 @@
import { type CdpFrame, type CdpTarget, cdpTargetKey } from "@/browser-driver/frame-graph";
import type { CapturedNode } from "./facts";
import { captureCheckpoint, type DecodedDocument, type DecodedNode } from "./facts";
import { createCaptureCheckpoint } from "./capture-abort";
import type { CapturedNode } from "./capture-types";
import type { DecodedDocument, DecodedNode } from "./facts";
export const REQUESTED_STYLES = [
"position",
"pointer-events",
@@ -8,9 +9,30 @@ export const REQUESTED_STYLES = [
"visibility",
"opacity",
] as const;
const STYLE_COL = Object.fromEntries(
REQUESTED_STYLES.map((name, index) => [name, index]),
) as Record<(typeof REQUESTED_STYLES)[number], number>;
export const VISUAL_STYLES = [
...REQUESTED_STYLES,
"display",
"overflow-x",
"overflow-y",
"transform",
"zoom",
"clip-path",
"mask-image",
"rotate",
"scale",
"perspective",
"clip",
"contain",
"overflow-clip-margin",
] as const;
/** The same profile owns request columns, decoding and visual-only derived facts. */
export const BASIC_SNAPSHOT = {
includeVisualFacts: false,
computedStyles: REQUESTED_STYLES,
} as const;
export const VISUAL_SNAPSHOT = { includeVisualFacts: true, computedStyles: VISUAL_STYLES } as const;
export type SnapshotProfile = typeof BASIC_SNAPSHOT | typeof VISUAL_SNAPSHOT;
/** Sparse array format Chrome uses for infrequently-set per-node fields. */
interface SparseArray {
index: number[];
@@ -48,6 +70,7 @@ export interface SnapshotDocument {
nodeIndex?: number[];
styles?: number[][];
bounds?: number[][];
clientRects?: number[][];
paintOrders?: number[];
};
}
@@ -87,7 +110,7 @@ function str(strings: string[], idx: number | undefined): string {
return strings[idx] ?? "";
}
function sparseIndexMap(sparse: SparseArray | undefined): Map<number, number> {
export function sparseIndexMap(sparse: SparseArray | undefined): Map<number, number> {
const out = new Map<number, number>();
if (!sparse?.index || !sparse.value) return out;
for (let i = 0; i < sparse.index.length; i++) {
@@ -102,7 +125,9 @@ export async function decodeDocument(
doc: SnapshotDocument,
strings: string[],
signal?: AbortSignal,
profile: SnapshotProfile = BASIC_SNAPSHOT,
): Promise<DecodedDocument> {
const checkpoint = createCaptureCheckpoint(signal);
const dn = doc.nodes;
const dl = doc.layout;
if (!dn?.backendNodeId) {
@@ -112,7 +137,10 @@ export async function decodeDocument(
const count = dn.backendNodeId.length;
const layoutByNode = new Map<number, number>();
for (let i = 0; i < (dl?.nodeIndex?.length ?? 0); i++) {
if (i % 256 === 0) await captureCheckpoint(signal);
if (i % 256 === 0) {
const pending = checkpoint();
if (pending) await pending;
}
layoutByNode.set(dl!.nodeIndex![i], i);
}
@@ -127,7 +155,10 @@ export async function decodeDocument(
const nodeTextContent = new Map<number, string[]>();
if (dn.nodeValue) {
for (let n = 0; n < count; n++) {
if (n % 256 === 0) await captureCheckpoint(signal);
if (n % 256 === 0) {
const pending = checkpoint();
if (pending) await pending;
}
const nvIdx = dn.nodeValue[n] ?? -1;
if (nvIdx < 0) continue;
const text = str(strings, nvIdx).trim();
@@ -144,7 +175,10 @@ export async function decodeDocument(
// Decode fields without assigning a coordinate projection or semantic policy.
const nodes: DecodedNode[] = [];
for (let n = 0; n < count; n++) {
if (n % 256 === 0) await captureCheckpoint(signal);
if (n % 256 === 0) {
const pending = checkpoint();
if (pending) await pending;
}
const backendNodeId = dn.backendNodeId[n];
const parentIdx = dn.parentIndex?.[n] ?? -1;
const parentBackendNodeId = parentIdx >= 0 ? (dn.backendNodeId[parentIdx] ?? null) : null;
@@ -159,13 +193,17 @@ export async function decodeDocument(
const li = layoutByNode.get(n);
const styleRow = li === undefined ? [] : (dl?.styles?.[li] ?? []);
const styles: Record<string, string> = {};
for (const name of REQUESTED_STYLES) styles[name] = str(strings, styleRow[STYLE_COL[name]]);
for (let i = 0; i < profile.computedStyles.length; i++)
styles[profile.computedStyles[i]] = str(strings, styleRow[i]);
const layout =
li === undefined
? undefined
: {
boundsSpace: "snapshot-document-css" as const,
boundsSpace: "snapshot-document-layout" as const,
bounds: dl?.bounds?.[li],
...(profile.includeVisualFacts && dl?.clientRects?.[li]
? { clientRect: dl.clientRects[li] }
: {}),
styles,
};
@@ -189,6 +227,10 @@ export async function decodeDocument(
backendNodeId,
nodeType: dn.nodeType?.[n],
parentBackendNodeId,
...(profile.includeVisualFacts &&
(dn.parentIndex?.[n] === undefined || (parentIdx >= 0 && parentBackendNodeId === null))
? { parentMissing: true }
: {}),
tag,
attrs,
@@ -0,0 +1,64 @@
import { createCaptureCheckpoint } from "./capture-abort";
import type { VisualCandidate, VisualDiscoveryResult } from "./visual-discovery";
export const MAX_VISUAL_DEDUP_KEYS = 50_000;
export interface VisualDedupResult extends VisualDiscoveryResult {
/** Known valid candidates in this capture, not the total Canvas count on the page. */
readonly candidateCount: number;
/** Candidates actually removed by exact coverage deduplication. */
readonly deduplicatedCount: number;
readonly dedupDegraded: boolean;
}
/** Keep the first real anchor for each exact DOM/parent/crop key, in encounter order.
* Once the key set is full, unknown keys pass through: capacity never drops candidates. */
export async function deduplicateVisualCandidates(
discovery: VisualDiscoveryResult,
signal?: AbortSignal,
): Promise<VisualDedupResult> {
const checkpoint = createCaptureCheckpoint(signal);
const keys = new Set<string>();
const candidates: VisualCandidate[] = [];
let deduplicatedCount = 0;
let dedupDegraded = false;
for (let i = 0; i < discovery.candidates.length; i++) {
if (i % 256 === 0) {
const pending = checkpoint();
if (pending) await pending;
}
const candidate = discovery.candidates[i];
const {
document,
region: { crop },
} = candidate;
const key = JSON.stringify([
document.attachmentId,
document.target.tabId,
document.target.sessionId ?? null,
document.frameId,
document.documentElementBackendNodeId,
candidate.parentBackendNodeId,
crop.x,
crop.y,
crop.width,
crop.height,
]);
if (keys.has(key)) {
deduplicatedCount++;
continue;
}
if (keys.size < MAX_VISUAL_DEDUP_KEYS) keys.add(key);
else dedupDegraded = true;
candidates.push(candidate);
}
const pending = checkpoint();
if (pending) await pending;
return {
...discovery,
candidates,
candidateCount: discovery.candidates.length,
deduplicatedCount,
dedupDegraded,
};
}
@@ -0,0 +1,321 @@
import { type CdpTarget } from "@/browser-driver/frame-graph";
import type { ViewportRect } from "../geometry";
import { createCaptureCheckpoint } from "./capture-abort";
import type {
CaptureIssue,
DocumentFacts,
DocumentIdentity,
NodeFacts,
ObservationFacts,
} from "./facts";
import type { FrameOwnedAxNode } from "./frame-document";
import {
EMPTY_VISUAL_CONTEXT,
extendVisualContext,
projectVisualBox,
resolveVisualRegion,
type VisualContext,
type VisualIssueReason,
type VisualRegionResult,
visualProjectionIssue,
} from "./visual-region";
export interface VisualCandidate {
readonly document: DocumentIdentity;
readonly backendNodeId: number;
readonly parentBackendNodeId: number | null;
readonly label?: string;
readonly region: Extract<VisualRegionResult, { status: "available" }>;
}
export type VisualDiscoveryIssue =
| {
readonly reason: "visual-facts-not-collected";
readonly target?: never;
readonly frameId?: never;
readonly backendNodeId?: never;
}
| {
readonly target: CdpTarget;
readonly frameId: string;
readonly backendNodeId?: number;
readonly reason: VisualIssueReason;
};
export interface VisualDiscoveryResult {
readonly candidates: readonly VisualCandidate[];
readonly issues: readonly VisualDiscoveryIssue[];
/** Preserve upstream failures, including those that do not invalidate Canvas geometry. */
readonly captureIssues: readonly CaptureIssue[];
readonly complete: boolean;
}
type Document = DocumentFacts<FrameOwnedAxNode>;
interface NodeContext {
self: VisualContext;
children: VisualContext;
}
function viewportRect(
rect: { x: number; y: number; w: number; h: number } | null | undefined,
): ViewportRect | null {
return rect ? { x: rect.x, y: rect.y, width: rect.w, height: rect.h } : null;
}
/** Client offsets are unscaled local CSS units. Only ordinary, untransformed
* overflow chains consume this adapter; other combinations are rejected by policy. */
function clientBox(node: NodeFacts, document: Document): ViewportRect | null {
const client = node.layout?.clientRect;
const bounds = node.layout?.bounds;
const border =
node.localRect ??
(bounds?.length === 4 && bounds.every(Number.isFinite) && document.geometry
? {
x:
bounds[0] / document.geometry.coordinates.layoutUnitsPerCssPixel -
document.geometry.coordinates.scrollCss.x,
y:
bounds[1] / document.geometry.coordinates.layoutUnitsPerCssPixel -
document.geometry.coordinates.scrollCss.y,
}
: null);
if (
!client ||
client.length !== 4 ||
!client.every(Number.isFinite) ||
client[2] < 0 ||
client[3] < 0 ||
!border ||
!document.geometry
)
return null;
return projectVisualBox(
{ x: border.x + client[0], y: border.y + client[1], width: client[2], height: client[3] },
document.geometry.projections,
);
}
/** Pure Facts consumer: no CDP, ref registration, selection, or semantic keep/drop.
* Both frame and node ancestry are memoized in this call, using iterative walks. */
export async function discoverVisualCandidates(
facts: ObservationFacts<FrameOwnedAxNode>,
signal?: AbortSignal,
): Promise<VisualDiscoveryResult> {
const checkpoint = createCaptureCheckpoint(signal);
if (!facts.visualFactsCollected)
return {
candidates: [],
complete: false,
captureIssues: facts.issues,
issues: [{ reason: "visual-facts-not-collected" }],
};
let work = 0;
const candidates: VisualCandidate[] = [];
const issues: VisualDiscoveryIssue[] = [];
const documents = new Map(facts.documents.map((document) => [document.frame.frameId, document]));
const contexts = new Map<string, Map<number, NodeContext>>();
const roots = new Map<string, VisualContext>();
async function nodeContext(document: Document, id: number): Promise<NodeContext> {
const cache = contexts.get(document.frame.frameId)!;
const path: NodeFacts[] = [];
let node = document.index.nodes.get(id);
while (node && !cache.has(node.backendNodeId)) {
if (work++ % 256 === 0) {
const pending = checkpoint();
if (pending) await pending;
}
if (
!document.index.ancestryComplete?.get(node.backendNodeId) ||
path.length >= document.index.nodes.size
) {
const incomplete = {
...roots.get(document.frame.frameId)!,
issue: roots.get(document.frame.frameId)!.issue ?? ("ancestry-incomplete" as const),
};
cache.set(node.backendNodeId, { self: incomplete, children: incomplete });
break;
}
path.push(node);
node =
node.parentBackendNodeId === null
? undefined
: document.index.nodes.get(node.parentBackendNodeId);
}
let parent = node
? cache.get(node.backendNodeId)!.children
: roots.get(document.frame.frameId)!;
for (let i = path.length - 1; i >= 0; i--) {
if (work++ % 256 === 0) {
const pending = checkpoint();
if (pending) await pending;
}
const current = path[i];
let self = parent;
let children = parent;
if (document.index.excludedBackendNodeIds.has(current.backendNodeId)) {
self = children = { ...parent, hidden: true };
} else if (current.nodeType === 1) {
if (!current.layout) {
// An element without layout is not evidence that descendants are hidden
// (e.g. a boxless ancestor). Keep the missing style evidence explicit.
self = children = { ...parent, issue: parent.issue ?? "facts-unavailable" };
} else if (document.identity) {
const ancestor = {
document: document.identity,
backendNodeId: current.backendNodeId,
styles: current.layout.styles,
clientBox: clientBox(current, document),
};
children = extendVisualContext(
parent,
ancestor,
current.tag !== "iframe" && current.tag !== "frame",
);
self = current.tag === "canvas" ? extendVisualContext(parent, ancestor, false) : children;
}
}
const result = { self, children };
cache.set(current.backendNodeId, result);
parent = children;
}
return (
cache.get(id) ?? {
self: { ...parent, issue: "ancestry-incomplete" },
children: { ...parent, issue: "ancestry-incomplete" },
}
);
}
// Process parent documents first, independent of snapshot/target completion order.
for (const document of facts.documents) {
const path: Document[] = [];
const visiting = new Set<string>();
let current: Document | undefined = document;
while (current && !roots.has(current.frame.frameId)) {
if (work++ % 256 === 0) {
const pending = checkpoint();
if (pending) await pending;
}
if (visiting.has(current.frame.frameId)) {
roots.set(current.frame.frameId, {
...EMPTY_VISUAL_CONTEXT,
issue: "ownership-unresolved",
});
contexts.set(current.frame.frameId, new Map());
break;
}
visiting.add(current.frame.frameId);
path.push(current);
current = current.frame.parentFrameId
? documents.get(current.frame.parentFrameId)
: undefined;
}
for (let i = path.length - 1; i >= 0; i--) {
const doc = path[i];
const { frame } = doc;
let root: VisualContext = roots.get(frame.frameId) ?? EMPTY_VISUAL_CONTEXT;
if (frame.parentFrameId) {
const parent = documents.get(frame.parentFrameId);
if (
!parent ||
!roots.has(parent.frame.frameId) ||
frame.ownerBackendNodeId === undefined ||
!parent.identity
) {
root = { ...root, issue: "ownership-unresolved" };
} else {
const owner = parent.index.nodes.get(frame.ownerBackendNodeId);
const ownerState = await nodeContext(parent, frame.ownerBackendNodeId);
root = {
...ownerState.children,
// CSS visibility inside a child document cannot override its hidden owner.
hidden:
ownerState.children.hidden ||
owner?.layout?.styles.visibility === "hidden" ||
owner?.layout?.styles.visibility === "collapse",
// Frame scaling is already interpreted by the shared content-quad projection.
transformed: false,
localClip: false,
unpositionedClip: false,
};
}
} else if (frame.frameId !== facts.rootFrameId)
root = { ...root, issue: "ownership-unresolved" };
root = {
...root,
issue:
root.issue ??
(!doc.identity ? "identity-unverified" : visualProjectionIssue(doc.geometry)),
};
roots.set(frame.frameId, root);
contexts.set(frame.frameId, new Map());
}
}
for (const document of facts.documents) {
for (const node of document.index.nodes.values()) {
if (work++ % 256 === 0) {
const pending = checkpoint();
if (pending) await pending;
}
if (node.tag !== "canvas" || document.index.excludedBackendNodeIds.has(node.backendNodeId))
continue;
// No layout/zero size is positive evidence of no screenshot area for this node.
if (!node.layout) continue;
const bounds = node.layout.bounds;
if (
bounds?.length === 4 &&
bounds.every(Number.isFinite) &&
(bounds[2] <= 0 || bounds[3] <= 0)
)
continue;
const context = (await nodeContext(document, node.backendNodeId)).self;
const local = viewportRect(node.localRect);
const borderBox =
local && document.geometry ? projectVisualBox(local, document.geometry.projections) : null;
const region = resolveVisualRegion({
borderBox,
frameVisibleBox: viewportRect(node.rect),
context,
visibility: node.layout.styles.visibility,
});
if (region.status === "unavailable") {
issues.push({
target: document.frame.target,
frameId: document.frame.frameId,
backendNodeId: node.backendNodeId,
reason: region.reason,
});
} else if (region.status === "available" && document.identity) {
const label = node.attrs["aria-label"]?.trim() || node.attrs.title?.trim();
candidates.push({
document: document.identity,
backendNodeId: node.backendNodeId,
parentBackendNodeId: node.parentBackendNodeId,
...(label ? { label } : {}),
region,
});
}
}
}
// AX-only/missing documents have no Canvas nodes to carry the missing evidence.
const reportedFrames = new Set(issues.map((issue) => issue.frameId));
for (const document of facts.documents) {
const reason = roots.get(document.frame.frameId)?.issue;
if (
reason &&
!reportedFrames.has(document.frame.frameId) &&
(!document.identity || !document.index.nodes.size)
)
issues.push({ target: document.frame.target, frameId: document.frame.frameId, reason });
}
const pending = checkpoint();
if (pending) await pending;
return {
candidates,
issues,
captureIssues: facts.issues,
complete: issues.length === 0 && facts.issues.length === 0,
};
}
@@ -0,0 +1,318 @@
import {
type GeometryProjection,
type Polygon,
projectPolygon,
rectPolygon,
regionBounds,
type ViewportRect,
} from "../geometry";
import type { DocumentGeometry, DocumentIdentity } from "./facts";
export type VisualIssueReason =
| "ancestry-incomplete"
| "facts-unavailable"
| "geometry-unavailable"
| "geometry-unsupported"
| "identity-unverified"
| "ownership-unresolved";
/** Top viewport CSS coordinates. Unconstrained axes have infinite bounds. */
export interface VisualClip {
readonly left: number;
readonly top: number;
readonly right: number;
readonly bottom: number;
}
/** Shared persistent chain: one entry per clipping ancestor, not per Canvas. */
export interface VisualClipSource {
readonly document: DocumentIdentity;
readonly backendNodeId: number;
readonly x: boolean;
readonly y: boolean;
readonly overflowX: string;
readonly overflowY: string;
readonly box: ViewportRect;
readonly parent?: VisualClipSource;
}
export interface VisualContext {
readonly clip: VisualClip;
readonly clips?: VisualClipSource;
readonly hidden: boolean;
/** A local CSS scale/zoom makes snapshot client offsets ambiguous. */
readonly transformed: boolean;
readonly localClip?: boolean;
/** Overflow between an absolute target and its nearest positioned ancestor. */
readonly unpositionedClip?: boolean;
readonly issue?: VisualIssueReason;
}
export const EMPTY_VISUAL_CONTEXT: VisualContext = {
clip: { left: -Infinity, top: -Infinity, right: Infinity, bottom: Infinity },
hidden: false,
transformed: false,
};
function axisRect(polygon: Polygon): boolean {
if (polygon.length !== 4 || polygon.some((p) => !Number.isFinite(p.x) || !Number.isFinite(p.y)))
return false;
const [a, b, c, d] = polygon;
return a.y === b.y && b.x === c.x && c.y === d.y && d.x === a.x && b.x > a.x && d.y > a.y;
}
/** Check evidence before projectRectToViewport can reduce a polygon to its bounds. */
export function visualProjectionIssue(
geometry: DocumentGeometry | undefined,
): VisualIssueReason | undefined {
if (!geometry || !geometry.projections.length || geometry.pageScale === undefined)
return "geometry-unavailable";
if (!Number.isFinite(geometry.pageScale) || geometry.pageScale <= 0)
return "geometry-unavailable";
if (geometry.pageScale !== 1) return "geometry-unsupported";
const { layoutUnitsPerCssPixel, scrollCss } = geometry.coordinates;
if (
!Number.isFinite(layoutUnitsPerCssPixel) ||
layoutUnitsPerCssPixel <= 0 ||
![scrollCss.x, scrollCss.y].every(Number.isFinite)
)
return "geometry-unavailable";
for (const projection of geometry.projections) {
if (
![projection.topViewport.width, projection.topViewport.height].every(
(n) => Number.isFinite(n) && n > 0,
)
)
return "geometry-unavailable";
if (projection.sourceClips.some((clip) => !axisRect(clip))) return "geometry-unsupported";
for (const edge of projection.edges) {
if (
![edge.sourceViewport.width, edge.sourceViewport.height].every(
(n) => Number.isFinite(n) && n > 0,
)
)
return "geometry-unavailable";
if (!axisRect(edge.destinationQuad) || edge.destinationClips?.some((clip) => !axisRect(clip)))
return "geometry-unsupported";
}
}
}
/** Uses the same projection math as DOM geometry, without clipping away box origins. */
export function projectVisualBox(
box: ViewportRect,
projections: readonly GeometryProjection[],
): ViewportRect | null {
let polygon: Polygon = rectPolygon({ x: box.x, y: box.y, w: box.width, h: box.height });
for (const projection of projections)
for (const edge of projection.edges) polygon = projectPolygon(polygon, edge);
// Preserve zero-size client boxes: they can prove a fully clipped axis.
if (box.width === 0 || box.height === 0) {
const [a, , c] = polygon;
return { x: a.x, y: a.y, width: c.x - a.x, height: c.y - a.y };
}
return regionBounds([polygon]);
}
function axisTransformScale(value: string): [number, number] | null {
if (value === "none") return [1, 1];
const match = /^matrix(3d)?\(([^)]+)\)$/.exec(value);
if (!match) return null;
const values = match[2].split(",").map((part) => Number(part.trim()));
if (!values.every(Number.isFinite)) return null;
if (match[1])
return values.length === 16 &&
values[0] > 0 &&
values[5] > 0 &&
values[10] === 1 &&
values[15] === 1 &&
[1, 2, 3, 4, 6, 7, 8, 9, 11, 14].every((index) => values[index] === 0)
? [values[0], values[5]]
: null;
return values.length === 6 && values[0] > 0 && values[3] > 0 && values[1] === 0 && values[2] === 0
? [values[0], values[3]]
: null;
}
export interface VisualAncestor {
readonly document: DocumentIdentity;
readonly backendNodeId: number;
readonly styles: Readonly<Record<string, string>>;
/** Already in top viewport CSS units, from a source-specific adapter. */
readonly clientBox?: ViewportRect | null;
}
/** One ancestor step, reusable by discovery and the future live local adapter. */
export function extendVisualContext(
parent: VisualContext,
node: VisualAncestor,
clipContents = true,
): VisualContext {
const styles = node.styles;
const opacity = styles.opacity?.trim() ? Number(styles.opacity) : NaN;
const zoom = styles.zoom === "normal" ? 1 : styles.zoom?.trim() ? Number(styles.zoom) : NaN;
const scale =
styles.scale === "none" ? [1] : (styles.scale ?? "").trim().split(/\s+/).map(Number);
const scaleSupported =
scale.length >= 1 &&
scale.length <= 3 &&
scale.every((n) => Number.isFinite(n) && n > 0) &&
(scale.length < 3 || scale[2] === 1);
const transformScale = axisTransformScale(styles.transform ?? "");
// Identity/translation changes origins, which bounds already capture, but
// leaves client offsets in the same units. Only scaling makes them ambiguous.
const transformed =
parent.transformed ||
transformScale?.some((n) => n !== 1) === true ||
zoom !== 1 ||
scale.some((n) => n !== 1);
const hidden = parent.hidden || opacity === 0 || styles.display === "none";
let issue = parent.issue;
if (
!Number.isFinite(opacity) ||
!Number.isFinite(zoom) ||
zoom <= 0 ||
!styles.display ||
!styles.visibility
)
issue ??= "facts-unavailable";
if (!styles.transform || !styles["clip-path"] || !styles["mask-image"])
issue ??= "facts-unavailable";
else if (!transformScale || styles["clip-path"] !== "none" || styles["mask-image"] !== "none")
issue ??= "geometry-unsupported";
if (
!["rotate", "scale", "perspective", "clip", "contain", "overflow-clip-margin"].every(
(key) => styles[key],
)
)
issue ??= "facts-unavailable";
else if (
!scaleSupported ||
!["none", "0deg"].includes(styles.rotate) ||
styles.perspective !== "none" ||
styles.clip !== "auto" ||
/(?:^|\s)(paint|strict|content)(?:$|\s)/.test(styles.contain)
)
issue ??= "geometry-unsupported";
// Do not pretend every DOM ancestor clips out-of-flow descendants. Support
// the ordinary positioned-container case; other containing-block cases stay explicit.
if (
(styles.position === "absolute" && parent.unpositionedClip) ||
(styles.position === "fixed" && parent.localClip)
)
issue ??= "geometry-unsupported";
const overflow = [styles["overflow-x"], styles["overflow-y"]];
if (clipContents && overflow.includes("clip") && styles["overflow-clip-margin"] !== "0px")
issue ??= "geometry-unsupported";
if (
overflow.some(
(value) => !["visible", "hidden", "clip", "scroll", "auto", "overlay"].includes(value),
)
)
issue ??= "facts-unavailable";
const [x, y] = overflow.map((value) => value !== "visible");
let clip = parent.clip;
let clips = parent.clips;
if (clipContents && (x || y)) {
if (transformed) issue ??= "geometry-unsupported";
const box = node.clientBox;
if (
!box ||
![box.x, box.y, box.width, box.height].every(Number.isFinite) ||
box.width < 0 ||
box.height < 0
)
issue ??= "geometry-unavailable";
else {
clip = {
left: x ? Math.max(clip.left, box.x) : clip.left,
right: x ? Math.min(clip.right, box.x + box.width) : clip.right,
top: y ? Math.max(clip.top, box.y) : clip.top,
bottom: y ? Math.min(clip.bottom, box.y + box.height) : clip.bottom,
};
clips = {
document: node.document,
backendNodeId: node.backendNodeId,
x,
y,
overflowX: styles["overflow-x"],
overflowY: styles["overflow-y"],
box,
parent: clips,
};
}
}
const clipsHere = clipContents && (x || y);
return {
hidden,
transformed,
issue,
clip,
clips,
localClip: parent.localClip || clipsHere,
unpositionedClip:
styles.position !== "static" ||
styles.transform !== "none" ||
/(?:^|\s)layout(?:$|\s)/.test(styles.contain ?? "")
? false
: parent.unpositionedClip || clipsHere,
};
}
/** A page screenshot rectangle, not an exact mask of visible Canvas pixels.
* Rounded corners are rendered by the browser and do not invalidate this range. */
export type VisualRegionResult =
| { status: "available"; borderBox: ViewportRect; crop: ViewportRect; clips?: VisualClipSource }
| { status: "empty" }
| { status: "unavailable"; reason: VisualIssueReason };
/** Box and frameVisibleBox must come from the same box kind (Canvas border-box).
* Frame/viewport clipping is supplied by the shared geometry adapter. */
export function resolveVisualRegion(input: {
borderBox: ViewportRect | null;
frameVisibleBox: ViewportRect | null;
context: VisualContext;
visibility: string;
}): VisualRegionResult {
const { borderBox, frameVisibleBox, context, visibility } = input;
if (context.hidden || visibility === "hidden" || visibility === "collapse")
return { status: "empty" };
if (context.issue) return { status: "unavailable", reason: context.issue };
if (visibility !== "visible") return { status: "unavailable", reason: "facts-unavailable" };
if (!borderBox) return { status: "unavailable", reason: "geometry-unavailable" };
if (!frameVisibleBox) return { status: "empty" };
if (
![
borderBox.x,
borderBox.y,
borderBox.width,
borderBox.height,
frameVisibleBox.x,
frameVisibleBox.y,
frameVisibleBox.width,
frameVisibleBox.height,
].every(Number.isFinite)
)
return { status: "unavailable", reason: "geometry-unavailable" };
if (
borderBox.width <= 0 ||
borderBox.height <= 0 ||
frameVisibleBox.width < 0 ||
frameVisibleBox.height < 0 ||
Object.values(context.clip).some(Number.isNaN)
)
return { status: "unavailable", reason: "geometry-unavailable" };
const x = Math.max(frameVisibleBox.x, context.clip.left);
const y = Math.max(frameVisibleBox.y, context.clip.top);
const right = Math.min(frameVisibleBox.x + frameVisibleBox.width, context.clip.right);
const bottom = Math.min(frameVisibleBox.y + frameVisibleBox.height, context.clip.bottom);
return right > x && bottom > y
? {
status: "available",
borderBox,
crop: { x, y, width: right - x, height: bottom - y },
clips: context.clips,
}
: { status: "empty" };
}
+54
View File
@@ -24,6 +24,7 @@ export type RpcErrorReason =
| "agent_window_scope"
| "element_not_visible"
| "ref_not_found"
| "ref_kind_unsupported"
| "selector_not_found"
| "target_not_fillable"
| "fill_value_invalid"
@@ -470,6 +471,59 @@ export interface HoverResult {
dialogs?: JavaScriptDialogInfo[];
}
export interface ScrollToParams {
session_id: string;
ref?: string;
selector?: string;
tab_id?: number;
timeout_ms?: number;
}
export interface ScrollToResult {
tab_id: number;
used_ref?: string;
used_selector?: string;
/** Clipped border-box bounds in top-level viewport CSS pixels; not an occlusion test. */
x: number;
y: number;
width: number;
height: number;
dialogs?: JavaScriptDialogInfo[];
}
export interface FocusParams {
session_id: string;
ref?: string;
selector?: string;
tab_id?: number;
timeout_ms?: number;
}
export interface FocusResult {
tab_id: number;
used_ref?: string;
used_selector?: string;
focused: boolean;
dialogs?: JavaScriptDialogInfo[];
}
export interface BlurParams {
session_id: string;
ref?: string;
selector?: string;
tab_id?: number;
timeout_ms?: number;
}
export interface BlurResult {
tab_id: number;
used_ref?: string;
used_selector?: string;
was_focused: boolean;
focused: boolean;
dialogs?: JavaScriptDialogInfo[];
}
export interface FillParams {
session_id: string;
value: string;
+12 -2
View File
@@ -55,10 +55,20 @@ Use this default loop:
```text
bsk navigate <url> --session <id>
bsk observe --session <id>
bsk click|hover|fill|select|press ... --session <id>
bsk click|hover|scroll-to|focus|blur|fill|select|press ... --session <id>
bsk observe --session <id> # after navigation or a meaningful DOM change
```
`bsk scroll-to <ref-or-selector> --session <id>` scrolls an element and its frame owners into view.
Use a fresh element ref for iframe/shadow-root targets; CSS selectors search the main document.
The result is the visible border-box portion's bounds in top-level viewport CSS pixels after
ancestor clipping. Partial visibility is enough; hidden or fully clipped targets fail with
`permission_denied` and `data.reason=element_not_visible`. This does not test occlusion by other elements.
For a specific tab or deadline: `bsk scroll-to @e3 --session <id> --tab-id 42 --timeout 5s`.
`bsk focus <ref>` explicitly focuses a target; `bsk blur <ref>` removes focus and reports whether
it was focused. Use these for UI states triggered by focus changes.
Prefer fresh `@eN` refs over CSS selectors. Navigation invalidates refs; large DOM changes may also
make them stale. Observe again before the next interaction.
@@ -114,7 +124,7 @@ This list of names is complete. Never invent a command outside it; read
session start|stop|list browsers status doctor update logs
navigate navigate-back navigate-forward reload wait-for-navigation wait-ms
observe snapshot get-html screenshot console network
click hover fill select press evaluate
click hover scroll-to focus blur fill select press evaluate
tab list|create|close|select|borrow|return window resize emulate
upload download request-help record start|stop
```
+103 -6
View File
@@ -73,6 +73,14 @@ where
P: Serialize + Send + 'static,
R: DeserializeOwned + Send + 'static,
{
let parent_cancel = stdin_cancellation();
if parent_cancel.as_ref().is_some_and(|rx| *rx.borrow()) {
return Err(CliError::from_rpc(RpcError {
code: ErrorCode::Cancelled,
message: "parent closed the cancellation pipe".into(),
data: Some(serde_json::json!({"reason": "cancelled_before_dispatch"})),
}));
}
let rpc_id: RpcId = format!("{}-{}", rpc_id_prefix, random_short_id());
let mut client = IpcClient::connect(sock.clone()).await?;
let rpc_id_for_call = rpc_id.clone();
@@ -88,16 +96,16 @@ where
let outcome = tokio::select! {
biased;
res = &mut main_fut => res?,
sig = wait_for_sigint() => {
sig.context("install SIGINT handler").map_err(CliError::Local)?;
debug!(rpc_id = %rpc_id_for_cancel, "SIGINT: forwarding cancel to daemon");
sig = wait_for_cancel(parent_cancel) => {
sig.context("wait for cancellation").map_err(CliError::Local)?;
debug!(rpc_id = %rpc_id_for_cancel, "forwarding cancel to daemon");
let _ = send_cancel(&sock, &rpc_id_for_cancel).await;
match tokio::time::timeout(CANCEL_RESPONSE_GRACE, &mut main_fut).await {
Ok(res) => res?,
Err(_) => {
debug!(
rpc_id = %rpc_id_for_cancel,
"SIGINT: cancel grace elapsed; synthesising cancelled error"
"cancel grace elapsed; synthesising cancelled error"
);
return Err(CliError::from_rpc(RpcError {
code: ErrorCode::Cancelled,
@@ -112,6 +120,46 @@ where
outcome.map_err(CliError::from_rpc)
}
/// Release caller-owned staging even after cancellation. This deliberately
/// exposes only transfer.release, not a general bypass for business RPCs.
/// The entire batch, including connection setup, shares one bounded budget.
pub fn release_transfers<'a>(
sock: &Path,
transfer_ids: impl IntoIterator<Item = &'a str>,
) -> Result<(), CliError> {
use bsk_protocol::tools::{TransferIdParams, TransferReleaseResult};
const CLEANUP_TIMEOUT: Duration = Duration::from_secs(5);
let ids: Vec<_> = transfer_ids.into_iter().collect();
if ids.is_empty() {
return Ok(());
}
let rt = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.context("build runtime for transfer cleanup")?;
rt.block_on(async {
tokio::time::timeout(CLEANUP_TIMEOUT, async {
let mut client = IpcClient::connect(sock).await?;
for id in ids {
client
.call::<_, TransferReleaseResult>(
"transfer-release",
Method::TransferRelease,
Some(TransferIdParams {
transfer_id: id.to_string(),
}),
CLEANUP_TIMEOUT,
)
.await?
.map_err(CliError::from_rpc)?;
}
Ok::<_, CliError>(())
})
.await
.context("transfer cleanup exceeded its total time budget")?
})
}
/// Send a `cancel { rpc_id }` frame over a fresh connection so it
/// lands on the daemon while the original call is still parked.
///
@@ -146,8 +194,57 @@ fn random_short_id() -> String {
bytes.iter().map(|b| format!("{b:02x}")).collect()
}
/// Wait for either Ctrl-C or process SIGTERM. Returning here means
/// the user has asked us to stop.
/// Opt-in control pipe for parent processes that cannot send Ctrl-C (Node on
/// Windows). One reader survives the short-lived runtimes used by multi-RPC
/// commands such as upload. Ordinary CLI stdin and terminal Ctrl-C are unchanged.
fn stdin_cancellation() -> Option<tokio::sync::watch::Receiver<bool>> {
if std::env::var("BSK_CANCEL_ON_STDIN_CLOSE").as_deref() != Ok("1") {
return None;
}
static CANCEL: std::sync::OnceLock<tokio::sync::watch::Receiver<bool>> =
std::sync::OnceLock::new();
Some(
CANCEL
.get_or_init(|| {
let (tx, rx) = tokio::sync::watch::channel(false);
std::thread::spawn(move || {
use std::io::Read;
let mut stdin = std::io::stdin().lock();
let mut buf = [0; 64];
loop {
match stdin.read(&mut buf) {
Ok(0) => break,
Ok(_) => {}
Err(err) if err.kind() == std::io::ErrorKind::Interrupted => continue,
Err(_) => break,
}
}
let _ = tx.send(true);
});
rx
})
.clone(),
)
}
async fn wait_for_cancel(
parent_cancel: Option<tokio::sync::watch::Receiver<bool>>,
) -> anyhow::Result<()> {
let parent_closed = async move {
match parent_cancel {
Some(mut rx) => {
let _ = rx.wait_for(|closed| *closed).await;
}
None => std::future::pending::<()>().await,
}
};
tokio::select! {
result = wait_for_sigint() => result,
_ = parent_closed => Ok(()),
}
}
/// Wait for terminal Ctrl-C.
async fn wait_for_sigint() -> anyhow::Result<()> {
tokio::signal::ctrl_c().await.context("listen for SIGINT")?;
Ok(())
+2 -9
View File
@@ -9,8 +9,7 @@ use anyhow::Context;
use base64::Engine;
use bsk_protocol::Method;
use bsk_protocol::tools::{
DownloadParams, DownloadResult, TransferChunkParams, TransferChunkResult, TransferIdParams,
TransferReleaseResult,
DownloadParams, DownloadResult, TransferChunkParams, TransferChunkResult,
};
use clap::Args;
use uuid::Uuid;
@@ -70,13 +69,7 @@ pub fn dispatch(args: DownloadArgs, format: Format) -> Result<(), CliError> {
CliError::Local(anyhow::anyhow!("daemon returned no download transfer id"))
})?;
let write_result = write_transfer(&info.sock_path, &transfer_id, &args.out, args.overwrite);
let _: Result<TransferReleaseResult, CliError> = crate::cli::business_rpc::call(
info.sock_path,
"transfer-release",
Method::TransferRelease,
Some(TransferIdParams { transfer_id }),
Duration::from_secs(5),
);
let _ = crate::cli::business_rpc::release_transfers(&info.sock_path, [transfer_id.as_str()]);
write_result?;
match format {
Format::Json => {
+123 -3
View File
@@ -1,4 +1,5 @@
//! `bsk click` / `bsk fill` / `bsk press` (M7 interaction tools).
//! `bsk click` / `bsk focus` / `bsk blur` / `bsk fill` / `bsk press`
//! interaction tools.
//!
//! The `<target>` positional accepts either a snapshot ref (`@e3`,
//! `e3`) or a CSS selector. Use `--ref` / `--selector` explicitly when
@@ -10,8 +11,9 @@ use std::time::Duration;
use anyhow::Context;
use bsk_protocol::Method;
use bsk_protocol::tools::{
ClickParams, ClickResult, FillParams, FillResult, HoverParams, HoverResult, KeyModifier,
MouseButton, PressParams, PressResult, SelectParams, SelectResult,
BlurParams, BlurResult, ClickParams, ClickResult, FillParams, FillResult, FocusParams,
FocusResult, HoverParams, HoverResult, KeyModifier, MouseButton, PressParams, PressResult,
SelectParams, SelectResult,
};
use clap::{Args, ValueEnum};
@@ -274,6 +276,124 @@ pub fn dispatch_hover(args: HoverArgs, format: Format) -> Result<(), CliError> {
Ok(())
}
// ---------------------------------------------------------------------------
// bsk focus / bsk blur
// ---------------------------------------------------------------------------
#[derive(Debug, Clone, Args)]
pub struct FocusArgs {
/// Snapshot ref (`@e3`, `e3`) or CSS selector. Optional when `--ref`/`--selector` is used.
pub target: Option<String>,
#[arg(long = "ref")]
pub ref_: Option<String>,
#[arg(long = "selector")]
pub selector: Option<String>,
#[arg(long)]
pub session: String,
#[arg(long = "tab-id")]
pub tab_id: Option<i64>,
#[arg(long, default_value = "30s", value_parser = parse_timeout_ms)]
pub timeout: u32,
}
pub fn dispatch_focus(args: FocusArgs, format: Format) -> Result<(), CliError> {
let info = ensure_daemon().context("ensure daemon is running")?;
let (ref_, selector) = split_target(args.target, args.ref_, args.selector)?;
let params = FocusParams {
session_id: args.session,
ref_,
selector,
tab_id: args.tab_id,
timeout_ms: Some(args.timeout),
};
let reply: FocusResult = call(
info.sock_path,
Method::ToolFocus,
params,
"focus-1",
args.timeout,
)?;
match format {
Format::Json => println!(
"{}",
serde_json::to_string_pretty(&reply)
.map_err(|e| CliError::Local(anyhow::anyhow!(e)))?
),
Format::Human => {
let target =
format_used_target(reply.used_ref.as_deref(), reply.used_selector.as_deref());
println!(
"focus ok tab={} target={target} focused={}",
reply.tab_id, reply.focused
);
print_dialog_summaries(&reply.dialogs);
}
}
Ok(())
}
#[derive(Debug, Clone, Args)]
pub struct BlurArgs {
/// Snapshot ref (`@e3`, `e3`) or CSS selector. Optional when `--ref`/`--selector` is used.
pub target: Option<String>,
#[arg(long = "ref")]
pub ref_: Option<String>,
#[arg(long = "selector")]
pub selector: Option<String>,
#[arg(long)]
pub session: String,
#[arg(long = "tab-id")]
pub tab_id: Option<i64>,
#[arg(long, default_value = "30s", value_parser = parse_timeout_ms)]
pub timeout: u32,
}
pub fn dispatch_blur(args: BlurArgs, format: Format) -> Result<(), CliError> {
let info = ensure_daemon().context("ensure daemon is running")?;
let (ref_, selector) = split_target(args.target, args.ref_, args.selector)?;
let params = BlurParams {
session_id: args.session,
ref_,
selector,
tab_id: args.tab_id,
timeout_ms: Some(args.timeout),
};
let reply: BlurResult = call(
info.sock_path,
Method::ToolBlur,
params,
"blur-1",
args.timeout,
)?;
match format {
Format::Json => println!(
"{}",
serde_json::to_string_pretty(&reply)
.map_err(|e| CliError::Local(anyhow::anyhow!(e)))?
),
Format::Human => {
let target =
format_used_target(reply.used_ref.as_deref(), reply.used_selector.as_deref());
println!(
"blur ok tab={} target={target} was_focused={} focused={}",
reply.tab_id, reply.was_focused, reply.focused
);
print_dialog_summaries(&reply.dialogs);
}
}
Ok(())
}
// ---------------------------------------------------------------------------
// bsk fill
// ---------------------------------------------------------------------------
+15 -1
View File
@@ -28,6 +28,7 @@ pub mod record_recovery;
pub mod record_state;
pub mod render_error;
pub mod screenshot;
pub mod scroll;
pub mod session;
pub mod snapshot;
pub mod status;
@@ -47,12 +48,15 @@ use crate::cli::evaluate::EvaluateArgs;
use crate::cli::get_html::GetHtmlArgs;
use crate::cli::human_loop::RequestHelpArgs;
use crate::cli::install_skill::InstallSkillArgs;
use crate::cli::interaction::{ClickArgs, FillArgs, HoverArgs, PressArgs, SelectArgs};
use crate::cli::interaction::{
BlurArgs, ClickArgs, FillArgs, FocusArgs, HoverArgs, PressArgs, SelectArgs,
};
use crate::cli::navigate::{NavigateCommand, NavigateHistoryArgs, ReloadArgs};
use crate::cli::network::NetworkArgs;
use crate::cli::observe::ObserveArgs;
use crate::cli::record::RecordCmd;
use crate::cli::screenshot::ScreenshotArgs;
use crate::cli::scroll::ScrollToArgs;
use crate::cli::session::SessionCmd;
use crate::cli::snapshot::SnapshotArgs;
use crate::cli::tab::TabCmd;
@@ -173,6 +177,16 @@ pub enum Command {
/// Hover a snapshot ref or CSS selector.
Hover(HoverArgs),
/// Scroll a snapshot ref or CSS selector into the visible viewport.
#[command(name = "scroll-to")]
ScrollTo(ScrollToArgs),
/// Focus a snapshot ref or CSS selector.
Focus(FocusArgs),
/// Remove focus from a snapshot ref or CSS selector.
Blur(BlurArgs),
/// Fill an input / textarea / contenteditable.
Fill(FillArgs),
+17
View File
@@ -40,6 +40,7 @@ use bsk_protocol::ErrorCode;
pub mod reason {
pub const AGENT_WINDOW_SCOPE: &str = "agent_window_scope";
pub const ELEMENT_NOT_VISIBLE: &str = "element_not_visible";
pub const REF_KIND_UNSUPPORTED: &str = "ref_kind_unsupported";
pub const REF_NOT_FOUND: &str = "ref_not_found";
pub const SELECTOR_NOT_FOUND: &str = "selector_not_found";
pub const TARGET_NOT_FILLABLE: &str = "target_not_fillable";
@@ -247,6 +248,13 @@ pub fn info_for_error(code: ErrorCode, data: Option<&serde_json::Value>) -> Rend
),
exit_code: base.exit_code,
},
(ErrorCode::Unsupported, reason::REF_KIND_UNSUPPORTED) => RenderInfo {
summary: "this tool does not support the ref target type",
hint: Some(
"use a DOM ref for DOM operations; visual-region refs require an available visual screenshot path",
),
exit_code: base.exit_code,
},
(ErrorCode::NotFound, reason::REF_NOT_FOUND) => RenderInfo {
summary: "snapshot ref was not found for this tab",
hint: Some("rerun `bsk snapshot` for the current tab and use one of the returned refs"),
@@ -664,6 +672,15 @@ mod tests {
assert!(info.summary.contains("timed out after browser dispatch"));
}
#[test]
fn visual_ref_type_error_has_specific_guidance() {
let data = serde_json::json!({ "reason": reason::REF_KIND_UNSUPPORTED });
let info = info_for_error(ErrorCode::Unsupported, Some(&data));
assert!(info.summary.contains("ref target type"));
assert!(info.hint.unwrap().contains("DOM ref"));
assert_eq!(info.exit_code, info_for(ErrorCode::Unsupported).exit_code);
}
#[test]
fn ref_not_found_overrides_not_found_copy() {
let data = serde_json::json!({ "reason": reason::REF_NOT_FOUND });
+100
View File
@@ -0,0 +1,100 @@
//! `bsk scroll-to` — bring a target element into the visible viewport.
use std::time::Duration;
use anyhow::Context;
use bsk_protocol::Method;
use bsk_protocol::tools::{ScrollToParams, ScrollToResult};
use clap::Args;
use crate::cli::dialogs::print_dialog_summaries;
use crate::cli::ensure_daemon::ensure_daemon;
use crate::cli::error::{CliError, Format};
use crate::cli::interaction::split_target;
use crate::cli::navigate::parse_timeout_ms;
#[derive(Debug, Clone, Args)]
pub struct ScrollToArgs {
/// Snapshot ref (`@e3`, `e3`) or CSS selector. Optional when `--ref`/`--selector` is used.
pub target: Option<String>,
#[arg(long = "ref")]
pub ref_: Option<String>,
#[arg(long = "selector")]
pub selector: Option<String>,
#[arg(long)]
pub session: String,
#[arg(long = "tab-id")]
pub tab_id: Option<i64>,
#[arg(long, default_value = "30s", value_parser = parse_timeout_ms)]
pub timeout: u32,
}
pub fn dispatch(args: ScrollToArgs, format: Format) -> Result<(), CliError> {
let (ref_, selector) = split_target(args.target, args.ref_, args.selector)?;
let info = ensure_daemon().context("ensure daemon is running")?;
let params = ScrollToParams {
session_id: args.session,
ref_,
selector,
tab_id: args.tab_id,
timeout_ms: Some(args.timeout),
};
let reply: ScrollToResult = crate::cli::business_rpc::call(
info.sock_path,
"scroll-to-1",
Method::ToolScrollTo,
Some(params),
ipc_timeout(args.timeout),
)?;
match format {
Format::Json => println!(
"{}",
serde_json::to_string_pretty(&reply)
.map_err(|e| CliError::Local(anyhow::anyhow!(e)))?
),
Format::Human => {
let target =
format_used_target(reply.used_ref.as_deref(), reply.used_selector.as_deref());
println!(
"scroll-to ok tab={} target={target} bounds=({}, {}, {}, {})",
reply.tab_id, reply.x, reply.y, reply.width, reply.height
);
print_dialog_summaries(&reply.dialogs);
}
}
Ok(())
}
fn format_used_target(used_ref: Option<&str>, used_selector: Option<&str>) -> String {
used_ref
.map(|r| format!("@{r}"))
.or_else(|| used_selector.map(str::to_string))
.unwrap_or_else(|| "?".into())
}
fn ipc_timeout(timeout_ms: u32) -> Duration {
Duration::from_millis(u64::from(timeout_ms))
.checked_add(Duration::from_secs(15))
.unwrap_or(Duration::from_secs(u64::from(timeout_ms / 1_000) + 15))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn split_target_detects_refs_and_selectors() {
let (ref_, selector) = split_target(Some("@e3".into()), None, None).unwrap();
assert_eq!(ref_.as_deref(), Some("@e3"));
assert!(selector.is_none());
let (ref_, selector) = split_target(Some("#target".into()), None, None).unwrap();
assert!(ref_.is_none());
assert_eq!(selector.as_deref(), Some("#target"));
}
}
+48 -51
View File
@@ -10,8 +10,7 @@ use base64::Engine;
use bsk_protocol::Method;
use bsk_protocol::tools::{
TransferBeginParams, TransferBeginResult, TransferChunkParams, TransferChunkResult,
TransferIdParams, TransferReadyResult, TransferReleaseResult, UploadFile, UploadMode,
UploadParams, UploadResult,
TransferIdParams, TransferReadyResult, UploadFile, UploadMode, UploadParams, UploadResult,
};
use clap::{Args, ValueEnum};
@@ -69,14 +68,12 @@ pub fn dispatch(args: UploadArgs, format: Format) -> Result<(), CliError> {
let (ref_, selector) = split_target(args.target, args.ref_, args.selector)?;
let mut staged = Vec::new();
for path in &args.files {
match stage_file(&info.sock_path, &args.session, path) {
Ok(file) => staged.push(file),
Err(err) => {
for (id, _) in &staged {
let _ = release(&info.sock_path, id);
}
return Err(err);
}
if let Err(err) = stage_file(&info.sock_path, &args.session, path, &mut staged) {
let _ = crate::cli::business_rpc::release_transfers(
&info.sock_path,
staged.iter().map(|(id, _)| id.as_str()),
);
return Err(err);
}
}
let params = UploadParams {
@@ -102,6 +99,17 @@ pub fn dispatch(args: UploadArgs, format: Format) -> Result<(), CliError> {
Some(params),
ipc_timeout(args.timeout),
);
// The helper can reject cancellation before handing any request to the
// daemon. In that case staging still belongs to this CLI.
if result.as_ref().is_err_and(|err| {
err.data()
.is_some_and(|data| data["reason"] == "cancelled_before_dispatch")
}) {
let _ = crate::cli::business_rpc::release_transfers(
&info.sock_path,
staged.iter().map(|(id, _)| id.as_str()),
);
}
// Once tool.upload is dispatched, staging ownership belongs to the
// session. A transport timeout cannot prove that Chrome did not attach
// the file, so releasing here could invalidate a late successful attach.
@@ -118,7 +126,12 @@ pub fn dispatch(args: UploadArgs, format: Format) -> Result<(), CliError> {
Ok(())
}
fn stage_file(sock: &Path, session: &str, path: &PathBuf) -> Result<(String, String), CliError> {
fn stage_file(
sock: &Path,
session: &str,
path: &PathBuf,
staged: &mut Vec<(String, String)>,
) -> Result<(), CliError> {
let mut file = File::open(path)
.with_context(|| format!("open upload file {}", path.display()))
.map_err(CliError::Local)?;
@@ -149,55 +162,39 @@ fn stage_file(sock: &Path, session: &str, path: &PathBuf) -> Result<(String, Str
}),
Duration::from_secs(10),
)?;
let staged = (|| {
let mut offset = 0u64;
let mut buf = vec![0u8; begin.chunk_size as usize];
loop {
let n = file.read(&mut buf).map_err(|e| CliError::Local(e.into()))?;
if n == 0 {
break;
}
let reply: TransferChunkResult = crate::cli::business_rpc::call(
sock.to_path_buf(),
"transfer-chunk",
Method::TransferChunk,
Some(TransferChunkParams {
transfer_id: begin.transfer_id.clone(),
offset,
data_base64: base64::engine::general_purpose::STANDARD.encode(&buf[..n]),
}),
Duration::from_secs(30),
)?;
offset = reply.next_offset;
// Record ownership immediately so one error path releases both completed
// files and the current partially staged file.
staged.push((begin.transfer_id.clone(), name));
let mut offset = 0u64;
let mut buf = vec![0u8; begin.chunk_size as usize];
loop {
let n = file.read(&mut buf).map_err(|e| CliError::Local(e.into()))?;
if n == 0 {
break;
}
let _: TransferReadyResult = crate::cli::business_rpc::call(
let reply: TransferChunkResult = crate::cli::business_rpc::call(
sock.to_path_buf(),
"transfer-finish",
Method::TransferFinish,
Some(TransferIdParams {
"transfer-chunk",
Method::TransferChunk,
Some(TransferChunkParams {
transfer_id: begin.transfer_id.clone(),
offset,
data_base64: base64::engine::general_purpose::STANDARD.encode(&buf[..n]),
}),
Duration::from_secs(10),
Duration::from_secs(30),
)?;
Ok::<_, CliError>(())
})();
if let Err(err) = staged {
let _ = release(sock, &begin.transfer_id);
return Err(err);
offset = reply.next_offset;
}
Ok((begin.transfer_id, name))
}
fn release(sock: &Path, id: &str) -> Result<TransferReleaseResult, CliError> {
crate::cli::business_rpc::call(
let _: TransferReadyResult = crate::cli::business_rpc::call(
sock.to_path_buf(),
"transfer-release",
Method::TransferRelease,
"transfer-finish",
Method::TransferFinish,
Some(TransferIdParams {
transfer_id: id.to_string(),
transfer_id: begin.transfer_id.clone(),
}),
Duration::from_secs(5),
)
Duration::from_secs(10),
)?;
Ok(())
}
fn ipc_timeout(timeout_ms: u32) -> Duration {
+3
View File
@@ -253,6 +253,9 @@ pub fn full_handler(status: DaemonStatus, state: Arc<DaemonState>) -> RpcHandler
| Method::ToolReload
| Method::ToolClick
| Method::ToolHover
| Method::ToolScrollTo
| Method::ToolFocus
| Method::ToolBlur
| Method::ToolFill
| Method::ToolPress
| Method::ToolSelect
+3
View File
@@ -94,6 +94,9 @@ fn dispatch(cli: Cli, format: Format) -> Result<(), CliError> {
Command::Reload(args) => cli::navigate::dispatch_reload(args, format),
Command::Click(args) => cli::interaction::dispatch_click(args, format),
Command::Hover(args) => cli::interaction::dispatch_hover(args, format),
Command::ScrollTo(args) => cli::scroll::dispatch(args, format),
Command::Focus(args) => cli::interaction::dispatch_focus(args, format),
Command::Blur(args) => cli::interaction::dispatch_blur(args, format),
Command::Fill(args) => cli::interaction::dispatch_fill(args, format),
Command::Press(args) => cli::interaction::dispatch_press(args, format),
Command::Select(args) => cli::interaction::dispatch_select(args, format),
+59
View File
@@ -255,6 +255,65 @@ fn parses_hover_with_settle() {
assert_eq!(args.settle, 300);
}
#[test]
fn parses_scroll_to_target() {
let cli = parse(&["bsk", "scroll-to", "@e2", "--session", "s1"]);
let Command::ScrollTo(args) = cli.command else {
panic!("expected scroll-to command");
};
assert_eq!(args.target.as_deref(), Some("@e2"));
}
#[test]
fn parses_scroll_to_explicit_target_tab_and_timeout() {
for flag in ["--ref", "--selector"] {
let cli = parse(&[
"bsk",
"scroll-to",
flag,
"e3",
"--session",
"s1",
"--tab-id",
"42",
"--timeout",
"5s",
]);
let Command::ScrollTo(args) = cli.command else {
panic!("expected scroll-to command");
};
assert_eq!(args.tab_id, Some(42));
assert_eq!(args.timeout, 5_000);
assert_eq!(
if flag == "--ref" {
args.ref_
} else {
args.selector
}
.as_deref(),
Some("e3")
);
}
}
#[test]
fn parses_focus_target() {
let cli = parse(&["bsk", "focus", "@e2", "--session", "s1"]);
let Command::Focus(args) = cli.command else {
panic!("expected focus command");
};
assert_eq!(args.target.as_deref(), Some("@e2"));
}
#[test]
fn parses_blur_selector() {
let cli = parse(&["bsk", "blur", "--selector", "#search", "--session", "s1"]);
let Command::Blur(args) = cli.command else {
panic!("expected blur command");
};
assert_eq!(args.selector.as_deref(), Some("#search"));
}
#[test]
fn rejects_zero_click_count() {
assert!(
+23 -13
View File
@@ -293,6 +293,15 @@ async fn session_user_interrupt_event_cancels_inflight_with_user_aborted() {
/// catch a tool dispatched arbitrarily later.
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn user_interrupt_rejects_next_mutating_tool_call_when_session_was_idle() {
assert_idle_interrupt_rejects(Method::ToolClick).await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn user_interrupt_rejects_scroll_to_before_forwarding() {
assert_idle_interrupt_rejects(Method::ToolScrollTo).await;
}
async fn assert_idle_interrupt_rejects(method: Method) {
let (handle, sock) = spawn_daemon().await;
let mut ws = connect_ext(handle.ws_addr()).await;
let _ = handshake_as_ext(&mut ws).await;
@@ -301,10 +310,11 @@ async fn user_interrupt_rejects_next_mutating_tool_call_when_session_was_idle()
// Fake extension: answer tool.session_start, observe other
// requests but never reply. We expect the daemon to reject the
// tool call BEFORE it ever forwards a tool.click frame.
// tool call BEFORE it ever forwards a browser-input frame.
let ws_sink_for_responder = Arc::clone(&ws_sink);
let observed_click_count = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let observed_click_count_clone = Arc::clone(&observed_click_count);
let observed_call_count = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let observed_call_count_clone = Arc::clone(&observed_call_count);
let forwarded_method = method.clone();
let responder = tokio::spawn(async move {
let mut ws_stream = ws_stream;
while let Some(Ok(msg)) = ws_stream.next().await {
@@ -329,8 +339,8 @@ async fn user_interrupt_rejects_next_mutating_tool_call_when_session_was_idle()
g.send(Message::Text(serde_json::to_string(&reply).unwrap()))
.await
.unwrap();
} else if req.method == Method::ToolClick {
observed_click_count_clone.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
} else if req.method == forwarded_method {
observed_call_count_clone.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
// Deliberately do NOT reply — daemon should reject before
// ever forwarding to us.
let _ = req;
@@ -377,12 +387,12 @@ async fn user_interrupt_rejects_next_mutating_tool_call_when_session_was_idle()
let state = handle.state();
wait_for_session_interrupt_pending(&state, &start.session_id).await;
// The CLI now issues a tool.click (mutating). The daemon must
// The CLI now issues a browser-input (mutating). The daemon must
// reject it WITHOUT forwarding to the extension.
let click_outcome = ipc
let outcome = ipc
.call::<_, serde_json::Value>(
"click-after-interrupt",
Method::ToolClick,
"input-after-interrupt",
method,
Some(json!({
"session_id": start.session_id,
"ref": "fake-ref-1",
@@ -391,7 +401,7 @@ async fn user_interrupt_rejects_next_mutating_tool_call_when_session_was_idle()
)
.await
.unwrap();
let err = click_outcome.expect_err("tool.click must be rejected");
let err = outcome.expect_err("browser-input must be rejected");
assert_eq!(
err.code,
ErrorCode::UserAborted,
@@ -399,11 +409,11 @@ async fn user_interrupt_rejects_next_mutating_tool_call_when_session_was_idle()
err
);
// Crucially: the extension never saw a tool.click frame.
// Crucially: the extension never saw a browser-input frame.
assert_eq!(
observed_click_count.load(std::sync::atomic::Ordering::SeqCst),
observed_call_count.load(std::sync::atomic::Ordering::SeqCst),
0,
"tool.click must NOT have been forwarded to the extension"
"browser-input must NOT have been forwarded to the extension"
);
responder.abort();
+138 -6
View File
@@ -1,5 +1,4 @@
//! M7 integration: drive the 7 new tool RPCs (navigate /
//! navigate_back / navigate_forward / reload / click / fill / press)
//! M7 integration: drive interaction and navigation tool RPCs
//! through the IPC + per-session queue + fake extension and assert
//! the wire shapes line up. The extension stub mirrors each method's
//! `*Result` so we exercise the daemon's serialise → forward →
@@ -13,11 +12,13 @@ use bsk::daemon::{self, DaemonConfig};
use bsk::ipc_client::IpcClient;
use bsk_protocol::system::{HandshakeParams, HandshakeResult};
use bsk_protocol::tools::{
ClickParams, ClickResult, FillParams, FillResult, KeyModifier, MouseButton, NavigateBackParams,
NavigateBackResult, NavigateForwardParams, NavigateForwardResult, NavigateParams,
NavigateResult, PressParams, PressResult, ReloadParams, ReloadResult, SelectParams,
SelectResult, SessionStartParams, SessionStartResult, WaitUntil,
BlurParams, BlurResult, ClickParams, ClickResult, FillParams, FillResult, FocusParams,
FocusResult, KeyModifier, MouseButton, NavigateBackParams, NavigateBackResult,
NavigateForwardParams, NavigateForwardResult, NavigateParams, NavigateResult, PressParams,
PressResult, ReloadParams, ReloadResult, SelectParams, SelectResult, SessionStartParams,
SessionStartResult, WaitUntil,
};
use bsk_protocol::tools::{ScrollToParams, ScrollToResult};
use bsk_protocol::{
BrowserPeerInfo, ErrorCode, Frame, Method, RequestFrame, ResponseBody, ResponseFrame, RpcError,
};
@@ -382,6 +383,137 @@ async fn click_round_trips_ref_and_modifiers() {
handle.shutdown().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn focus_round_trips_verified_state() {
let (handle, sock) = spawn_daemon().await;
let mut ws = connect_ext(handle.ws_addr()).await;
let _ = do_handshake(&mut ws).await;
run_extension(ws, |req| {
assert_eq!(req.method, Method::ToolFocus);
let params = req.params.clone().unwrap();
let p: FocusParams = serde_json::from_value(params).unwrap();
assert_eq!(p.ref_.as_deref(), Some("@e3"));
ResponseBody::Ok(
serde_json::to_value(FocusResult {
tab_id: 9,
used_ref: Some("e3".into()),
used_selector: None,
focused: true,
dialogs: vec![],
})
.unwrap(),
)
});
let session_id = ipc_session_start(&sock).await;
let result: FocusResult = ipc_tool_call(
&sock,
Method::ToolFocus,
FocusParams {
session_id,
ref_: Some("@e3".into()),
selector: None,
tab_id: None,
timeout_ms: Some(5_000),
},
)
.await
.expect("focus ok");
assert!(result.focused);
assert_eq!(result.used_ref.as_deref(), Some("e3"));
handle.shutdown().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn blur_round_trips_previous_focus_state() {
let (handle, sock) = spawn_daemon().await;
let mut ws = connect_ext(handle.ws_addr()).await;
let _ = do_handshake(&mut ws).await;
run_extension(ws, |req| {
assert_eq!(req.method, Method::ToolBlur);
let params = req.params.clone().unwrap();
let p: BlurParams = serde_json::from_value(params).unwrap();
assert_eq!(p.selector.as_deref(), Some("#search"));
ResponseBody::Ok(
serde_json::to_value(BlurResult {
tab_id: 9,
used_ref: None,
used_selector: Some("#search".into()),
was_focused: true,
focused: false,
dialogs: vec![],
})
.unwrap(),
)
});
let session_id = ipc_session_start(&sock).await;
let result: BlurResult = ipc_tool_call(
&sock,
Method::ToolBlur,
BlurParams {
session_id,
ref_: None,
selector: Some("#search".into()),
tab_id: None,
timeout_ms: Some(5_000),
},
)
.await
.expect("blur ok");
assert!(result.was_focused);
assert!(!result.focused);
assert_eq!(result.used_selector.as_deref(), Some("#search"));
handle.shutdown().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn scroll_to_round_trips_visible_bounds() {
let (handle, sock) = spawn_daemon().await;
let mut ws = connect_ext(handle.ws_addr()).await;
let _ = do_handshake(&mut ws).await;
run_extension(ws, |req| {
assert_eq!(req.method, Method::ToolScrollTo);
let params = req.params.clone().unwrap();
let parsed: ScrollToParams = serde_json::from_value(params).unwrap();
assert_eq!(parsed.ref_.as_deref(), Some("@e3"));
ResponseBody::Ok(
serde_json::to_value(ScrollToResult {
tab_id: 9,
used_ref: Some("e3".into()),
used_selector: None,
x: 10.0,
y: 20.0,
width: 100.0,
height: 40.0,
dialogs: vec![],
})
.unwrap(),
)
});
let session_id = ipc_session_start(&sock).await;
let result: ScrollToResult = ipc_tool_call(
&sock,
Method::ToolScrollTo,
ScrollToParams {
session_id,
ref_: Some("@e3".into()),
selector: None,
tab_id: None,
timeout_ms: Some(5_000),
},
)
.await
.expect("scroll-to ok");
assert_eq!(result.used_ref.as_deref(), Some("e3"));
assert_eq!(result.width, 100.0);
handle.shutdown().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn fill_round_trips_clear_before_default() {
let (handle, sock) = spawn_daemon().await;
@@ -0,0 +1,342 @@
//! Real Windows CLI cancellation, including caller-owned transfer cleanup.
#![cfg(windows)]
use std::path::Path;
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
use bsk::daemon::{file_transfer::TransferRegistry, info::DaemonInfo};
use bsk_protocol::tools::{TransferBeginParams, TransferIdParams};
use bsk_protocol::{ErrorCode, Frame, Method, ResponseBody, ResponseFrame, RpcError};
use serde_json::json;
use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader};
use tokio::net::windows::named_pipe::ServerOptions;
use tokio::process::Command;
use tokio::sync::{Notify, watch};
fn response<T: serde::Serialize>(result: Result<T, RpcError>) -> ResponseBody {
match result {
Ok(value) => ResponseBody::Ok(serde_json::to_value(value).unwrap()),
Err(err) => ResponseBody::Err(err),
}
}
#[test]
fn stdin_close_cancels_business_but_releases_resources_with_a_bounded_budget() {
// Resolve the production registry's root before starting any runtime threads.
let registry_home = tempfile::tempdir().unwrap();
let original_home = std::env::var_os("BSK_HOME");
unsafe {
std::env::set_var("BSK_HOME", registry_home.path());
}
let transfers = Arc::new(TransferRegistry::new().unwrap());
unsafe {
match original_home {
Some(value) => std::env::set_var("BSK_HOME", value),
None => std::env::remove_var("BSK_HOME"),
}
}
transfers.initialize().unwrap();
tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.unwrap()
.block_on(exercise(
transfers,
&registry_home.path().join("run/transfers"),
));
}
async fn exercise(transfers: Arc<TransferRegistry>, staging_root: &Path) {
for mode in [
"wait",
"upload",
"upload-multi",
"upload-before-dispatch",
"download",
"upload-multi-stall",
"ordinary",
] {
let home = tempfile::tempdir().unwrap();
let pipe_name = format!(r"\\.\pipe\bsk-parent-cancel-{}", uuid::Uuid::new_v4());
let info = DaemonInfo::now(std::process::id(), (&pipe_name).into(), 0, "0.2.0");
std::fs::write(
home.path().join("daemon.json"),
serde_json::to_vec(&info).unwrap(),
)
.unwrap();
let mut pipe = ServerOptions::new()
.first_pipe_instance(true)
.create(&pipe_name)
.unwrap();
let registered = Arc::new(Notify::new());
let pending = Arc::new(Mutex::new(None::<String>));
let ids = Arc::new(Mutex::new(Vec::<String>::new()));
let released = Arc::new(Mutex::new(Vec::<String>::new()));
let received = Arc::new(Mutex::new(Vec::<Method>::new()));
let download_id = if mode == "download" {
let staging = transfers.begin_download("fixture").unwrap();
let browser_dir = home.path().join("BrowserSkill").join(&staging.transfer_id);
std::fs::create_dir_all(&browser_dir).unwrap();
let file = browser_dir.join("download.txt");
std::fs::write(&file, b"download fixture").unwrap();
transfers
.import_download(&staging.transfer_id, &file)
.unwrap();
ids.lock().unwrap().push(staging.transfer_id.clone());
Some(staging.transfer_id)
} else {
None
};
let (cancel, cancelled) = watch::channel(false);
let ready = Arc::clone(&registered);
let name = pipe_name.clone();
let registry = Arc::clone(&transfers);
let server_ids = Arc::clone(&ids);
let server_released = Arc::clone(&released);
let server_received = Arc::clone(&received);
let server = tokio::spawn(async move {
let mut connections = tokio::task::JoinSet::new();
loop {
pipe.connect().await.unwrap();
let next = ServerOptions::new().create(&name).unwrap();
let connection = std::mem::replace(&mut pipe, next);
let ready = Arc::clone(&ready);
let pending = Arc::clone(&pending);
let cancel = cancel.clone();
let mut cancelled = cancelled.clone();
let name = name.clone();
let registry = Arc::clone(&registry);
let ids = Arc::clone(&server_ids);
let released = Arc::clone(&server_released);
let received = Arc::clone(&server_received);
let download_id = download_id.clone();
connections.spawn(async move {
let (read, mut write) = tokio::io::split(connection);
let mut reader = BufReader::new(read);
let mut line = String::new();
while reader.read_line(&mut line).await.unwrap_or(0) > 0 {
let Frame::Request(req) = serde_json::from_str::<Frame>(&line).unwrap()
else {
panic!("expected request");
};
line.clear();
received.lock().unwrap().push(req.method.clone());
let params = req.params.clone().unwrap_or_default();
let body = match req.method {
Method::SystemStatus => ResponseBody::Ok(json!({
"daemon_version": "0.2.0", "protocol_version": "1.1",
"pid": std::process::id(), "uptime_secs": 0,
"ws_port": 0, "sock_path": name,
"browsers": [], "sessions": [], "version_skew_browsers": []
})),
Method::TransferBegin => {
let begin = registry
.begin_upload(serde_json::from_value(params).unwrap())
.unwrap();
ids.lock().unwrap().push(begin.transfer_id.clone());
response(Ok(begin))
}
Method::ToolDownload => ResponseBody::Ok(json!({
"tab_id": 1, "suggested_filename": "download.txt",
"byte_size": 16, "transfer_id": download_id,
})),
Method::ToolWaitMs if mode == "ordinary" => {
ResponseBody::Ok(json!({"waited_ms": 1}))
}
Method::TransferChunk
if mode == "upload-before-dispatch"
|| (mode.contains("multi")
&& ids.lock().unwrap().len() == 1) =>
{
response(
registry.write_chunk(serde_json::from_value(params).unwrap()),
)
}
Method::TransferFinish if mode != "upload-before-dispatch" => response(
registry.finish_upload(serde_json::from_value(params).unwrap()),
),
Method::ToolWaitMs
| Method::TransferChunk
| Method::TransferRead
| Method::TransferFinish => {
*pending.lock().unwrap() = Some(req.id.clone());
ready.notify_one();
cancelled.wait_for(|value| *value).await.unwrap();
if mode == "upload-before-dispatch" {
// Finish won the race with cancellation, but tool.upload must
// not be dispatched and the CLI still owns the staging.
response(
registry
.finish_upload(serde_json::from_value(params).unwrap()),
)
} else {
ResponseBody::Err(RpcError {
code: ErrorCode::Cancelled,
message: "parent cancellation reached daemon".into(),
data: None,
})
}
}
Method::Cancel => {
let target = params["rpc_id"].as_str().unwrap();
assert_eq!(pending.lock().unwrap().as_deref(), Some(target));
cancel.send(true).unwrap();
ResponseBody::Ok(json!({"cancelled": true}))
}
Method::TransferRelease => {
let params: TransferIdParams =
serde_json::from_value(params).unwrap();
released.lock().unwrap().push(params.transfer_id.clone());
if mode.contains("stall") {
std::future::pending::<()>().await;
}
response(Ok(registry.release(params)))
}
other => panic!("unexpected request after cancellation: {other:?}"),
};
let response = Frame::Response(ResponseFrame { id: req.id, body });
let mut bytes = serde_json::to_vec(&response).unwrap();
bytes.push(b'\n');
if write.write_all(&bytes).await.is_err() {
break;
}
}
});
}
});
let mut cmd = Command::new(env!("CARGO_BIN_EXE_bsk"));
cmd.env("BSK_HOME", home.path())
.env("BSK_AUTO_UPDATE", "off")
.env_remove("BSK_CANCEL_ON_STDIN_CLOSE")
.stdout(std::process::Stdio::piped())
.stderr(std::process::Stdio::piped())
.creation_flags(0x0800_0000)
.kill_on_drop(true);
if mode.starts_with("upload") {
let source = home.path().join("upload.txt");
// Declare 300 MiB without allocating or transferring that much data.
std::fs::File::create(&source)
.unwrap()
.set_len(if mode == "upload-before-dispatch" {
16
} else {
300 * 1024 * 1024
})
.unwrap();
cmd.args(["upload", "--session", "fixture", "--selector", "input"]);
if mode.contains("multi") {
let first = home.path().join("first.txt");
std::fs::write(&first, b"first file").unwrap();
cmd.arg("--file").arg(first);
}
cmd.arg("--file").arg(source);
} else if mode == "download" {
let out = home.path().join("out.txt");
std::fs::write(&out, b"existing output").unwrap();
cmd.args([
"download",
"--session",
"fixture",
"--selector",
"a",
"--overwrite",
"--out",
])
.arg(out);
} else {
cmd.args(["wait-ms", if mode == "ordinary" { "1ms" } else { "60s" }]);
}
cmd.arg("--json");
if mode == "ordinary" {
cmd.stdin(std::process::Stdio::null());
} else {
cmd.env("BSK_CANCEL_ON_STDIN_CLOSE", "1")
.stdin(std::process::Stdio::piped());
}
let mut child = cmd.spawn().unwrap();
if mode != "ordinary" {
tokio::time::timeout(Duration::from_secs(10), registered.notified())
.await
.expect("CLI registered business RPC");
child
.stdin
.as_mut()
.unwrap()
.write_all(b"still connected")
.await
.unwrap();
drop(child.stdin.take());
}
let start = Instant::now();
let output = tokio::time::timeout(Duration::from_secs(8), child.wait_with_output())
.await
.expect("CLI must settle promptly, including cleanup")
.unwrap();
server.abort();
let _ = server.await;
assert_eq!(
output.status.code(),
Some(if mode == "ordinary" { 0 } else { 2 }),
"{mode}: stdout={} stderr={}",
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr)
);
if mode != "ordinary" {
let body: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap();
assert_eq!(body["code"], "cancelled");
if mode != "upload-before-dispatch" {
assert_eq!(body["message"], "parent cancellation reached daemon");
}
let received = received.lock().unwrap();
let cancel = received.iter().position(|m| *m == Method::Cancel).unwrap();
assert!(
received[cancel + 1..]
.iter()
.all(|m| *m == Method::TransferRelease),
"only cleanup may follow cancellation: {received:?}"
);
}
let ids = ids.lock().unwrap();
if mode.contains("stall") {
assert_eq!(released.lock().unwrap().len(), 1);
assert!(start.elapsed() >= Duration::from_secs(4));
// The CLI cannot force an unresponsive daemon to clean up.
transfers.release_session("fixture");
} else {
assert_eq!(
*released.lock().unwrap(),
*ids,
"{mode}: every allocation must be released"
);
for id in ids.iter() {
assert!(!staging_root.join(id).exists(), "staging leaked: {id}");
}
if mode.starts_with("upload") {
let retry = transfers
.begin_upload(TransferBeginParams {
session_id: "fixture".into(),
name: "retry.txt".into(),
byte_size: 300 * 1024 * 1024,
})
.expect("same session must have room for another 300 MiB upload");
transfers.release(TransferIdParams {
transfer_id: retry.transfer_id,
});
}
}
if mode == "download" {
assert_eq!(
std::fs::read(home.path().join("out.txt")).unwrap(),
b"existing output"
);
assert!(
!std::fs::read_dir(home.path())
.unwrap()
.flatten()
.any(|entry| entry.file_name().to_string_lossy().ends_with(".part"))
);
}
}
}
@@ -0,0 +1,42 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "BlurParams",
"type": "object",
"required": [
"session_id"
],
"properties": {
"ref": {
"description": "Optional `@e<N>` ref allocated by the last observation. Mutually exclusive with `selector`.",
"type": [
"string",
"null"
]
},
"selector": {
"type": [
"string",
"null"
]
},
"session_id": {
"type": "string"
},
"tab_id": {
"description": "Target tab. Defaults to the Agent Window's active tab.",
"type": [
"integer",
"null"
],
"format": "int64"
},
"timeout_ms": {
"type": [
"integer",
"null"
],
"format": "uint32",
"minimum": 1.0
}
}
}
@@ -0,0 +1,112 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "BlurResult",
"type": "object",
"required": [
"focused",
"tab_id",
"was_focused"
],
"properties": {
"dialogs": {
"type": "array",
"items": {
"$ref": "#/definitions/JavaScriptDialogInfo"
}
},
"focused": {
"description": "Always false for a successful blur; included for machine verification.",
"type": "boolean"
},
"tab_id": {
"type": "integer",
"format": "int64"
},
"used_ref": {
"type": [
"string",
"null"
]
},
"used_selector": {
"type": [
"string",
"null"
]
},
"was_focused": {
"description": "Whether the target held focus before the call.",
"type": "boolean"
}
},
"definitions": {
"JavaScriptDialogHandledAction": {
"description": "How the extension resolved the dialog so CDP could continue.",
"type": "string",
"enum": [
"accepted",
"dismissed"
]
},
"JavaScriptDialogInfo": {
"description": "One observed + handled JavaScript dialog during a tool call.",
"type": "object",
"required": [
"handled",
"message",
"sequence",
"tab_id",
"type"
],
"properties": {
"default_prompt": {
"type": [
"string",
"null"
]
},
"handled": {
"$ref": "#/definitions/JavaScriptDialogHandledAction"
},
"has_browser_handler": {
"type": [
"boolean",
"null"
]
},
"message": {
"type": "string"
},
"sequence": {
"description": "Monotonic per-tab sequence for ordering within a session.",
"type": "integer",
"format": "uint64",
"minimum": 0.0
},
"tab_id": {
"type": "integer",
"format": "int64"
},
"type": {
"$ref": "#/definitions/JavaScriptDialogType"
},
"url": {
"type": [
"string",
"null"
]
}
}
},
"JavaScriptDialogType": {
"description": "Native JS dialog kind reported by CDP.",
"type": "string",
"enum": [
"alert",
"confirm",
"prompt",
"beforeunload"
]
}
}
}
@@ -0,0 +1,42 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "FocusParams",
"type": "object",
"required": [
"session_id"
],
"properties": {
"ref": {
"description": "Optional `@e<N>` ref allocated by the last observation. Mutually exclusive with `selector`.",
"type": [
"string",
"null"
]
},
"selector": {
"type": [
"string",
"null"
]
},
"session_id": {
"type": "string"
},
"tab_id": {
"description": "Target tab. Defaults to the Agent Window's active tab.",
"type": [
"integer",
"null"
],
"format": "int64"
},
"timeout_ms": {
"type": [
"integer",
"null"
],
"format": "uint32",
"minimum": 1.0
}
}
}
@@ -0,0 +1,107 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "FocusResult",
"type": "object",
"required": [
"focused",
"tab_id"
],
"properties": {
"dialogs": {
"type": "array",
"items": {
"$ref": "#/definitions/JavaScriptDialogInfo"
}
},
"focused": {
"description": "Whether the target holds DOM focus in its document or shadow root after the call.",
"type": "boolean"
},
"tab_id": {
"type": "integer",
"format": "int64"
},
"used_ref": {
"type": [
"string",
"null"
]
},
"used_selector": {
"type": [
"string",
"null"
]
}
},
"definitions": {
"JavaScriptDialogHandledAction": {
"description": "How the extension resolved the dialog so CDP could continue.",
"type": "string",
"enum": [
"accepted",
"dismissed"
]
},
"JavaScriptDialogInfo": {
"description": "One observed + handled JavaScript dialog during a tool call.",
"type": "object",
"required": [
"handled",
"message",
"sequence",
"tab_id",
"type"
],
"properties": {
"default_prompt": {
"type": [
"string",
"null"
]
},
"handled": {
"$ref": "#/definitions/JavaScriptDialogHandledAction"
},
"has_browser_handler": {
"type": [
"boolean",
"null"
]
},
"message": {
"type": "string"
},
"sequence": {
"description": "Monotonic per-tab sequence for ordering within a session.",
"type": "integer",
"format": "uint64",
"minimum": 0.0
},
"tab_id": {
"type": "integer",
"format": "int64"
},
"type": {
"$ref": "#/definitions/JavaScriptDialogType"
},
"url": {
"type": [
"string",
"null"
]
}
}
},
"JavaScriptDialogType": {
"description": "Native JS dialog kind reported by CDP.",
"type": "string",
"enum": [
"alert",
"confirm",
"prompt",
"beforeunload"
]
}
}
}
@@ -0,0 +1,42 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "ScrollToParams",
"type": "object",
"required": [
"session_id"
],
"properties": {
"ref": {
"description": "Optional `@e<N>` ref allocated by the last observation. Mutually exclusive with `selector`.",
"type": [
"string",
"null"
]
},
"selector": {
"type": [
"string",
"null"
]
},
"session_id": {
"type": "string"
},
"tab_id": {
"description": "Target tab. Defaults to the Agent Window's active tab.",
"type": [
"integer",
"null"
],
"format": "int64"
},
"timeout_ms": {
"type": [
"integer",
"null"
],
"format": "uint32",
"minimum": 1.0
}
}
}
@@ -0,0 +1,123 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "ScrollToResult",
"type": "object",
"required": [
"height",
"tab_id",
"width",
"x",
"y"
],
"properties": {
"dialogs": {
"type": "array",
"items": {
"$ref": "#/definitions/JavaScriptDialogInfo"
}
},
"height": {
"type": "number",
"format": "double"
},
"tab_id": {
"type": "integer",
"format": "int64"
},
"used_ref": {
"type": [
"string",
"null"
]
},
"used_selector": {
"type": [
"string",
"null"
]
},
"width": {
"type": "number",
"format": "double"
},
"x": {
"description": "Bounding rectangle of the element's visible border-box portion, in top-level viewport CSS pixels, after ancestor and viewport clipping. Partial visibility is sufficient. This is not an occlusion or hit test.",
"type": "number",
"format": "double"
},
"y": {
"type": "number",
"format": "double"
}
},
"definitions": {
"JavaScriptDialogHandledAction": {
"description": "How the extension resolved the dialog so CDP could continue.",
"type": "string",
"enum": [
"accepted",
"dismissed"
]
},
"JavaScriptDialogInfo": {
"description": "One observed + handled JavaScript dialog during a tool call.",
"type": "object",
"required": [
"handled",
"message",
"sequence",
"tab_id",
"type"
],
"properties": {
"default_prompt": {
"type": [
"string",
"null"
]
},
"handled": {
"$ref": "#/definitions/JavaScriptDialogHandledAction"
},
"has_browser_handler": {
"type": [
"boolean",
"null"
]
},
"message": {
"type": "string"
},
"sequence": {
"description": "Monotonic per-tab sequence for ordering within a session.",
"type": "integer",
"format": "uint64",
"minimum": 0.0
},
"tab_id": {
"type": "integer",
"format": "int64"
},
"type": {
"$ref": "#/definitions/JavaScriptDialogType"
},
"url": {
"type": [
"string",
"null"
]
}
}
},
"JavaScriptDialogType": {
"description": "Native JS dialog kind reported by CDP.",
"type": "string",
"enum": [
"alert",
"confirm",
"prompt",
"beforeunload"
]
}
}
}
@@ -79,6 +79,12 @@ fn main() {
dump!(ClickResult, "tool_click_result");
dump!(HoverParams, "tool_hover_params");
dump!(HoverResult, "tool_hover_result");
dump!(ScrollToParams, "tool_scroll_to_params");
dump!(ScrollToResult, "tool_scroll_to_result");
dump!(FocusParams, "tool_focus_params");
dump!(FocusResult, "tool_focus_result");
dump!(BlurParams, "tool_blur_params");
dump!(BlurResult, "tool_blur_result");
dump!(FillParams, "tool_fill_params");
dump!(FillResult, "tool_fill_result");
dump!(PressParams, "tool_press_params");
+18
View File
@@ -72,6 +72,12 @@ pub enum Method {
ToolClick,
#[serde(rename = "tool.hover")]
ToolHover,
#[serde(rename = "tool.scroll_to")]
ToolScrollTo,
#[serde(rename = "tool.focus")]
ToolFocus,
#[serde(rename = "tool.blur")]
ToolBlur,
#[serde(rename = "tool.fill")]
ToolFill,
#[serde(rename = "tool.press")]
@@ -168,6 +174,9 @@ impl Method {
| Method::ToolNavigateForward
| Method::ToolReload
| Method::ToolClick
| Method::ToolScrollTo
| Method::ToolFocus
| Method::ToolBlur
| Method::ToolFill
| Method::ToolPress
| Method::ToolSelect
@@ -305,6 +314,9 @@ mod tests {
assert!(Method::ToolNavigateForward.is_mutating());
assert!(Method::ToolReload.is_mutating());
assert!(Method::ToolClick.is_mutating());
assert!(Method::ToolScrollTo.is_mutating());
assert!(Method::ToolFocus.is_mutating());
assert!(Method::ToolBlur.is_mutating());
assert!(Method::ToolFill.is_mutating());
assert!(Method::ToolPress.is_mutating());
assert!(Method::ToolSelect.is_mutating());
@@ -349,6 +361,9 @@ mod tests {
assert_eq!(Method::ToolHover.effect(), MethodEffect::TransientInput);
assert_eq!(Method::ToolObserve.effect(), MethodEffect::TransientInput);
assert_eq!(Method::ToolClick.effect(), MethodEffect::BrowserMutation);
assert_eq!(Method::ToolScrollTo.effect(), MethodEffect::BrowserMutation);
assert_eq!(Method::ToolFocus.effect(), MethodEffect::BrowserMutation);
assert_eq!(Method::ToolBlur.effect(), MethodEffect::BrowserMutation);
assert_eq!(Method::Cancel.effect(), MethodEffect::ControlPlane);
}
@@ -358,6 +373,9 @@ mod tests {
assert!(Method::ToolHover.requires_interrupt_gate());
assert!(Method::ToolObserve.requires_interrupt_gate());
assert!(Method::ToolClick.requires_interrupt_gate());
assert!(Method::ToolScrollTo.requires_interrupt_gate());
assert!(Method::ToolFocus.requires_interrupt_gate());
assert!(Method::ToolBlur.requires_interrupt_gate());
assert!(!Method::Cancel.requires_interrupt_gate());
}
}
+109 -2
View File
@@ -1,5 +1,5 @@
//! DOM interaction tools (`tool.click`, `tool.fill`, `tool.press`,
//! `tool.select`).
//! DOM interaction tools (`tool.click`, `tool.hover`, `tool.focus`,
//! `tool.blur`, `tool.fill`, `tool.press`, `tool.select`).
//!
//! Element-targeted tools accept either a snapshot `ref` (`@e<N>` form,
//! normalised against the session's RefStore) **or** a CSS selector
@@ -131,6 +131,82 @@ pub struct HoverResult {
pub dialogs: Vec<JavaScriptDialogInfo>,
}
// ---------------------------------------------------------------------------
// focus / blur
// ---------------------------------------------------------------------------
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct FocusParams {
pub session_id: String,
/// Optional `@e<N>` ref allocated by the last observation.
/// Mutually exclusive with `selector`.
#[serde(
rename = "ref",
alias = "ref_",
default,
skip_serializing_if = "Option::is_none"
)]
pub ref_: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub selector: Option<String>,
/// Target tab. Defaults to the Agent Window's active tab.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub tab_id: Option<i64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(range(min = 1))]
pub timeout_ms: Option<u32>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct FocusResult {
pub tab_id: i64,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub used_ref: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub used_selector: Option<String>,
/// Whether the target holds DOM focus in its document or shadow root after the call.
pub focused: bool,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub dialogs: Vec<JavaScriptDialogInfo>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct BlurParams {
pub session_id: String,
/// Optional `@e<N>` ref allocated by the last observation.
/// Mutually exclusive with `selector`.
#[serde(
rename = "ref",
alias = "ref_",
default,
skip_serializing_if = "Option::is_none"
)]
pub ref_: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub selector: Option<String>,
/// Target tab. Defaults to the Agent Window's active tab.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub tab_id: Option<i64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(range(min = 1))]
pub timeout_ms: Option<u32>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct BlurResult {
pub tab_id: i64,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub used_ref: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub used_selector: Option<String>,
/// Whether the target held focus before the call.
pub was_focused: bool,
/// Always false for a successful blur; included for machine verification.
pub focused: bool,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub dialogs: Vec<JavaScriptDialogInfo>,
}
// ---------------------------------------------------------------------------
// fill
// ---------------------------------------------------------------------------
@@ -325,6 +401,37 @@ mod tests {
assert_eq!(round, p);
}
#[test]
fn focus_params_serialise_ref_field_name() {
let p = FocusParams {
session_id: "abcd".into(),
ref_: Some("@e4".into()),
selector: None,
tab_id: Some(42),
timeout_ms: Some(5_000),
};
let v = serde_json::to_value(&p).unwrap();
assert_eq!(v.get("ref").and_then(|v| v.as_str()), Some("@e4"));
assert!(v.get("ref_").is_none());
let round: FocusParams = serde_json::from_value(v).unwrap();
assert_eq!(round, p);
}
#[test]
fn blur_result_round_trips_focus_state() {
let result = BlurResult {
tab_id: 42,
used_ref: Some("e4".into()),
used_selector: None,
was_focused: true,
focused: false,
dialogs: vec![],
};
let value = serde_json::to_value(&result).unwrap();
let round: BlurResult = serde_json::from_value(value).unwrap();
assert_eq!(round, result);
}
#[test]
fn modifiers_render_as_lowercase_strings() {
let v = serde_json::to_value(KeyModifier::Ctrl).unwrap();
+2
View File
@@ -14,6 +14,7 @@ mod record_common;
mod record_v2;
mod record_v3;
pub mod script;
pub mod scroll;
pub mod session;
pub mod tabs;
pub mod waits;
@@ -30,6 +31,7 @@ pub use network::*;
pub use observation::*;
pub use record::*;
pub use script::*;
pub use scroll::*;
pub use session::*;
pub use tabs::*;
pub use waits::*;
+67
View File
@@ -0,0 +1,67 @@
//! Element scrolling primitive (`tool.scroll_to`).
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
use super::JavaScriptDialogInfo;
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct ScrollToParams {
pub session_id: String,
/// Optional `@e<N>` ref allocated by the last observation.
/// Mutually exclusive with `selector`.
#[serde(
rename = "ref",
alias = "ref_",
default,
skip_serializing_if = "Option::is_none"
)]
pub ref_: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub selector: Option<String>,
/// Target tab. Defaults to the Agent Window's active tab.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub tab_id: Option<i64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(range(min = 1))]
pub timeout_ms: Option<u32>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct ScrollToResult {
pub tab_id: i64,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub used_ref: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub used_selector: Option<String>,
/// Bounding rectangle of the element's visible border-box portion, in
/// top-level viewport CSS pixels, after ancestor and viewport clipping.
/// Partial visibility is sufficient. This is not an occlusion or hit test.
pub x: f64,
pub y: f64,
pub width: f64,
pub height: f64,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub dialogs: Vec<JavaScriptDialogInfo>,
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn params_serialise_ref_field_name() {
let params = ScrollToParams {
session_id: "abcd".into(),
ref_: Some("@e3".into()),
selector: None,
tab_id: Some(42),
timeout_ms: Some(5_000),
};
let value = serde_json::to_value(&params).unwrap();
assert_eq!(value.get("ref").and_then(|v| v.as_str()), Some("@e3"));
assert!(value.get("ref_").is_none());
let round: ScrollToParams = serde_json::from_value(value).unwrap();
assert_eq!(round, params);
}
}
+5
View File
@@ -84,6 +84,11 @@ Shared Rust types + JSON Schema generation. TypeScript mirrors frame shapes in
4. Extension dispatcher validates sandbox rules, invokes CDP via `BrowserDriver`.
5. Response travels CLI ← daemon ← extension; CLI prints result and exits.
The [scroll-to primitive reference](scroll-to.md) documents `tool.scroll_to`,
including its CLI/plugin mappings, visible-bounds contract and cancellation
behavior. It follows the same routing path and is classified as a browser
mutation for session queueing and user-interruption gating.
## Session and sandbox model
- **Session** = opaque ID (4 lowercase letters in v0.1) + dedicated **Agent Window**
+131
View File
@@ -0,0 +1,131 @@
# Scroll-to primitive
`scroll-to` brings an existing element and its containing frames into view, then
returns the visible portion's bounds. Use it to reveal a target before inspecting
the page or taking a screenshot. It accepts an element target rather than a pixel
distance or scroll direction.
| Entry point | Name |
| --- | --- |
| CLI | `bsk scroll-to` |
| Wire protocol | `tool.scroll_to` |
| DeepSeek Harness | `browser_interact` with `action: "scroll-to"` |
## CLI
Use a session you started and a fresh ref from an observation. The ids below are
examples; substitute the session, tab and ref returned by your own calls.
```sh
bsk scroll-to @e3 --session abcd --tab-id 42 --timeout 5s --json
bsk scroll-to --selector '#details' --session abcd
```
Supply exactly one target: a positional ref or selector, `--ref`, or `--selector`.
Refs accept `@e3` and `e3`. CSS selectors search only the main document; use refs
for elements in iframes (including out-of-process iframes) or shadow roots.
Refs belong to one session and tab and may become stale after page changes.
`--session` is required. `--tab-id` defaults to the Agent Window's active tab.
`--timeout` defaults to `30s` and must be positive; durations such as `5000ms` and
`5s` are accepted. The command operates on Agent Window tabs, including user tabs
explicitly borrowed into that window.
Example JSON output:
```json
{
"tab_id": 42,
"used_ref": "e3",
"x": 10,
"y": 20,
"width": 100,
"height": 80
}
```
`used_ref` or `used_selector` identifies the resolved target. The optional
`dialogs` array reports JavaScript dialogs handled during the call.
## Result contract
`x` and `y` are the upper-left corner, and `width` and `height` are the size of
the visible border-box portion's bounding rectangle. All four values use
**top-level viewport CSS pixels**, after scrolling and clipping by ancestor
containers and frame viewports. They are not document or screenshot pixel
coordinates.
Partial visibility is sufficient: a 400px-high element clipped by an 80px-high
scroll container can succeed with `height: 80`. Hidden or fully clipped targets
fail. The rectangle is not an occlusion or hit test: another element can cover
the target, and the rectangle's center is not a guaranteed click point. Layout
can change after the result; use fresh refs for subsequent interactions.
## Protocol and plugin
The CLI sends a normal request through the daemon to the extension:
```json
{
"id": "scroll-1",
"method": "tool.scroll_to",
"params": {
"session_id": "abcd",
"ref": "@e3",
"tab_id": 42,
"timeout_ms": 5000
}
}
```
Protocol callers must supply exactly one nonempty `ref` or `selector`.
`tab_id` is optional; `timeout_ms` is a positive integer and defaults to 30000.
These target constraints are enforced by the handler. See the generated
[parameter schema](../crates/bsk-protocol/schema/tool_scroll_to_params.json) and
[result schema](../crates/bsk-protocol/schema/tool_scroll_to_result.json) for field
types. The wire response wraps the result above in `{ "id": "scroll-1", "result": ... }`.
The plugin exposes the same action with its usual camelCase parameters:
```json
{
"action": "scroll-to",
"session": "abcd",
"target": "@e3",
"tabId": 42,
"timeoutMs": 5000
}
```
Call this through `browser_interact` using a plugin-owned session. Its result
contains `session`, `tabId`, `x`, `y`, `width` and `height` with the same bounds
semantics. Omitted `session` uses the plugin's current session.
## Errors and interruption
The wire response uses the normal `error.code`, `error.message` and optional
`error.data.reason` fields. Common cases are:
| Code | `data.reason` | Meaning |
| --- | --- | --- |
| `invalid_params` | — | Missing/conflicting target or invalid timeout/tab id |
| `not_found` | `ref_not_found` | Ref is unknown, expired or belongs to another tab |
| `not_found` | `selector_not_found` | Main-document selector matched no element |
| `permission_denied` | `agent_window_scope` | Target tab has not been borrowed into the Agent Window |
| `permission_denied` | `element_not_visible` | No visible area remains after scrolling |
| `cancelled` | — | The call was cancelled |
| `timeout` | — | The action deadline expired |
| `cdp_failed` | varies | Browser command, frame geometry or visibility measurement failed |
The protocol classifies scroll-to as a browser mutation, so it uses the existing
per-session queue and user-interruption gate. The extension releases any retained
hover state before dispatch. Cancellation and deadline checks surround each CDP
step, including frame lookup, scrolling and measurement. Once detected, they
prevent further scrolling and a successful response; remote-object cleanup still
runs. Already performed scrolling is not rolled back, so inspect the current
page before retrying an interrupted call.
The implementation is scoped to scroll-to: it reuses frame scrolling and
coordinate projection, with a separate visibility measurement that accounts for
ancestor clipping and hidden frame owners. Shared click, hover and screenshot
geometry is unchanged.
@@ -0,0 +1,41 @@
# OOPIF occupied-scrollbar regression
An iframe's content quad includes its child viewport's scrollbar space, while
`Page.getLayoutMetrics().cssLayoutViewport` excludes that space. Mapping the latter
onto the whole quad stretches positions and sizes. The full target-local viewport
must determine scale; the visible viewport must still clip content at every OOPIF
boundary and constrain target-local action points.
This fixture alternates loopback hostnames across two nested OOPIFs. It includes
scrolling, borders, padding, scaled iframe owners, a partially clipped button and
a fully clipped button. Custom scrollbars occupy different widths and heights.
The `scrollbars` query parameter accepts `both` (default), `vertical`, `horizontal`
or `none`.
Run the numerical regression with Node 22+ and a local Chrome executable:
```sh
BSK_GEOMETRY_CHROME=/path/to/chrome pnpm --filter @browser-skill/extension exec vitest run \
src/tools/__tests__/snapshot-coordinates.browser.test.ts
```
The existing browser runner is shared with the snapshot-unit regression. The test:
- Verifies actual OOPIF targets and occupied scrollbar dimensions.
- Compares snapshot and live geometry with independent DOM rectangles and clipping.
- Dispatches real root-target clicks and verifies the intended frame/button received them.
- Rejects fully clipped controls and checks snapshot measurement reuse per target.
- Covers five device-scale/browser-zoom combinations plus single-axis and no-scrollbar cases.
The runner creates and removes isolated browser profiles. It is opt-in and skipped
in normal unit runs when `BSK_GEOMETRY_CHROME` is unset.
Run the CLI fixture smoke with a connected test extension:
```sh
BSK_AUTO_UPDATE=off pnpm eval:browser smoke --case oopif-scrollbars --bsk ./target/debug/bsk
```
The smoke assertions verify both nested frames have occupied scrollbars, that the
marker is observed and that the session is closed. Numeric geometry and actual
click assertions are exercised by the browser test above, not by smoke alone.
@@ -0,0 +1,39 @@
{
"$schema": "../../../schemas/case.schema.json",
"schemaVersion": 1,
"id": "oopif-scrollbars",
"title": "Nested OOPIFs with occupied scrollbars and clipped controls",
"suite": "regression",
"tags": ["geometry", "iframe", "scroll"],
"fixture": { "startPath": "/oopif-scrollbars" },
"prompts": {
"en": "Open {url}, observe the page and its frames, report the OOPIF-SCROLLBARS marker, then close the session.",
"zh-CN": "打开 {url},观察页面及其 iframe,报告 OOPIF-SCROLLBARS 标记,然后关闭会话。"
},
"coverage": ["session.start", "session.stop", "page.navigate", "inspect.observe"],
"assertions": {
"site": [
{
"label": "nested frame fixture settled",
"type": "geometry.ready",
"where": { "data.root": true },
"minCount": 1
},
{
"label": "both child frames have occupied scrollbars",
"type": "geometry.scrollbars",
"where": { "data.vertical": true, "data.horizontal": true },
"minCount": 2
}
],
"response": [{ "label": "observed the fixture marker", "includes": "OOPIF-SCROLLBARS" }],
"adapter": [{ "label": "browser session was closed", "key": "sessionStopped" }]
},
"smoke": {
"steps": [
{ "action": "navigate" },
{ "action": "wait-site-event", "type": "geometry.ready", "where": { "root": true } },
{ "action": "observe" }
]
}
}
@@ -0,0 +1,56 @@
import { page, withRun } from "../../../lib/fixtures.mjs";
export default {
id: "oopif-scrollbars",
routes: ["/oopif-scrollbars", "/oopif-scrollbars/frame", "/oopif-scrollbars/nested"],
render({ pathname, runId, query }) {
const root = pathname === "/oopif-scrollbars";
const nested = pathname.endsWith("/nested");
const mode = query.get("scrollbars") ?? "both";
return page({
title: "OOPIF scrollbar geometry regression",
body: `
<style>
main { margin: 0; width: auto; }
body { width: 2000px; height: 2400px; }
html { overflow-anchor: none; }
${!root ? `html { overflow-x: ${mode === "vertical" || mode === "none" ? "hidden" : "scroll"}; overflow-y: ${mode === "horizontal" || mode === "none" ? "hidden" : "scroll"}; } html::-webkit-scrollbar { width: 17px; height: 11px; }` : ""}
button { position: absolute; padding: 0; border: 0; border-radius: 0; }
#probe { left: ${root ? 180 : nested ? 30 : 100}px; top: ${root ? 360 : nested ? 40 : 160}px; width: 60px; height: 20px; }
#edge { position: fixed; right: -20px; bottom: -15px; width: 80px; height: 60px; }
#outside { position: fixed; left: calc(100% + 2px); top: calc(100% + 2px); width: 10px; height: 10px; }
iframe { position: absolute; box-sizing: content-box; width: 420px; height: 320px;
left: 340px; top: 400px; border: 6px solid black; padding: 8px; transform: scale(1.2); transform-origin: 0 0; }
#nested { left: 150px; top: 210px; width: 200px; height: 120px; border-width: 4px; padding: 6px; transform: scale(0.8); }
</style>
<button id="probe" data-geometry-probe>OOPIF-SCROLLBARS</button>
${!root ? '<button id="edge" data-geometry-probe>EDGE</button><button id="outside" data-geometry-probe>OUTSIDE</button>' : ""}
${!nested ? `<iframe id="${root ? "cross" : "nested"}" name="${root ? "cross" : "nested"}" title="${root ? "outer" : "nested"} cross-site frame"></iframe>` : ""}
`,
script: `
history.scrollRestoration = "manual";
const child = document.querySelector("iframe");
if (child) {
const url = new URL(${JSON.stringify(withRun(root ? "/oopif-scrollbars/frame" : "/oopif-scrollbars/nested", runId, { scrollbars: mode }))}, location.href);
url.hostname = location.hostname === "127.0.0.1" ? "localhost" : "127.0.0.1";
child.src = url.href;
}
document.addEventListener("click", event => {
const probe = event.target.closest("[data-geometry-probe]");
if (probe) browserEval.send("geometry.clicked", { probe: probe.id });
});
window.addEventListener("load", () => {
scrollTo(${root ? "80, 240" : nested ? "10, 20" : "40, 100"});
requestAnimationFrame(() => requestAnimationFrame(() => {
document.documentElement.dataset.geometryReady = "true";
if (!${root}) browserEval.send("geometry.scrollbars", {
vertical: innerWidth > document.documentElement.clientWidth,
horizontal: innerHeight > document.documentElement.clientHeight,
});
browserEval.send("geometry.ready", { root: ${root} });
}));
});
`,
});
},
};
@@ -0,0 +1,41 @@
# DOMSnapshot layout-unit regression (#195)
Source: https://github.com/Tencent/BrowserSkill/pull/195
DOMSnapshot bounds and document scroll offsets retain Blink layout units. Treating them as CSS
pixels changes both positions and sizes under device scaling/browser zoom, before iframe
projection or viewport clipping even starts.
The fixture contains a scrolled root, a scrolled same-process iframe, an OOPIF served from the
other loopback hostname, and a nested iframe in each. Owners have borders, padding and CSS
transforms. The geometry test compares the production capture's local and top-level rectangles
against independent DOM border boxes, with at most 2 CSS pixels of layout rounding. It also
checks that each target reads layout metrics only once. No credentials or personal tabs are used.
Run the geometric assertions with Node 22+ and a local Chrome executable:
```sh
BSK_GEOMETRY_CHROME=/path/to/chrome pnpm --filter @browser-skill/extension exec vitest run \
src/tools/__tests__/snapshot-coordinates.browser.test.ts
```
The test launches and cleans up its own headless Chrome profiles. It explicitly verifies the
requested browser zoom and OOPIF topology, so unsupported setups fail rather than silently
testing a different configuration. It is skipped in ordinary unit runs unless the executable
is specified. No browser download or new package dependency is required.
The corpus smoke case checks fixture readiness and observable semantics through the real CLI:
```sh
BSK_AUTO_UPDATE=off pnpm eval:browser smoke --case snapshot-coordinates --bsk ./target/debug/bsk
```
Smoke alone does **not** certify coordinate accuracy; CLI observations do not expose raw boxes.
Use the geometric test above for the regression's numeric assertions.
## Related scrollbar regression
The unit-conversion fixture keeps non-occupying OOPIF scrollbars by default. The same
browser test also runs the dedicated [OOPIF scrollbar fixture](../oopif-scrollbars/README.md),
which covers occupied scrollbars, two OOPIF boundaries, clipping and real clicks.
Append `&classic-scrollbars` to this fixture's URL to enable its normal OOPIF scrollbars.
@@ -0,0 +1,99 @@
import { spawn } from "node:child_process";
import { once } from "node:events";
import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
// This regression owns its browser/profile. It never attaches to a user's Chrome.
export async function withChrome({ executable, deviceScale, zoom }, run) {
const profile = await mkdtemp(join(tmpdir(), "bsk-snapshot-coordinates-"));
let chrome;
let socket;
const pending = new Map();
try {
await mkdir(join(profile, "Default"));
await writeFile(
join(profile, "Default", "Preferences"),
JSON.stringify({
partition: { default_zoom_level: { x: Math.log(zoom) / Math.log(1.2) } },
}),
);
chrome = spawn(
executable,
[
"--headless=new",
"--no-first-run",
"--no-default-browser-check",
"--disable-extensions",
"--site-per-process",
"--remote-debugging-port=0",
"--window-size=1600,1200",
`--force-device-scale-factor=${deviceScale}`,
`--user-data-dir=${profile}`,
"about:blank",
],
{ stdio: ["ignore", "ignore", "pipe"] },
);
const endpoint = await new Promise((resolve, reject) => {
let output = "";
const timeout = setTimeout(
() => reject(new Error(`Chrome startup timed out: ${output}`)),
15_000,
);
const fail = (error) => {
clearTimeout(timeout);
reject(error);
};
chrome.once("error", fail);
chrome.once("exit", (code) => fail(new Error(`Chrome exited (${code}): ${output}`)));
chrome.stderr.on("data", (chunk) => {
output += chunk;
const match = output.match(/DevTools listening on (ws:\/\/\S+)/);
if (match) {
clearTimeout(timeout);
resolve(match[1]);
}
});
});
socket = new WebSocket(endpoint);
await new Promise((resolve, reject) => {
socket.addEventListener("open", resolve, { once: true });
socket.addEventListener("error", reject, { once: true });
});
socket.addEventListener("message", ({ data }) => {
const reply = JSON.parse(data);
const request = pending.get(reply.id);
if (!request) return;
clearTimeout(request.timeout);
pending.delete(reply.id);
if (reply.error) request.reject(new Error(JSON.stringify(reply.error)));
else request.resolve(reply.result);
});
let sequence = 0;
const send = (method, params = {}, sessionId) =>
new Promise((resolve, reject) => {
const id = ++sequence;
const timeout = setTimeout(() => {
pending.delete(id);
reject(new Error(`CDP timed out: ${method}`));
}, 10_000);
pending.set(id, { resolve, reject, timeout });
socket.send(JSON.stringify({ id, method, params, sessionId }));
});
await run(send);
} finally {
for (const request of pending.values()) {
clearTimeout(request.timeout);
request.reject(new Error("regression browser closed"));
}
socket?.close();
if (chrome && chrome.exitCode === null && chrome.signalCode === null) {
const exited = once(chrome, "exit");
const timeout = setTimeout(() => chrome.kill("SIGKILL"), 5_000);
chrome.kill();
await exited;
clearTimeout(timeout);
}
await rm(profile, { recursive: true, force: true });
}
}
@@ -0,0 +1,33 @@
{
"$schema": "../../../schemas/case.schema.json",
"schemaVersion": 1,
"id": "snapshot-coordinates",
"title": "Scrolled and nested frame coordinate fixture",
"suite": "regression",
"tags": ["geometry", "iframe", "scroll"],
"fixture": { "startPath": "/snapshot-coordinates" },
"prompts": {
"en": "Open {url}, observe the page and its frames, report the GEOMETRY marker, then close the session.",
"zh-CN": "打开 {url},观察页面及其 iframe,报告 GEOMETRY 标记,然后关闭会话。"
},
"coverage": ["session.start", "session.stop", "page.navigate", "inspect.observe"],
"assertions": {
"site": [
{
"label": "root fixture scrolled and settled",
"type": "geometry.ready",
"where": { "data.root": true },
"minCount": 1
}
],
"response": [{ "label": "observed the fixture marker", "includes": "GEOMETRY-195" }],
"adapter": [{ "label": "browser session was closed", "key": "sessionStopped" }]
},
"smoke": {
"steps": [
{ "action": "navigate" },
{ "action": "wait-site-event", "type": "geometry.ready", "where": { "root": true } },
{ "action": "observe" }
]
}
}
@@ -0,0 +1,52 @@
import { page, withRun } from "../../../lib/fixtures.mjs";
export default {
id: "snapshot-coordinates",
routes: ["/snapshot-coordinates", "/snapshot-coordinates/frame", "/snapshot-coordinates/nested"],
render({ pathname, runId, query }) {
const root = pathname === "/snapshot-coordinates";
const nested = pathname.endsWith("/nested");
return page({
title: "Snapshot coordinate regression",
body: `
<style>
main { margin: 0; width: auto; }
body { width: 2000px; height: 2400px; }
html { overflow-anchor: none; }
${query.get("scrollbars") === "none" ? "html { scrollbar-width: none; }" : ""}
button { position: absolute; box-sizing: border-box; padding: 0; border: 2px solid black; border-radius: 0; }
#probe { left: ${root ? 180 : 100}px; top: ${root ? 360 : 160}px; width: 120px; height: 40px; }
iframe { position: absolute; box-sizing: content-box; width: 320px; height: 220px;
border: 6px solid black; padding: 8px; transform: scale(1.25); transform-origin: 0 0; }
#same { left: 340px; top: 400px; }
#cross { left: 820px; top: 400px; }
#nested { left: 150px; top: 210px; width: 130px; height: 70px; border-width: 2px; padding: 4px; transform: scale(0.8); }
${nested ? "#probe { left: 30px; top: 40px; width: 60px; height: 20px; }" : ""}
</style>
<button id="probe" data-geometry-probe>GEOMETRY-195</button>
${root ? `<iframe id="same" name="same" title="same-process frame" src="${withRun("/snapshot-coordinates/frame", runId)}"></iframe><iframe id="cross" name="cross" title="cross-site frame"></iframe>` : ""}
${!root && !nested ? `<iframe id="nested" name="nested" title="nested frame" src="${withRun("/snapshot-coordinates/nested", runId)}"></iframe>` : ""}
`,
script: `
history.scrollRestoration = "manual";
const cross = document.querySelector("#cross");
if (cross) {
const url = new URL(${JSON.stringify(withRun("/snapshot-coordinates/frame", runId))}, location.href);
url.hostname = location.hostname === "127.0.0.1" ? "localhost" : "127.0.0.1";
// Isolate unit conversion from the pre-existing OOPIF scrollbar-width
// projection issue. ?classic-scrollbars retains that separate reproduction.
if (!new URLSearchParams(location.search).has("classic-scrollbars")) url.searchParams.set("scrollbars", "none");
cross.src = url.href;
}
window.addEventListener("load", () => {
scrollTo(${root ? "80, 240" : nested ? "10, 20" : "40, 100"});
requestAnimationFrame(() => requestAnimationFrame(() => {
window.geometryReady = true;
document.documentElement.dataset.geometryReady = "true";
browserEval.send("geometry.ready", { root: ${root} });
}));
});
`,
});
},
};
+10 -4
View File
@@ -41,17 +41,23 @@ test("case manifests are discovered, ordered, and grouped into suites", () => {
"diagnostics",
"mobile-emulation",
"generated-form",
"oopif-scrollbars",
"snapshot-coordinates",
],
);
assert.deepEqual(repositorySummary(cases, fixtureRegistry).suites, { core: 6, matrix: 1 });
assert.deepEqual(repositorySummary(cases, fixtureRegistry).suites, {
core: 6,
matrix: 1,
regression: 2,
});
});
test("repository validation links every case to a fixture and valid workflow evidence", () => {
assert.deepEqual(validateRepositoryCases(cases, fixtureRegistry), []);
const summary = repositorySummary(cases, fixtureRegistry);
assert.equal(summary.cases, 7);
assert.equal(summary.fixtureModules, 8);
assert.equal(summary.fixtureRoutes, 12);
assert.equal(summary.cases, 9);
assert.equal(summary.fixtureModules, 10);
assert.equal(summary.fixtureRoutes, 18);
});
test("manifest validation rejects unknown operations and incomplete workflow steps", () => {
+41 -15
View File
@@ -21,6 +21,7 @@ $ErrorActionPreference = "Stop"
$Repo = if ($env:BSK_REPO) { $env:BSK_REPO } else { "Tencent/BrowserSkill" }
$InstallDir = if ($env:BSK_INSTALL_DIR) { $env:BSK_INSTALL_DIR } else { Join-Path $HOME ".local\bin" }
$InstallDir = $ExecutionContext.SessionState.Path.GetUnresolvedProviderPathFromPSPath($InstallDir)
$GitHub = "https://github.com/${Repo}"
function Write-Log {
@@ -64,7 +65,7 @@ function Get-PlatformTriple {
function Add-ToUserPath {
param([string]$Dir)
$currentUserPath = [Environment]::GetEnvironmentVariable("PATH", "User") -split ";" | Where-Object { $_ }
$currentUserPath = @([Environment]::GetEnvironmentVariable("PATH", "User") -split ";" | Where-Object { $_ })
if ($currentUserPath -contains $Dir) {
Write-Log "$Dir is already in your user PATH"
@@ -90,30 +91,59 @@ function Add-ToSessionPath {
# ── Git Bash (bash environment) PATH helper ──────────────────────────────────
function Add-ToBashProfile {
param([string]$Dir)
param([string]$Dir, [string]$BashRc = (Join-Path $HOME ".bashrc"))
# Convert Windows path (e.g. C:\Users\foo\.local\bin) to Git-Bash Unix-style (/c/Users/foo/.local/bin)
$unixPath = $Dir -replace '\\', '/'
if ($unixPath -match '^([A-Z]):(.*)$') {
$unixPath = '/' + $matches[1].ToLower() + $matches[2]
}
$bashRc = Join-Path $HOME ".bashrc"
$exportLine = "export PATH=""${unixPath}:`$PATH"" # bsk CLI"
# Single-quote the literal directory; only the existing PATH is expanded.
$shellQuote = "'" + [char]34 + "'" + [char]34 + "'"
$quotedPath = "'" + $unixPath.Replace("'", $shellQuote) + "'"
$exportLine = "export PATH=${quotedPath}:`"`$PATH`" # bsk CLI"
if (Test-Path $bashRc) {
$content = Get-Content $bashRc -Raw -ErrorAction SilentlyContinue
if ($content -match [regex]::Escape($unixPath)) {
if (Test-Path -LiteralPath $BashRc) {
$content = [System.IO.File]::ReadAllText($BashRc)
if ($content.Contains($exportLine)) {
Write-Log "$unixPath is already in ~/.bashrc"
return
}
}
Add-Content $bashRc "`n$exportLine" -Encoding ASCII
# Explicit BOM-less UTF-8 also works in Windows PowerShell 5.1.
[System.IO.File]::AppendAllText($BashRc, "`n$exportLine`n", (New-Object System.Text.UTF8Encoding($false)))
Write-Log "added ${unixPath} to ~/.bashrc"
}
# ── Main ──────────────────────────────────────────────────────────────────────
# Stage on the destination volume before stopping the daemon. Never truncate
# the installed executable: failed replacement must leave it usable.
function Install-Binary {
param([string]$Source, [string]$Target)
$staged = "$Target.install-$([Guid]::NewGuid().ToString('N'))"
try {
[System.IO.File]::Copy($Source, $staged)
if ([System.IO.File]::Exists($Target)) {
# Use the downloaded CLI: older versions have broken Windows
# liveness checks. Stop verifies daemon identity before terminating it.
& $Source daemon stop
if ($LASTEXITCODE -ne 0) { throw "could not stop bsk daemon; existing installation was not replaced" }
# PowerShell 5.1 converts $null to an empty path for string parameters.
[System.IO.File]::Replace($staged, $Target, [NullString]::Value)
Write-Log "daemon will restart automatically on the next browser command"
}
else {
[System.IO.File]::Move($staged, $Target)
}
}
finally {
if ([System.IO.File]::Exists($staged)) { [System.IO.File]::Delete($staged) }
}
}
function Main {
$platform = Get-PlatformTriple
@@ -184,7 +214,7 @@ function Main {
New-Item -ItemType Directory -Path $InstallDir -Force | Out-Null
}
Copy-Item -Path (Join-Path $tempDir "bsk.exe") -Destination (Join-Path $InstallDir "bsk.exe") -Force
Install-Binary -Source (Join-Path $tempDir "bsk.exe") -Target (Join-Path $InstallDir "bsk.exe")
Write-Log "installed bsk to $InstallDir\bsk.exe"
@@ -199,12 +229,8 @@ function Main {
# Verify
$bskPath = Join-Path $InstallDir "bsk.exe"
if (Get-Command bsk -ErrorAction SilentlyContinue) {
& bsk --version
}
else {
Write-Log "verify install: & ""$bskPath"" --version"
}
& $bskPath --version
if ($LASTEXITCODE -ne 0) { throw "installed bsk failed verification" }
Write-Log "done"
Write-Host ""
+141 -154
View File
@@ -1,13 +1,51 @@
# dsh-plugin-browserskill
# BrowserSkill for DeepSeek Harness
npm: [`@wxg-prc-cpg/browser-skill-dsh-plugin`](https://www.npmjs.com/package/@wxg-prc-cpg/browser-skill-dsh-plugin)
[![npm version](https://img.shields.io/npm/v/@wxg-prc-cpg/browser-skill-dsh-plugin)](https://www.npmjs.com/package/@wxg-prc-cpg/browser-skill-dsh-plugin)
A [DeepSeek Harness](https://github.com/deepseek-ai/deepseek-harness) (dsh) tool plugin that exposes
[BrowserSkill](https://github.com/Tencent/BrowserSkill) (`bsk`) browser automation to the model.
Use [BrowserSkill](https://github.com/Tencent/BrowserSkill) in
[DeepSeek Harness](https://github.com/deepseek-ai/deepseek-harness) (`dsh`) to browse
websites, fill forms, and capture screenshots through native `browser_*` tools.
Browser tasks run in Agent Windows, with a live view in the dsh Web UI.
Each action maps to one `bsk <cmd> --json` invocation: the plugin spawns the bsk CLI, parses its
structured JSON output, and returns a canonical typed value. The bsk daemon, browser, and browser
extension keep owning the actual browser control — this package is a thin, well-typed bridge.
## Installation
Before installing the plugin:
- Install [DeepSeek Harness](https://github.com/deepseek-ai/deepseek-harness) and
[pnpm](https://pnpm.io/installation), which dsh uses to manage plugins.
- Install the `bsk` CLI and connect the BrowserSkill extension in Chrome or Edge.
Follow the [BrowserSkill setup guide](https://github.com/Tencent/BrowserSkill#quick-start).
- Make sure `bsk` is on the `PATH` used to start dsh, or set `bskPath` in the plugin configuration.
Install the plugin into the `web` profile, then start it:
```sh
dsh plugin --profile web add @wxg-prc-cpg/browser-skill-dsh-plugin
dsh --profile web
```
Replace `web` with your profile name if you use a different profile. The plugin
includes the `browser-skill` skill; no separate `bsk install-skill` step is needed.
In a conversation, try:
```text
/browser-skill open example.com and summarize the page.
```
By default, the browser tools become available when the skill is invoked.
## Updating
Installed plugins do not update automatically. To upgrade this plugin to npm's
`latest` version, including versions outside the profile's saved dependency range:
```sh
dsh plugin --profile web update @wxg-prc-cpg/browser-skill-dsh-plugin --latest
```
Restart that dsh profile after upgrading. This command updates the plugin; update
the `bsk` CLI and browser extension separately when a release requires it.
## Tools
@@ -16,27 +54,14 @@ extension keep owning the actual browser control — this package is a thin, wel
| `browser_session` | `start`, `stop`, `list` | Manage plugin-owned Agent Window sessions. |
| `browser_page` | `navigate`, `back`, `forward`, `reload`, `wait` | Navigate the active tab and wait for page lifecycle events. |
| `browser_inspect` | `observe`, `snapshot`, `html`, `screenshot`, `console`, `network` | Read semantic or diagnostic page state and capture screenshots. |
| `browser_interact` | `click`, `hover`, `fill`, `select`, `press` | Interact with controls using fresh refs or selectors. |
| `browser_interact` | `click`, `hover`, `scroll-to`, `focus`, `blur`, `fill`, `select`, `press` | Interact with controls using fresh refs or selectors. |
| `browser_tabs` | `list`, `create`, `select`, `close`, `borrow`, `return` | Manage Agent Window tabs and temporarily borrow user tabs. |
| `browser_assist` | `resize`, `emulate`, `request-help` | Resize or emulate the browser and pause for human-only steps. |
`bsk evaluate` and `bsk record` are intentionally not exposed by this plugin: arbitrary page
evaluation is a higher-risk capability, while recording is long-running and needs a dedicated
DeepSeek Harness lifecycle before it can be added safely.
For `browser_interact` with `action: "scroll-to"`, see the
[scroll-to reference](../../docs/scroll-to.md) for parameters, visible bounds and errors.
## Agent skill (progressive disclosure)
Beyond the tools, the plugin publishes the **`browser-skill` agent skill** through the harness's
official skill seam (`ctx.skills.register`): the catalog entry (name + routing description) is
resident in `<available_skills>`, and the body is loaded only when the model invokes the `skill`
tool. Its single source is the DSH-specific `skill/SKILL.md`, which documents only structured
`browser_*` calls and their plugin semantics. The repository-root CLI skill is intentionally not
concatenated: its command examples belong to a different execution interface and would bypass
the plugin's ownership, live observation UI, cancellation, and cleanup path if followed directly.
The build rejects internal CLI-name leakage, command-line code blocks, unknown browser tools, and
missing supported browser tools before embedding the Markdown. Registration and every pre-step
catalog snapshot are pure in-memory reads (no disk/process/daemon); compositions without the skill
seam degrade silently.
Arbitrary page-script evaluation and interaction recording are not supported.
## Multi-session model
@@ -55,136 +80,73 @@ an explicit `session` argument naming a foreign or unknown id is rejected, the `
`browser_session` shows plugin-created sessions only (no daemon-wide view), and stop/unload cleanup
can never touch a session owned by another program.
## Installation
The plugin follows the standard dsh bundle layout (`dsh.bundle` manifest + `cordis.patch.yml`):
```sh
dsh plugin --profile <name> add @wxg-prc-cpg/browser-skill-dsh-plugin
dsh --profile <name>
```
Prerequisite: the `bsk` CLI must be installed and on `PATH`, and the BrowserSkill browser extension
(Chrome or Edge) must be connected — see the
[BrowserSkill README](https://github.com/Tencent/BrowserSkill). When bsk is missing, tool calls fail
with install guidance instead of a bare spawn error.
## Configuration
All fields are optional and validated through the plugin's Schemastery `Config`:
After installing the plugin, edit your profile's `cordis.patch.yml`. For the `web`
profile, the default location is `~/.dsh/profiles/web/cordis.patch.yml`. If you set
`DSH_HOME`, use `$DSH_HOME/profiles/web/cordis.patch.yml` instead. Replace `web` with
your profile name as needed.
If the file contains only comments and `[]`, keep the comments and replace `[]`
with the YAML below. If it already contains patch entries, add this entry to the
existing list or edit its existing `id: browserskill` entry. Keep a single
top-level YAML list. This overrides the plugin registered by the installed bundle:
```yaml
# cordis.patch.yml override example
- insert:
- id: browserskill
name: "@wxg-prc-cpg/browser-skill-dsh-plugin"
config:
bskPath: bsk # path to the bsk binary (default: resolve from PATH)
defaultTimeoutMs: 120000
maxSessions: 5
# observationEnabled: true # live PiP/overlay observation (below)
# thumbnailIntervalMs: 1500 # frame cadence while a session is active
# idleIntervalMs: 8000 # idle cadence / recent-activity window
# lazyTools: true # reveal browser_* tools only after the skill is invoked
- id: browserskill
config:
bskPath: bsk
defaultTimeoutMs: 120000
maxSessions: 5
observationEnabled: true
thumbnailIntervalMs: 1500
idleIntervalMs: 8000
lazyTools: true
```
- **`lazyTools` (default `true`)** — the final progressive-disclosure stage: the six
`browser_*` tool schemas stay OUT of the system prompt (zero schema tokens) until the
`browser-skill` skill is actually invoked — the skill catalog entry is the only
advertisement. One successful invocation (model tool call, or a `/browser-skill` user
gesture) registers the whole suite for the rest of the process; repeated invocations are
no-ops, and sessions resumed with a past invocation in their durable log reveal the suite
on entry. Set `false` for the legacy always-on registration.
Change `bskPath` to the full path of your CLI binary if it is not on dsh's `PATH`.
A patch replaces the entry's entire `config` object, so keep all overrides you need
together in that object.
## Observation overlay (PiP mini-window)
Configuration changes follow `dsh.profile.patchReload` in the profile's
`package.json`: `live` (the default for `web`) applies changes when you save the
patch file; `startup` requires restarting the profile. Restart after upgrading
the plugin in either case.
When the plugin runs inside the dsh Web UI, an **observation overlay** floats over the app
(registered into the `shell.overlay` seat): a breathing thumbnail per owned session plus its
current action and elapsed time. The card docks at the top-right of the content area (clear of
the composer and the shell's header controls) and wears the **BrowserSkill product family's own
look**: the overlay reuses `@browser-skill/ui` components (`Button`, `cn`) and its oklch design
tokens (`--card`, `--primary`, `--destructive`, `--ring`, …), status dots spec'd after the
extension popup's connection indicator, and Remix icons. The BSK utility sheet is compiled
scoped under the `.bsk-obs` root class (`scripts/build-client-css.mjs`), so the overlay looks
like the BrowserSkill extension without leaking a single selector into the host shell — and the
shell's theme cannot bleed back in.
All fields are optional; omitted fields use the defaults below:
- **Lifecycle**: hidden while the plugin owns no sessions; appears on the first
`browser_session` start action; disappears when all sessions stop (or the plugin unloads).
- **Focus view**: status row (green/idle/red dot + session + action + mm:ss), the latest page
frame (while viewed, refreshes every ~1.5s while active, ~8s when idle; the last good frame stays on
stage while the next one loads, and is kept on errors so the card does not flash),
and a compact icon toolbar (Interrupt + Pop out, hover for the label).
- **Interrupt**: one click kills the in-flight bsk command of the focus session (same semantics
as the chat Stop button — the current action fails, the agent run may continue). Strip items
carry their own hover interrupt button.
- **Multi-session strip**: every session gets a tile (mini frame + id + status dot); focus
auto-follows the most recently active session; clicking a tile pins focus (pin badge, click
again to release); errored sessions get a red edge without stealing focus; sessions the daemon
lost are greyed out; prolonged daemon/browser outage shows "browser unavailable" and greys
the interrupt button until captures recover.
- **Drag & resize**: drag the header to move the card; drag any of the four
corners to resize (min 240×180, max 80% of the viewport; no visible grip).
Both are remembered for the page lifetime.
- **Pop out (PiP)**: upgrades the card into a native Document PiP window (requires a user
gesture, per browser rules), sized from the current card; closing the PiP falls back to the
in-page card with state intact. Browsers without Document PiP simply hide the button.
- **Screenshot demand**: the client requests periodic screenshots while an observation view is
visible, using the PiP document's visibility when popped out. Hidden or collapsed views can
keep subscribing to state without requesting screenshots. The first screenshot subscriber
starts capture scheduling; the last one's departure cancels timers and skips queued captures.
An already running capture may finish. Demand applies to the service's owned sessions, not just
the selected session.
- **Wire**: `GET /bsk-observation/events` is the live synchronization channel (SSE). Every connection,
including reconnects, starts with `{ type: "snapshot", sessions, available }`, followed by
`upsert`, `remove`, `reset`, and `availability` events on the same stream. Replace local state
with each snapshot before applying subsequent events. Use `?thumbnails=0` for state-only
subscriptions or `?thumbnails=1` to request screenshots; omitting the parameter defaults to
requesting screenshots for compatibility. Closing the stream releases its screenshot demand.
`GET /bsk-observation/state` remains a one-time `{ sessions, available }` read and does not
request screenshots. It has no ordering guarantee relative to a separate SSE connection;
live clients should initialize and resynchronize from the stream's snapshot instead.
The host also serves `POST /bsk-observation/interrupt`, `POST /bsk-observation/stop`, and
`GET /bsk-observation/thumbnail/<attachmentId>` through the dsh `webServer` route seam
(dsh 0.1's Typert Remote pipeline is closed to out-of-tree packages). All commands for one
session — tool calls and frame captures alike — run through a per-session FIFO, because the
daemon accepts only one unfinished command per session.
- **Programmatic subscription**: `ObservationService.subscribe(listener, { thumbnails: false })`
subscribes to state without requesting screenshots. On an active service, the listener receives
the initial `snapshot` synchronously, before `subscribe` returns, then receives subsequent
changes. `thumbnails` defaults to `true`, so callers should choose it explicitly. The returned
unsubscribe function is safe to call repeatedly and releases this subscription's screenshot
demand. Subscribing after service disposal is a no-op.
- **Trust model**: these routes expose live screenshots (and an interrupt write), so they
replicate the browser-trust fence dsh applies to its own `/api` routes: the request Host
must be a loopback authority (`localhost`, `127.0.0.0/8`, `[::1]`), a present Origin must
match the Host, `sec-fetch-site: cross-site` is refused, and POST requires an
`application/json` body (cross-site simple requests can never satisfy that). The channel is
therefore built for **loopback-only serving** — binding the dsh web server to `0.0.0.0` and
reaching it through a LAN address will (deliberately) fail the fence; do not put these
routes behind a non-loopback reverse proxy without adding your own authentication.
- Configure with `observationEnabled` / `thumbnailIntervalMs` / `idleIntervalMs`.
| Option | Default | Purpose |
| --- | --- | --- |
| `bskPath` | `bsk` | Path to the CLI binary. |
| `defaultTimeoutMs` | `120000` | Default command timeout in milliseconds. |
| `maxSessions` | `5` | Maximum concurrent sessions started by this plugin. |
| `observationEnabled` | `true` | Enable live browser observation. |
| `thumbnailIntervalMs` | `1500` | Screenshot interval for active sessions, in milliseconds. |
| `idleIntervalMs` | `8000` | Screenshot interval for idle sessions and the recent-activity window, in milliseconds. |
| `lazyTools` | `true` | Reveal the browser tools when the skill is invoked. Set `false` to register them at startup. |
## Behavior notes
With `lazyTools: true`, only the skill's catalog entry is initially advertised to
the model. The six `browser_*` tool schemas are added to the system prompt after
the `browser-skill` skill is successfully invoked, either by the model or through
`/browser-skill`. Set `lazyTools: false` to make the tools available immediately.
- **Cancellation**: aborting a tool call (`exec.signal`) kills the underlying bsk child process,
matching BrowserSkill's cooperative tool-cancellation model.
- **UI cards**: calls render as terminal cards (command line as title, output as the completed
card). Screenshots additionally attach the image itself when the host mounts an attachment store
and the active model route declares image input; otherwise the PNG path is returned.
- **Web UI toolview (browser half)**: the package is dual-face. `dsh.client` (platform `web`) ships
`lib/client.cjs`, which registers a keyed `tool.call.toolview` view for `browser_inspect`. The
custom view keeps a terminal block for every inspect action and, when a screenshot result carries
an image block, resolves the durable attachment through the client session's authorized
`readAttachment` RPC and renders it with the shared `MessageImage` thumbnail/lightbox atoms.
The other five tools keep the stock terminal card. The bundle follows the dsh client
contract: a CJS closure factory handed to `window.__ModuleLoader__.load`, platform modules
(`react`, `dsh-client-ui-*`) external, everything else inlined, CSS Modules compiled by
lightningcss.
- **Errors**: non-zero bsk exits surface the CLI's JSON error envelope (`code`, `message`, `hint`)
so the model gets the daemon's actionable guidance.
- **Long-running work** (e.g. `bsk record`) is not backgrounded via `ctx.jobs` yet — tracked as a
follow-up.
## Live browser view
The dsh Web UI shows the plugin's browser sessions in a floating panel. If your
profile provides the `dsh-better-sidebar` integration, the view appears in a
**Browser Skill** sidebar tab instead.
- See the current action, elapsed time, and recent screenshot for each session.
- Select a session to focus on it. The sidebar view follows the current conversation.
- Use **Interrupt** to cancel the current browser command. The agent may continue
with another action afterward.
- Drag or resize the floating panel, or use **Pop out** to open a Picture-in-Picture
window in browsers that support it.
- Periodic screenshots are requested while a browser observation view is visible.
Configure the active and idle intervals with the options above.
The observation endpoints require a loopback address such as `localhost` or
`127.0.0.1`. Access through a LAN hostname or non-loopback reverse proxy is not supported.
## Development
@@ -195,22 +157,47 @@ pnpm --filter @wxg-prc-cpg/browser-skill-dsh-plugin test # unit tests mock b
pnpm --filter @wxg-prc-cpg/browser-skill-dsh-plugin build # tsdown -> lib/
```
See the [development notes](https://github.com/Tencent/BrowserSkill/blob/main/packages/dsh-plugin-browserskill/docs/development.md)
for skill registration, tool results, and observation APIs in the current source.
## Publishing
The GitHub Actions workflow **Release dsh plugin** publishes this package to npm
as `@wxg-prc-cpg/browser-skill-dsh-plugin`. The Cordis plugin id, client
bundle registration, and `cordis.patch.yml` name are the same specifier so
dsh can import and materialize the plugin without a ModuleLoader id mismatch.
The [Release dsh plugin workflow](https://github.com/Tencent/BrowserSkill/blob/main/.github/workflows/release-dsh-plugin.yml)
publishes the package and this README to npm. Pushing a `dsh-plugin-vX.Y.Z` tag
triggers it; ordinary commits to `main` do not.
Trigger it by pushing a tag that matches `package.json`'s `version`:
1. Commit this README and any other changes intended for the release.
2. From the repository root, run the
[release script](https://github.com/Tencent/BrowserSkill/blob/main/scripts/release.mjs)
with a new stable version, replacing `<version>` below. The script updates the
CLI, extension, and DSH plugin versions, commits the version changes, and creates
their release tags:
```sh
git tag dsh-plugin-v0.1.2
git push origin dsh-plugin-v0.1.2
```
```sh
node scripts/release.mjs <version>
```
Or run the workflow from the Actions tab (`workflow_dispatch`). The job reads the
`NPM_TOKEN` secret from the `npm-publish` GitHub Environment.
3. Push the version commit to the release branch, then push the DSH plugin tag
created by the script, using the same `<version>`:
```sh
git push origin HEAD
git push origin dsh-plugin-v<version>
```
This publishes the DSH plugin. Push the CLI and extension tags separately when
those components are ready for release.
The workflow checks the version, runs typechecks and tests, builds the package,
and publishes it to npm.
You can also run the workflow manually from GitHub Actions on the intended release
ref. Both triggers require an unpublished version and the `NPM_TOKEN` secret in the
`npm-publish` GitHub Environment.
npm updates the package README only when a new version is published, including
for documentation-only changes. Published versions cannot be overwritten. See
[npm's README update rules](https://docs.npmjs.com/about-package-readme-files/).
## License
@@ -0,0 +1,90 @@
# DSH plugin development notes
For installation, usage, and configuration, see the [plugin README](../README.md).
These notes describe the current source on this branch; a published npm version may
not include later changes.
## Tool execution
The plugin runs the `bsk` CLI and parses its JSON output. The daemon and browser
extension perform the browser operations. The published package name is also the
Cordis plugin id, client bundle registration id, and `cordis.patch.yml` plugin name.
Keep these identifiers aligned so dsh can load both halves of the plugin.
## Agent skill
Beyond the tools, the plugin publishes the **`browser-skill` agent skill** through the harness's
official skill seam (`ctx.skills.register`): the catalog entry (name + routing description) is
resident in `<available_skills>`, and the body is loaded only when the model invokes the `skill`
tool. Its single source is the DSH-specific `skill/SKILL.md`, which documents only structured
`browser_*` calls and their plugin semantics. The repository-root CLI skill is intentionally not
concatenated: its command examples belong to a different execution interface and would bypass
the plugin's ownership, live observation UI, cancellation, and cleanup path if followed directly.
The build rejects internal CLI-name leakage, command-line code blocks, unknown browser tools, and
missing supported browser tools before embedding the Markdown. Registration and every pre-step
catalog snapshot are pure in-memory reads (no disk/process/daemon); compositions without the skill
seam degrade silently.
With `lazyTools: true`, the six tool schemas stay out of the system prompt until
the skill is successfully invoked. Registration happens once and lasts until the
plugin is unloaded. Repeated invocations do not register duplicate tools. Entering
a resumed conversation whose history contains a successful skill invocation also
registers the tools. Setting `lazyTools: false` registers them at plugin startup.
## Observation subscriptions and routes
- **Screenshot demand**: the client requests periodic screenshots while an observation view is
visible, using the PiP document's visibility when popped out. Hidden or collapsed views can
keep subscribing to state without requesting screenshots. The first screenshot subscriber
starts capture scheduling; the last one's departure cancels timers and skips queued captures.
An already running capture may finish. Demand applies to the service's owned sessions, not just
the selected session.
- **Wire**: `GET /bsk-observation/events` is the live synchronization channel (SSE). Every connection,
including reconnects, starts with `{ type: "snapshot", sessions, available }`, followed by
`upsert`, `remove`, `reset`, and `availability` events on the same stream. Replace local state
with each snapshot before applying subsequent events. Use `?thumbnails=0` for state-only
subscriptions or `?thumbnails=1` to request screenshots; omitting the parameter defaults to
requesting screenshots for compatibility. Closing the stream releases its screenshot demand.
`GET /bsk-observation/state` remains a one-time `{ sessions, available }` read and does not
request screenshots. It has no ordering guarantee relative to a separate SSE connection;
live clients should initialize and resynchronize from the stream's snapshot instead.
The host also serves `POST /bsk-observation/interrupt`, `POST /bsk-observation/stop`, and
`GET /bsk-observation/thumbnail/<attachmentId>` through the dsh `webServer` route seam
(dsh 0.1's Typert Remote pipeline is closed to out-of-tree packages). All commands for one
session — tool calls and frame captures alike — run through a per-session FIFO, because the
daemon accepts only one unfinished command per session.
- **Programmatic subscription**: `ObservationService.subscribe(listener, { thumbnails: false })`
subscribes to state without requesting screenshots. On an active service, the listener receives
the initial `snapshot` synchronously, before `subscribe` returns, then receives subsequent
changes. `thumbnails` defaults to `true`, so callers should choose it explicitly. The returned
unsubscribe function is safe to call repeatedly and releases this subscription's screenshot
demand. Subscribing after service disposal is a no-op.
- **Trust model**: these routes expose live screenshots (and an interrupt write), so they
replicate the browser-trust fence dsh applies to its own `/api` routes: the request Host
must be a loopback authority (`localhost`, `127.0.0.0/8`, `[::1]`), a present Origin must
match the Host, `sec-fetch-site: cross-site` is refused, and POST requires an
`application/json` body (cross-site simple requests can never satisfy that). The channel is
therefore built for **loopback-only serving** — binding the dsh web server to `0.0.0.0` and
reaching it through a LAN address will (deliberately) fail the fence; do not put these
routes behind a non-loopback reverse proxy without adding your own authentication.
## Tool results and cancellation
- **Cancellation**: aborting a tool call (`exec.signal`) kills the underlying bsk child process,
matching BrowserSkill's cooperative tool-cancellation model.
- **UI cards**: calls render as terminal cards (command line as title, output as the completed
card). Screenshots additionally attach the image itself when the host mounts an attachment store
and the active model route declares image input; otherwise the PNG path is returned.
- **Web UI toolview (browser half)**: the package is dual-face. `dsh.client` (platform `web`) ships
`lib/client.cjs`, which registers a keyed `tool.call.toolview` view for `browser_inspect`. The
custom view keeps a terminal block for every inspect action and, when a screenshot result carries
an image block, resolves the durable attachment through the client session's authorized
`readAttachment` RPC and renders it with the shared `MessageImage` thumbnail/lightbox atoms.
The other five tools keep the stock terminal card. The bundle follows the dsh client
contract: a CJS closure factory handed to `window.__ModuleLoader__.load`, platform modules
(`react`, `dsh-client-ui-*`) external, everything else inlined, CSS Modules compiled by
lightningcss.
- **Errors**: non-zero bsk exits surface the CLI's JSON error envelope (`code`, `message`, `hint`)
so the model gets the daemon's actionable guidance.
- **Long-running work** (e.g. `bsk record`) is not backgrounded via `ctx.jobs` yet — tracked as a
follow-up.
@@ -1,7 +1,7 @@
{
"name": "@wxg-prc-cpg/browser-skill-dsh-plugin",
"description": "DeepSeek Harness tool plugin that exposes BrowserSkill browser automation (browser_* tools) to the model",
"version": "0.2.0",
"version": "0.2.1",
"license": "MIT",
"publishConfig": {
"access": "public",
@@ -51,8 +51,13 @@ Use `browser_inspect` action `observe` as the primary semantic page view. It ret
text, and `@eN` refs. Prefer fresh refs over raw selectors. Refs invalidate after navigation and may
also become stale after large DOM changes, so observe again before the next interaction.
Use `browser_interact` for click, hover, fill, select, and key actions. An observation marks a
hover-only surface as `@e1 button "Products" [hover first: Shoes | Bags]`. The listed items are
Use `browser_interact` with `action=scroll-to` to bring an element into view. Its bounds are the
visible border-box portion in top-level viewport CSS pixels after ancestor clipping; partial
visibility is enough. Hidden or fully clipped targets fail. This does not test occlusion by other
elements. Use fresh element refs for iframe/shadow-root targets; selectors search the main document.
Use `browser_interact` for click, hover, scroll-to, focus, blur, fill, select, and key actions. Focus and blur
explicitly enter or leave UI states triggered by focus changes. An observation marks a hover-only surface as `@e1 button "Products" [hover first: Shoes | Bags]`. The listed items are
labels, not usable refs: hover the trigger, observe again, then act on the revealed item's own ref.
Do not click the trigger itself unless the user wants the trigger's action.
@@ -167,12 +167,15 @@ const BROWSER_TOOL_SPECS: BrowserToolSpec[] = [
{
name: "browser_interact",
description:
"Interact with an element in the active Agent Window tab. Actions: click, hover, fill, select, " +
"press. click/hover/fill/select require target; fill also requires value; select requires " +
"Interact with an element in the active Agent Window tab. Actions: click, hover, scroll-to, focus, blur, fill, select, " +
"press. click/hover/scroll-to/focus/blur/fill/select require target; fill also requires value; select requires " +
"values; press requires key and may optionally focus target first.",
actions: {
click: "interact.click",
hover: "interact.hover",
"scroll-to": "interact.scroll-to",
focus: "interact.focus",
blur: "interact.blur",
fill: "interact.fill",
select: "interact.select",
press: "interact.press",
@@ -562,6 +562,10 @@ export function actionForLabel(label: string): string {
return "clicking";
case "hover":
return "hovering";
case "focus":
return "focusing";
case "blur":
return "blurring";
case "fill":
return "filling";
case "select":
@@ -13,7 +13,7 @@ import type { ToolDeps } from "./tools";
const MODIFIERS = ["alt", "ctrl", "meta", "shift"] as const;
/** Add hover and select without bypassing session ownership or observation. */
/** Add interaction primitives without bypassing session ownership or observation. */
export function registerPhaseOneInteractionTools(
deps: ToolDeps,
register: ToolRegistrar,
@@ -100,6 +100,171 @@ export function registerPhaseOneInteractionTools(
}),
);
register(
defineTool({
name: "interact.scroll-to",
description:
"Scroll an element and its frame owners into view. Returns the visible portion's " +
"bounds in top-level viewport CSS pixels; fails if no visible area remains. " +
"Use a fresh ref for iframe targets; selectors search the main document.",
parameters: {
target: {
type: "string",
required: true,
description: "Element ref (@e3 / e3) or main-document CSS selector to scroll into view.",
},
session: SESSION_PARAM,
tabId: TAB_ID_PARAM,
timeoutMs: TIMEOUT_MS_PARAM,
},
output: {
schema: {
type: "object",
additionalProperties: false,
properties: {
session: { type: "string", required: true },
tabId: { type: "integer", required: true },
x: { type: "number", required: true },
y: { type: "number", required: true },
width: { type: "number", required: true },
height: { type: "number", required: true },
},
},
render: (_args, value) => [
{
type: "text",
text:
`[session ${value.session}] scrolled into view on tab ${value.tabId}: ` +
`(${value.x}, ${value.y}, ${value.width}, ${value.height})`,
},
],
},
async execute(args, exec) {
requireNonEmpty(args.target, "target");
requirePositive(args.timeoutMs, "timeoutMs");
const sessionId = registry.resolve(args.session, "browser_interact(action=scroll-to)");
const cmdArgs = ["scroll-to", "--session", sessionId];
appendTabId(cmdArgs, args.tabId);
if (args.timeoutMs !== undefined) cmdArgs.push("--timeout", `${args.timeoutMs}ms`);
appendTarget(cmdArgs, args.target);
const reply = (await runtime.run(
exec,
cmdArgs,
"scroll-to",
sessionId,
runnerTimeout(deps, args.timeoutMs),
)) as {
tab_id: number;
x: number;
y: number;
width: number;
height: number;
};
return {
session: sessionId,
tabId: reply.tab_id,
x: reply.x,
y: reply.y,
width: reply.width,
height: reply.height,
};
},
presentCall: (args) => ({
card: "terminal",
title: runtime.commandLine([
"scroll-to",
args.target,
"--session",
args.session ?? "(current)",
]),
description: "Scroll an element into view",
}),
presentResult: runtime.presentTerminalResult,
}),
);
for (const action of ["focus", "blur"] as const) {
register(
defineTool({
name: `interact.${action}`,
description:
action === "focus"
? "Focus an element and verify its DOM focus state."
: "Remove focus from an element and report whether it was focused.",
parameters: {
target: {
type: "string",
required: true,
description: "Snapshot ref (@e3 / e3) or CSS selector of the element.",
},
session: SESSION_PARAM,
tabId: TAB_ID_PARAM,
timeoutMs: TIMEOUT_MS_PARAM,
},
output: {
schema: {
type: "object",
additionalProperties: false,
properties: {
session: { type: "string", required: true },
tabId: { type: "integer", required: true },
focused: { type: "boolean", required: true },
wasFocused: {
type: "boolean",
description: "Whether the target was focused before blur.",
},
},
},
render: (_args, value) => [
{
type: "text",
text:
`[session ${value.session}] ${action} on tab ${value.tabId}: focused=${value.focused}` +
(value.wasFocused === undefined ? "" : `, wasFocused=${value.wasFocused}`),
},
],
},
async execute(args, exec) {
requireNonEmpty(args.target, "target");
requirePositive(args.timeoutMs, "timeoutMs");
const sessionId = registry.resolve(args.session, `browser_interact(action=${action})`);
const cmdArgs = [action, "--session", sessionId];
appendTabId(cmdArgs, args.tabId);
if (args.timeoutMs !== undefined) cmdArgs.push("--timeout", `${args.timeoutMs}ms`);
appendTarget(cmdArgs, args.target);
const reply = (await runtime.run(
exec,
cmdArgs,
action,
sessionId,
runnerTimeout(deps, args.timeoutMs),
)) as {
tab_id: number;
focused: boolean;
was_focused?: boolean;
};
return {
session: sessionId,
tabId: reply.tab_id,
focused: reply.focused,
...(action === "blur" ? { wasFocused: reply.was_focused } : {}),
};
},
presentCall: (args) => ({
card: "terminal",
title: runtime.commandLine([
action,
args.target,
"--session",
args.session ?? "(current)",
]),
description: action === "focus" ? "Focus an element" : "Remove focus from an element",
}),
presentResult: runtime.presentTerminalResult,
}),
);
}
register(
defineTool({
name: "interact.select",
+51 -10
View File
@@ -5,7 +5,7 @@
* map the CLI's JSON error envelope onto a thrown `BskError`.
*/
import { type ChildProcess, spawn } from "node:child_process";
import { type ChildProcess, type SpawnOptionsWithoutStdio, spawn } from "node:child_process";
/** Shape of the JSON error envelope `bsk --json` prints on failure. */
export interface BskErrorBody {
@@ -53,7 +53,11 @@ export interface BskRunOptions {
}
/** Minimal spawn signature so tests can substitute a fake child process. */
export type SpawnImpl = (command: string, args: string[]) => ChildProcess;
export type SpawnImpl = (
command: string,
args: string[],
options?: SpawnOptionsWithoutStdio,
) => ChildProcess;
export interface BskRunner {
/** Run `bsk <args...> --json` and collect its output. */
@@ -64,29 +68,66 @@ export interface BskRunner {
killFor(tag: string): number;
}
// Business RPCs translate SIGINT into the daemon's cancel(rpc_id) protocol.
// Give that bounded reconciliation path time to settle before the hard kill.
// Business RPCs translate Ctrl-C / opt-in stdin EOF into cancel(rpc_id).
// Allow reconciliation before hard-killing an old or unresponsive CLI.
const KILL_GRACE_MS = 3000;
// Windows IPC may spend 5s connecting, 2s cancelling, 2s settling,
// and up to 5s releasing the entire batch of caller-owned transfers.
const WINDOWS_KILL_GRACE_MS = 15_000;
const SESSION_BUSY_RETRY_DELAY_MS = 100;
export function createBskRunner(bskPath: string, spawnImpl: SpawnImpl = spawn): BskRunner {
const live = new Map<ChildProcess, string | undefined>();
const windows = process.platform === "win32";
const cancelling = new Set<ChildProcess>();
function killChild(child: ChildProcess): void {
if (child.exitCode !== null || child.signalCode !== null) return;
child.kill("SIGINT");
const force = setTimeout(() => {
if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL");
}, KILL_GRACE_MS);
if (child.exitCode !== null || child.signalCode !== null || cancelling.has(child)) return;
cancelling.add(child);
// Node kills Windows children outright for SIGINT. EOF asks the CLI to
// send its existing cancel RPC and wait for browser reconciliation.
if (windows && child.stdin) child.stdin.end();
else child.kill("SIGINT");
const force = setTimeout(
() => {
if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL");
},
windows ? WINDOWS_KILL_GRACE_MS : KILL_GRACE_MS,
);
force.unref();
child.once("close", () => {
clearTimeout(force);
cancelling.delete(child);
});
}
return {
run(args, options = {}) {
if (options.signal?.aborted) {
return Promise.resolve({
code: null,
stdout: "",
stderr: "",
timedOut: false,
aborted: true,
});
}
return new Promise<BskRunResult>((resolve, reject) => {
let child: ChildProcess;
try {
child = spawnImpl(bskPath, [...args, "--json"]);
child = spawnImpl(
bskPath,
[...args, "--json"],
windows
? {
windowsHide: true,
env: { ...process.env, BSK_CANCEL_ON_STDIN_CLOSE: "1" },
}
: undefined,
);
// A child exiting while cancellation closes stdin may report EPIPE.
// Its close/error event remains the authority for the run result.
child.stdin?.on("error", () => {});
} catch (error) {
reject(error);
return;

Some files were not shown because too many files have changed in this diff Show More