Compare commits

...
63 Commits
Author SHA1 Message Date
drakezhang eefbe03258 fix(observation): preserve snapshot unit normalization after document extraction 2026-09-09 15:58:28 +08:00
Ljy-0827 1d275f047c perf(observation): reduce document processing scheduling overhead 2026-09-09 15:54:41 +08:00
Ljy-0827 388b3ae2d5 refactor(observation): isolate document processing from capture orchestration 2026-09-09 15:54:41 +08:00
Ljy-0827 36d0d91419 fix(observation): preserve snapshot document provenance and URL metadata 2026-09-09 15:54:41 +08:00
Ljy-0827 bcb9beb4f8 refactor(observation): defer visual facts until visual discovery 2026-09-09 15:54:41 +08:00
Ljy-0827 34ccb96fd6 fix(observation): await capture task cleanup before propagating cancellation 2026-09-09 15:54:41 +08:00
Ljy-0827 8dfa097e55 refactor(observation): organize capture around document facts 2026-09-09 15:54:41 +08:00
drakezhang 383e3128eb fix(geometry): normalize snapshot layout bounds and scroll together 2026-09-09 15:53:35 +08:00
Ljy-0827 2d2636030d perf(geometry): batch iframe owner size measurements 2026-09-09 13:20:21 +08:00
Ljy-0827 a28b9d55e7 fix(vom): surface incomplete iframe geometry in observations 2026-09-09 13:20:21 +08:00
Ljy-0827 c39874591f perf(observation): measure sibling iframe geometry with bounded concurrency 2026-09-09 13:20:21 +08:00
Ljy-0827 1485addfd5 fix(geometry): unify frame projection and screenshot coordinates 2026-09-09 13:20:21 +08:00
Zhang GH 90b0ff301b Merge pull request #204 from Tencent/release/0.2.1
Release/0.2.1
2026-09-09 11:35:05 +08:00
drakezhang e74f354e90 fix(release): support changelogs without unreleased section 2026-09-08 20:38:17 +08:00
drakezhang 05c036518a chore(release): 0.2.1 2026-09-08 20:35:14 +08:00
Zhang GH 020fb3212e Merge pull request #201 from Tencent/fix/windows-install-and-cancellation
fix(windows): harden installation and parent cancellation
2026-09-08 20:15:12 +08:00
Zhang GH 0822c53c65 fix(windows): preserve transfer cleanup after cancellation 2026-09-08 20:04:24 +08:00
Zhang GH 4450fbe28c Merge pull request #196 from Tencent/codex/dsh-plugin-docs
docs(dsh-plugin): clarify installation, configuration, and releases
2026-09-08 19:43:24 +08:00
drakezhang a3bb81bfab docs(dsh-plugin): clarify patch editing and reload behavior 2026-09-08 19:34:25 +08:00
Zhang GH caea98cd52 fix(windows): harden installation and parent cancellation 2026-09-08 19:21:54 +08:00
drakezhang 187b088013 docs(dsh-plugin): restore configuration guidance and simplify releases 2026-09-08 19:03:40 +08:00
drakezhang 5046b5ab44 docs(dsh-plugin): clarify setup and verify npm publishing 2026-09-08 18:44:20 +08:00
Zhang GH e8137b42cc Merge pull request #194 from Tencent/fix/dsh-runtime-peer-dependency
fix(dsh-plugin): remove obsolete client runtime host dependency
2026-09-08 17:34:33 +08:00
Zhang GH cb5dc148d5 fix(dsh-plugin): remove obsolete client runtime host dependency
Keep dsh-client-runtime as a development-only type dependency. Remove its peer metadata, client injection entry, and runtime external allowance so newer hosts do not preflight a package they no longer ship.

Fixes #193
2026-09-08 16:58:20 +08:00
Zhang GH 3c5f838c56 Merge pull request #192 from Tencent/fix/windows-update-reliability
fix(update): make Windows self-updates reliable
2026-09-07 20:37:32 +08:00
iuyo5678 9337bd557b test(update): use blocking connections in Windows release server 2026-09-07 20:27:04 +08:00
iuyo5678 162a31ae92 fix(update): make Windows self-updates reliable 2026-09-07 20:04:15 +08:00
Zhang GH 2f84234a31 Merge pull request #191 from Tencent/fix/windows-ipc-process-liveness
fix(windows): preserve pipe continuity and verify process liveness
2026-09-07 19:06:38 +08:00
iuyo5678 72c10aab75 fix(windows): preserve pipe continuity and verify process liveness 2026-09-07 18:25:32 +08:00
Zhang GH 1eeb55af02 Merge pull request #190 from Tencent/fix/dsh-observation-sync-and-capture
fix(dsh): synchronize observation state and manage thumbnail capture
2026-09-07 17:46:31 +08:00
drakezhang 4992a54864 docs(dsh): clarify observation subscription and snapshot contracts 2026-09-07 17:38:35 +08:00
drakezhang 8b195eb0df test(extension): return tab id from navigation recovery mock 2026-09-07 17:26:01 +08:00
drakezhang eae4db1729 Merge main for navigation recovery test compatibility 2026-09-07 17:25:22 +08:00
drakezhang e961395be9 fix(dsh): synchronize observation state and manage thumbnail capture 2026-09-07 16:24:04 +08:00
Zhang GH a6d1d02acc Merge pull request #179 from Tencent/fix/cdp-extension-frame-recovery
fix: recover navigation from cross-extension CDP restrictions
2026-09-07 16:05:46 +08:00
Zhang GH 5ebe609c40 Merge pull request #189 from Tencent/fix/verify-fill-before-success
fix(extension): verify fill results before reporting success
2026-09-07 15:59:21 +08:00
drakezhang 321494d93f test(fill): cover caret positioning and clarify append behavior 2026-09-07 15:56:13 +08:00
drakezhang f2184e74c5 fix(fill): support background input and automatic append 2026-09-07 15:28:21 +08:00
drakezhang eeb633d68e fix(extension): verify fill results before reporting success 2026-09-07 13:25:54 +08:00
Zhang GH 6d181105f2 Merge pull request #188 from Tencent/fix/match-form-state-by-element
fix(extension): match form state by element identity
2026-09-07 12:57:42 +08:00
drakezhang ebad0ed1b5 fix(extension): match form state by element identity 2026-09-07 12:21:01 +08:00
Zhang GH 498f8fb4a7 Merge pull request #187 from Tencent/fix/release-automation-state-on-tab-return
Fix/release automation state on tab return
2026-09-07 12:12:41 +08:00
drakezhang cef59137e4 fix(extension): handle tab return cleanup edge cases 2026-09-07 11:44:11 +08:00
drakezhang f70978bfcf fix(extension): release automation state when returning borrowed tabs 2026-09-07 11:30:01 +08:00
Zhang GH 1498cd425c Merge pull request #186 from Tencent/fix/closed-borrowed-tabs-block-session-stop
fix(extension): prevent closed borrowed tabs from blocking session stop
2026-09-07 11:19:12 +08:00
drakezhang e831d65db3 fix(extension): prevent closed borrowed tabs from blocking session stop 2026-09-07 11:12:44 +08:00
Zhang GH 47ac947f18 Merge pull request #174 from iAstro/fix/extension-locale-normalization
fix(i18n): match browser locale by language subtag instead of exact code
2026-09-04 17:29:31 +08:00
Zhang GH 69d64c4eb1 Merge pull request #176 from Tencent/fix/windows-update-detach
fix(bsk): fix Windows self-update failing to replace binary
2026-09-04 17:11:31 +08:00
Zhang GH 99c997d6d5 Merge pull request #66 from iAstro/fix/issue-57-user-tab-freedom
fix(extension): keep user-opened tabs free & fix agent-tab leak on session stop (issue #57)
2026-09-04 17:10:15 +08:00
iAstro 93853ebc65 refactor(i18n): drive locale normalisation from shipped resource keys
Per review feedback on #174: the hardcoded if/else in
normalizeLanguageCode required touching the function for every new
language, so a bare `ko` would miss the `ko-KR` resource #161 is about
to add and Korean users would land on English despite the fix.

Replace it with createLanguageNormalizer(resourceKeys), a factory that
derives its mapping from the resource keys i18next is configured with:

  1. exact resource key              -> unchanged
  2. one resource for the language   -> that key (ko -> ko-KR)
  3. several (zh-CN / zh-TW)         -> script subtag, then region
  4. nothing shipped                 -> unchanged, fallbackLng applies

getLanguageDetectionOptions now takes the resource keys and i18n.ts
passes Object.keys(resources). Chinese disambiguation stays as the only
special case - one language, two scripts; every other mapping derives
from the index automatically.

Test expectations for the zh-TW family change from "zh-TW" to "zh-CN":
only zh-CN ships today and the factory maps to shipped keys only, so
Traditional resolves directly to Simplified instead of leaking into the
fallback chain. The rendered UI is identical either way, and the new
extensibility cases lock in the automatic disambiguation once a zh-TW
bundle registers - plus the auto-resolution of a newly registered
language like ko-KR.

28 -> 31 tests; extension suite 832/832; tsc, biome and ext:build clean.
2026-09-04 15:42:40 +08:00
drakezhang 248bb90a2d test(extension): return home tab ids from mocks 2026-09-04 14:37:04 +08:00
drakezhang 6e6c178434 Merge remote-tracking branch 'origin/main' into fix/issue-57-explicit-tab-claims 2026-09-04 14:31:48 +08:00
drakezhang 386ef91c24 fix(extension): preserve existing tab scope semantics 2026-09-04 13:14:26 +08:00
drakezhang 0dc11b30ea fix(extension): claim agent tabs explicitly 2026-09-04 11:17:05 +08:00
iAstro 759eebd4b6 fix(i18n): match browser locale by language subtag instead of exact code
The extension ships en-US and zh-CN resources, but only en-US was ever
selected: every other browser language fell through to fallbackLng
("zh-CN"), so users on en, en-GB, en-CN, en-AU, ja-JP, fr-FR saw Chinese.

Match on the BCP 47 language subtag and normalise it to a resource key:

  en*                                -> en-US
  zh-Hans* / zh-CN / zh-SG / zh      -> zh-CN
  zh-Hant* / zh-TW / zh-HK / zh-MO   -> zh-TW
  anything else                      -> unchanged (fallbackLng default: en-US)

Simplified vs Traditional is decided by the script subtag (Hans/Hant) when
present, then by the region subtag, since Chrome reports zh-CN / zh-TW
rather than script subtags. The zh-TW branch is written now so adding a
Traditional translation later needs no code change.

chrome.i18n.getUILanguage() is now preferred over navigator.language, which
is unreliable on the chrome-extension:// origin the popup runs on.

Adds a locale-resolution regression matrix (28 cases).

Closes #168
2026-09-04 10:03:43 +08:00
drakezhang fd3ce202a1 fix: recover navigation from cross-extension CDP restrictions
Classify Chrome extension-frame access failures and provide an actionable CLI hint. Recover navigate and reload only when their initial CDP setup is denied, using browser navigation events and verifying that CDP access returns.

Preserve normal navigation, Agent Window checks, transfer error details, and network-idle semantics. Never replay an already-dispatched navigation.
2026-09-03 18:21:21 +08:00
drakezhangandCursor 618bd6a0df fix(bsk): fix Windows self-update failing to replace binary
Two issues prevented the detached update helper from completing:

1. Only CREATE_NO_WINDOW was set — the child cmd.exe process was still
   attached to the parent console session and got killed when the
   terminal closed. Add CREATE_NEW_PROCESS_GROUP to fully detach it.

2. `timeout /t 1 /nobreak` requires a console and fails silently in a
   no-window context, causing the retry loop to spin with no delay.
   Replace with `ping -n 2 127.0.0.1` which works in any environment.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-09-03 12:24:53 +08:00
Zhang GH c455a09fc3 Merge pull request #163 from dayi/fix/recording-http-random-uuid
fix(extension): support UUID generation on insecure origins
2026-09-03 10:46:12 +08:00
Dean Wang b6be356e40 test(extension): cover UUID fallback during recording startup 2026-09-03 07:40:57 +08:00
iAstro a77353bce8 fix(extension): classify the home tab by id, not a racy boot flag (issue #57)
SessionManager.classifyNewTab's `initializing` branch relied on a
windowInitializing flag that start() cleared *before* the home tab's
chrome.tabs.onCreated event was dispatched (the event is asynchronous), so
the branch never ran in production and the home tab's classification was
subject to a timing race. Match the home tab by its tab id instead — the id
is already returned by ensureActiveTab and is reliable regardless of event
ordering.

Also refresh the overlay-bridge docstring: overlay.ready now makes the
background *push* the per-tab overlay state (and the Agent Window also
pushes on tab create / control-mode change), rather than a request/reply.
2026-09-01 15:45:36 +08:00
iAstro 91621b2545 fix(extension): release the pending agent-tab slot when tab creation fails
tool.tab_create increments pendingAgentTabCount *before* chrome.tabs.create
so the onCreated listener can tell an agent tab from a user-opened tab. When
the create failed, no onCreated event arrived and the counter leaked, so the
next tab the user opened was misclassified as agent-created — the exact
confusion the counter exists to prevent, a variant of issue #57.

handleTabCreate now releases the slot on failure via
SessionManager.releaseAgentTabPending. An abort (cancelled) is excluded: the
tab did open (and was cleaned up), so its onCreated still consumes the slot.

Added a regression test asserting that after a failed tab_create, a
user-opened tab is still classified as 'user' (not 'agent').
2026-09-01 15:36:44 +08:00
Ljy-0827 a4fed136e3 fix(extension): keep user-opened tabs free & fix agent-tab leak on session stop (issue #57)
- session_stop: exclude tabs still tracked in agentCreatedTabs from the
  release decision so a tab that failed to close is no longer mistaken for
  a user tab (which previously kept the window open and leaked the tab).
- Distinguish user-created tabs from agent-created tabs in the Agent Window
  via a pending-tab counter consumed by chrome.tabs.onCreated, so tabs the
  user opens themselves (new-tab button / Cmd+T / open in new tab) are never
  taken over by the BrowserSkill control mask.
- background overlay: decide per-tab state BEFORE showing controls
  (overlayStateForTab + pushOverlayStateForTab) so a user tab receives hidden
  on its first overlay.ready ping instead of flashing control then hiding.
- AgentWindowApi.ensureActiveTab now resolves with the activated/created tab
  id instead of void, so SessionManager can record homeTabId. This fixes the
  tsc failure flagged in review (the interface previously returned
  Promise<void> while callers treated the result as number | null).
- Wire the production tabManagement.tabs / tabsQuery deps into
  tool.session_stop, both in the dispatcher and in the disconnect cleanup.
  They were never injected, which left the agent-tab cleanup and the
  window-release path as dead code outside tests (behaviour identical to
  main). Added a dispatcher-level integration test that fails without the
  wiring: a surviving user tab must release the window, not close it.
2026-09-01 11:24:56 +08:00
Dean Wang 3c77ef7b01 fix(extension): support UUID generation on insecure origins 2026-09-01 10:30:40 +08:00
120 changed files with 9829 additions and 1516 deletions
+29
View File
@@ -37,6 +37,35 @@ jobs:
- name: Run Rust tests
run: cargo test --workspace --locked
windows-daemon:
name: Windows daemon, installer and cancellation regression tests
runs-on: windows-latest
steps:
- uses: actions/checkout@v6
- uses: dtolnay/rust-toolchain@stable
- name: Run Windows named-pipe tests
run: cargo test -p bsk --lib --locked daemon::ipc::windows::tests
- name: Run Windows process liveness tests
run: cargo test -p bsk --lib --locked daemon::lockfile::tests
- name: Run Windows update tests
run: cargo test -p bsk --lib --locked cli::update::tests
- name: Run Windows update and cancellation process tests
run: cargo test -p bsk --test windows_update --test windows_parent_cancel --locked
- name: Run installer tests (Windows PowerShell 5.1)
shell: powershell
run: ./scripts/install-windows.test.ps1
- name: Run installer tests (PowerShell 7)
shell: pwsh
run: ./scripts/install-windows.test.ps1
frontend:
name: Frontend lint, typecheck, tests, build
runs-on: ubuntu-latest
+18 -1
View File
@@ -5,7 +5,24 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/).
Starting from 0.2.0, CLI / Extension / DSH Plugin share the same version number.
## [Unreleased]
## [0.2.1] - 2026-09-09
### Changed
- DSH Plugin: clarified installation, configuration, profile restart, and upgrade instructions
### Fixed
- CLI: reliable Windows self-updates, including executable replacement and daemon restart
- CLI: Windows named-pipe connection continuity and stale daemon process detection
- Windows installer: PowerShell 5.1 compatibility, path quoting, and staged executable replacement
- CLI / DSH Plugin: cancellation propagation and upload/download cleanup when the parent process exits
- Extension: tab ownership and session cleanup when agent tabs fail to open or borrowed tabs are returned or closed
- Extension: navigation recovery when other extensions restrict Chrome DevTools Protocol access
- Extension: background input filling, automatic append behavior, and verification of field values before reporting success
- Extension: match observed form state to the correct element after page changes
- Extension: UUID generation fallback for recording and overlays on insecure origins
- Browser locale matching for regional language variants
- DSH Plugin: observation state synchronization, thumbnail capture lifecycle, and stale preview cleanup
- DSH Plugin: removed the obsolete client runtime host dependency
## [0.2.0] - 2026-09-02
Generated
+2 -2
View File
@@ -134,7 +134,7 @@ dependencies = [
[[package]]
name = "bsk"
version = "0.2.0"
version = "0.2.1"
dependencies = [
"anyhow",
"base64",
@@ -169,7 +169,7 @@ dependencies = [
[[package]]
name = "bsk-protocol"
version = "0.2.0"
version = "0.2.1"
dependencies = [
"schemars",
"semver",
+2 -2
View File
@@ -7,7 +7,7 @@ members = [
default-members = ["crates/bsk-cli"]
[workspace.package]
version = "0.2.0"
version = "0.2.1"
edition = "2024"
rust-version = "1.85"
license = "MIT"
@@ -43,7 +43,7 @@ rand = "0.8"
uuid = { version = "1", features = ["v4", "serde"] }
# Workspace crates (version required for `cargo publish` of dependents)
bsk-protocol = { version = "0.2.0", path = "crates/bsk-protocol" }
bsk-protocol = { version = "0.2.1", path = "crates/bsk-protocol" }
[profile.release]
strip = "symbols"
+13 -7
View File
@@ -147,20 +147,26 @@ Start a new Agent session and write a prompt that needs the browser, for example
Using [DeepSeek Harness](https://github.com/deepseek-ai/deepseek-harness) (`dsh`)?
BrowserSkill ships a first-class dsh plugin on npm as
[`@wxg-prc-cpg/browser-skill-dsh-plugin`](https://www.npmjs.com/package/@wxg-prc-cpg/browser-skill-dsh-plugin).
It injects native `browser_*` tools (no shelling out to `bsk`) and a live Web UI
overlay of each Agent Window.
It gives the agent native `browser_*` tools and a live view of its browser sessions
in the Web UI. The plugin runs `bsk` on the agent's behalf.
Add it to a dsh profile, then start that profile:
Install the `bsk` CLI and connect the browser extension first. Then add the plugin
to a dsh profile and start it (replace `web` with your profile name):
```sh
dsh plugin --profile web add @wxg-prc-cpg/browser-skill-dsh-plugin
dsh --profile web
```
The plugin carries its own copy of the skill, so `bsk install-skill` is not needed
for dsh — but the `bsk` CLI and the browser extension are still prerequisites. See
the [plugin README](packages/dsh-plugin-browserskill/README.md) for the tool list,
configuration, and the observation overlay.
The plugin includes the `browser-skill` skill, so `bsk install-skill` is not needed
for dsh. Installed plugins do not update automatically. To upgrade this plugin:
```sh
dsh plugin --profile web update @wxg-prc-cpg/browser-skill-dsh-plugin --latest
```
Restart the profile after upgrading. See the
[plugin README](packages/dsh-plugin-browserskill/README.md) for usage and configuration.
## How It Works
+9 -3
View File
@@ -125,16 +125,22 @@ bsk install-skill
## DeepSeek Harness 插件
在用 [DeepSeek Harness](https://github.com/deepseek-ai/deepseek-harness)(`dsh`)?BrowserSkill 提供了官方 dsh 插件,已发布到 npm:[`@wxg-prc-cpg/browser-skill-dsh-plugin`](https://www.npmjs.com/package/@wxg-prc-cpg/browser-skill-dsh-plugin)。它会注入原生 `browser_*` 工具(无需再通过 Shell 调用 `bsk`),并在 Web UI 中实时观察每个 Agent Window。
在用 [DeepSeek Harness](https://github.com/deepseek-ai/deepseek-harness)(`dsh`)?BrowserSkill 提供了官方 dsh 插件,已发布到 npm:[`@wxg-prc-cpg/browser-skill-dsh-plugin`](https://www.npmjs.com/package/@wxg-prc-cpg/browser-skill-dsh-plugin)。它为 Agent 提供原生 `browser_*` 工具,由插件代为调用 `bsk`,并在 Web UI 中实时展示浏览器会话。
把它装进某个 dsh profile,然后启动该 profile:
先安装 `bsk` CLI 并连接浏览器扩展,再将插件装进 dsh profile 并启动(将 `web` 替换为你的 profile 名称):
```sh
dsh plugin --profile web add @wxg-prc-cpg/browser-skill-dsh-plugin
dsh --profile web
```
插件自带 skill,所以在 dsh 下无需执行 `bsk install-skill`;但 `bsk` CLI 和浏览器扩展仍是前置条件。工具清单、配置项与观察浮层见[插件 README](packages/dsh-plugin-browserskill/README.md)。
插件自带 `browser-skill` skill,所以在 dsh 下无需执行 `bsk install-skill`。已安装的插件不会自动更新;升级此插件请运行:
```sh
dsh plugin --profile web update @wxg-prc-cpg/browser-skill-dsh-plugin --latest
```
升级后重启该 profile。用法与配置见[插件 README](packages/dsh-plugin-browserskill/README.md)。
## 工作原理
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@browser-skill/extension",
"version": "0.2.0",
"version": "0.2.1",
"private": true,
"type": "module",
"scripts": {
@@ -254,6 +254,69 @@ describe("ChromiumCdp", () => {
expect(cdp.isAttached(7)).toBe(false);
});
it.each([
false,
true,
])("waits for detach before reconnecting a returned tab (detach rejects: %s)", async (rejectDetach) => {
const { api } = fakeApi();
const attached = new Set<number>();
let finishDetach!: () => void;
vi.mocked(api.attach).mockImplementation(async ({ tabId }) => {
if (attached.has(tabId!)) throw new Error("Another debugger is already attached");
attached.add(tabId!);
});
vi.mocked(api.detach).mockImplementation(async ({ tabId }) => {
attached.delete(tabId!);
});
vi.mocked(api.detach).mockImplementationOnce(
({ tabId }) =>
new Promise<void>((resolve, reject) => {
finishDetach = () => {
attached.delete(tabId!);
if (rejectDetach) reject(new Error("tab already detached"));
else resolve();
};
}),
);
const cdp = new ChromiumCdp(api);
cdp.trackSessionTab("aa11", 7);
await cdp.ensureAttached(7);
const returning = cdp.releaseSessionTab("aa11", 7);
await vi.waitFor(() => expect(api.detach).toHaveBeenCalledOnce());
let duplicateDetachFinished = false;
const duplicateDetach = cdp.detach(7).then(() => {
duplicateDetachFinished = true;
});
cdp.trackSessionTab("bb22", 7);
// Register rejection handlers immediately; the pre-fix driver rejects
// both commands while the browser still has the previous attachment.
const pending = Promise.allSettled([
cdp.send(7, "Runtime.evaluate", { expression: "document.title" }),
cdp.send(7, "DOM.getDocument"),
]);
await cdp.send(8, "DOM.getDocument");
const finishedEarly = duplicateDetachFinished;
finishDetach();
await Promise.all([returning, duplicateDetach]);
expect(await pending).toEqual([
{ status: "fulfilled", value: { ok: true } },
{ status: "fulfilled", value: { ok: true } },
]);
expect(finishedEarly).toBe(false);
expect(vi.mocked(api.attach).mock.calls.filter(([target]) => target.tabId === 7)).toHaveLength(
2,
);
expect(cdp.isAttached(7)).toBe(true);
expect(cdp.isAttached(8)).toBe(true);
await cdp.detachSession("bb22");
expect(api.detach).toHaveBeenCalledTimes(2);
expect(cdp.isAttached(7)).toBe(false);
expect(attached.has(7)).toBe(false);
expect(cdp.isAttached(8)).toBe(true);
});
it("detachAll() iterates every cached tab", async () => {
const { api } = fakeApi();
const cdp = new ChromiumCdp(api);
@@ -651,6 +714,155 @@ describe("ChromiumCdp", () => {
expect(cdp.networkEntriesSince(33, 0, 50, 1000).entries[0].url).toBeUndefined();
});
it("releases only the returning session's claim on a tab", async () => {
const { api } = fakeApi();
const cdp = new ChromiumCdp(api);
await cdp.ensureAttached(1);
await cdp.ensureAttached(2);
cdp.trackSessionTab("aa11", 1);
cdp.trackSessionTab("bb22", 1);
cdp.trackSessionTab("aa11", 2);
await cdp.releaseSessionTab("unknown", 1);
await cdp.releaseSessionTab("aa11", 1);
expect(api.detach).not.toHaveBeenCalled();
await cdp.releaseSessionTab("bb22", 1);
await cdp.releaseSessionTab("bb22", 1);
expect(api.detach).toHaveBeenCalledExactlyOnceWith({ tabId: 1 });
expect(cdp.isAttached(2)).toBe(true);
await cdp.detachSession("aa11");
expect(api.detach).toHaveBeenCalledTimes(2);
expect(api.detach).toHaveBeenLastCalledWith({ tabId: 2 });
});
it.each([
false,
true,
])("releases a tab with an attachment in flight (new owner: %s)", async (newOwner) => {
const { api } = fakeApi();
let finishAttach!: () => void;
vi.mocked(api.attach).mockImplementationOnce(
() =>
new Promise<void>((resolve) => {
finishAttach = resolve;
}),
);
const cdp = new ChromiumCdp(api);
cdp.trackSessionTab("aa11", 1);
const attaching = cdp.ensureAttached(1);
const releasing = cdp.releaseSessionTab("aa11", 1);
if (newOwner) cdp.trackSessionTab("bb22", 1);
finishAttach();
await Promise.all([attaching, releasing]);
expect(cdp.isAttached(1)).toBe(newOwner);
expect(api.detach).toHaveBeenCalledTimes(newOwner ? 0 : 1);
});
it("stops accepting dialogs on a returned tab even when another session keeps CDP attached", async () => {
const { api, onEvent } = fakeApi();
let controlled = true;
const cdp = new ChromiumCdp(api, { shouldAutoAcceptDialog: () => controlled });
cdp.trackSessionTab("aa11", 1);
cdp.trackSessionTab("bb22", 1);
await cdp.ensureAttached(1);
onEvent.fire({ tabId: 1 }, "Page.javascriptDialogOpening", {
type: "confirm",
message: "before return",
});
await vi.waitFor(() => expect(cdp.dialogsSince(1, 0)).toHaveLength(1));
vi.mocked(api.sendCommand).mockClear();
controlled = false;
await cdp.releaseSessionTab("aa11", 1);
expect(cdp.isAttached(1)).toBe(true);
onEvent.fire({ tabId: 1 }, "Page.javascriptDialogOpening", {
type: "confirm",
message: "after return",
});
await Promise.resolve();
expect(api.sendCommand).not.toHaveBeenCalled();
expect(cdp.dialogsSince(1, 0)).toHaveLength(1);
controlled = true;
onEvent.fire({ tabId: 1 }, "Page.javascriptDialogOpening", {
type: "alert",
message: "borrowed again",
});
await vi.waitFor(() => expect(cdp.dialogsSince(1, 0)).toHaveLength(2));
expect(api.sendCommand).toHaveBeenCalledExactlyOnceWith(
{ tabId: 1 },
"Page.handleJavaScriptDialog",
{ accept: true },
);
});
it("ignores dialog events after detach, including a pending eligibility check", async () => {
const { api, onEvent } = fakeApi();
let resolveEligibility!: (value: boolean) => void;
const shouldAutoAcceptDialog = vi.fn(
() =>
new Promise<boolean>((resolve) => {
resolveEligibility = resolve;
}),
);
const cdp = new ChromiumCdp(api, { shouldAutoAcceptDialog });
await cdp.ensureAttached(1);
onEvent.fire({ tabId: 1 }, "Page.javascriptDialogOpening", { type: "alert" });
await cdp.detach(1);
resolveEligibility(true);
await Promise.resolve();
onEvent.fire({ tabId: 1 }, "Page.javascriptDialogOpening", { type: "alert" });
expect(shouldAutoAcceptDialog).toHaveBeenCalledOnce();
expect(api.sendCommand).not.toHaveBeenCalledWith(
expect.anything(),
"Page.handleJavaScriptDialog",
expect.anything(),
);
expect(cdp.dialogsSince(1, 0)).toEqual([]);
});
it("does not accept dialogs when the tab's current scope cannot be determined", async () => {
const { api, onEvent } = fakeApi();
const shouldAutoAcceptDialog = vi.fn(async () => {
throw new Error("tab is gone");
});
const cdp = new ChromiumCdp(api, { shouldAutoAcceptDialog });
await cdp.ensureAttached(1);
vi.mocked(api.sendCommand).mockClear();
onEvent.fire({ tabId: 1 }, "Page.javascriptDialogOpening", { type: "confirm" });
await Promise.resolve();
expect(shouldAutoAcceptDialog).toHaveBeenCalledWith(1);
expect(api.sendCommand).not.toHaveBeenCalled();
expect(cdp.dialogsSince(1, 0)).toEqual([]);
});
it("does not restore dialog state when an acceptance completes after detach", async () => {
const { api, onEvent } = fakeApi();
const cdp = new ChromiumCdp(api);
await cdp.ensureAttached(1);
let finishAccept!: () => void;
vi.mocked(api.sendCommand).mockImplementationOnce(
() =>
new Promise<void>((resolve) => {
finishAccept = resolve;
}),
);
onEvent.fire({ tabId: 1 }, "Page.javascriptDialogOpening", { type: "alert" });
await cdp.detach(1);
finishAccept();
await Promise.resolve();
expect(cdp.dialogsSince(1, 0)).toEqual([]);
expect(cdp.dialogCursor(1)).toBe(0);
});
it("detachSession only detaches tabs no other session owns", async () => {
const { api } = fakeApi();
const cdp = new ChromiumCdp(api);
@@ -165,6 +165,7 @@ export class ChromiumCdp {
private readonly api: CdpDebuggerApi;
private readonly attachedTabs = new Set<number>();
private readonly attachInFlight = new Map<number, Promise<void>>();
private readonly detachInFlight = new Map<number, Promise<void>>();
private readonly tabOwners = new Map<number, Set<string>>();
private readonly dialogBuffers = new Map<number, JavaScriptDialogInfo[]>();
private readonly dialogSequences = new Map<number, number>();
@@ -182,7 +183,13 @@ export class ChromiumCdp {
private networkSubscription: { dispose(): void } | null = null;
private frameTargetSubscription: { dispose(): void } | null = null;
constructor(api: CdpDebuggerApi = chromeDebuggerApi) {
constructor(
api: CdpDebuggerApi = chromeDebuggerApi,
private readonly options: {
/** CDP observation alone does not authorize dismissing user dialogs. */
shouldAutoAcceptDialog?: (tabId: number) => boolean | Promise<boolean>;
} = {},
) {
this.api = api;
this.bindAutoDetach();
this.bindDialogHandler();
@@ -193,6 +200,10 @@ export class ChromiumCdp {
/** Attach to `tabId` if we haven't already in this driver. */
async ensureAttached(tabId: number): Promise<void> {
// Returning a tab clears the cache before Chrome finishes detaching.
// New observers must wait before opening the next connection to that tab.
const detaching = this.detachInFlight.get(tabId);
if (detaching) await detaching;
if (this.attachedTabs.has(tabId)) return;
const existing = this.attachInFlight.get(tabId);
if (existing) {
@@ -425,6 +436,11 @@ export class ChromiumCdp {
/** Detach if attached; never throws. */
async detach(tabId: number): Promise<void> {
const existing = this.detachInFlight.get(tabId);
if (existing) {
await existing;
return;
}
this.attachInFlight.delete(tabId);
if (!this.attachedTabs.has(tabId)) return;
this.attachedTabs.delete(tabId);
@@ -432,13 +448,19 @@ export class ChromiumCdp {
this.clearConsoleState(tabId);
this.clearNetworkState(tabId);
this.clearFrameState(tabId);
try {
await this.api.detach({ tabId });
} catch (err) {
// Tab may already be gone — Chrome auto-detaches on close. Log
// at debug so production builds aren't noisy.
console.debug("[bsk cdp] detach failed (likely tab already closed)", err);
}
const detach = (async () => {
try {
await this.api.detach({ tabId });
} catch (err) {
// Tab may already be gone — Chrome auto-detaches on close. Log
// at debug so production builds aren't noisy.
console.debug("[bsk cdp] detach failed (likely tab already closed)", err);
}
})().finally(() => {
this.detachInFlight.delete(tabId);
});
this.detachInFlight.set(tabId, detach);
await detach;
}
/** True iff `ensureAttached(tabId)` has succeeded since the last detach. */
@@ -453,6 +475,17 @@ export class ChromiumCdp {
this.tabOwners.set(tabId, owners);
}
/** Release one session's claim, preserving attachments still used by another. */
async releaseSessionTab(sessionId: string, tabId: number): Promise<void> {
const owners = this.tabOwners.get(tabId);
if (!owners?.delete(sessionId) || owners.size > 0) return;
this.tabOwners.delete(tabId);
// An observation may still be attaching when the tab is returned. Wait
// for it so detach cannot miss the attachment or remove a new owner's claim.
await this.attachInFlight.get(tabId)?.catch(() => {});
if (!this.tabOwners.has(tabId)) await this.detach(tabId);
}
/** Subscribe to all CDP events. Returned disposable removes the listener. */
onEvent(handler: (source: CdpDebuggee, method: string, params: unknown) => void): {
dispose(): void;
@@ -639,13 +672,23 @@ export class ChromiumCdp {
}
private async onJavaScriptDialogOpening(tabId: number, params: unknown): Promise<void> {
if (!this.attachedTabs.has(tabId) && !this.attachInFlight.has(tabId)) return;
const parsed = parseDialogOpeningParams(params);
try {
if (
this.options.shouldAutoAcceptDialog &&
!(await this.options.shouldAutoAcceptDialog(tabId))
) {
return;
}
// The tab may have been returned while its current scope was checked.
if (!this.attachedTabs.has(tabId) && !this.attachInFlight.has(tabId)) return;
const handleParams: { accept: boolean; promptText?: string } = { accept: true };
if (parsed.type === "prompt") {
handleParams.promptText = parsed.defaultPrompt ?? "";
}
await this.api.sendCommand({ tabId }, "Page.handleJavaScriptDialog", handleParams);
if (!this.attachedTabs.has(tabId) && !this.attachInFlight.has(tabId)) return;
const sequence = (this.dialogSequences.get(tabId) ?? 0) + 1;
this.dialogSequences.set(tabId, sequence);
this.appendDialog(tabId, {
@@ -835,15 +878,9 @@ export class ChromiumCdp {
/** Detach tabs only when no other live session has claimed them. */
async detachSession(sessionId: string): Promise<void> {
const tabsToDetach: number[] = [];
for (const [tabId, owners] of this.tabOwners) {
owners.delete(sessionId);
if (owners.size === 0) {
this.tabOwners.delete(tabId);
tabsToDetach.push(tabId);
}
}
await Promise.all(tabsToDetach.map((tabId) => this.detach(tabId)));
await Promise.all(
Array.from(this.tabOwners.keys(), (tabId) => this.releaseSessionTab(sessionId, tabId)),
);
}
}
@@ -45,10 +45,32 @@ function portHarness() {
describe("RecordFrameAgent", () => {
afterEach(() => {
vi.unstubAllGlobals();
document.documentElement.removeAttribute(RECORD_DOCUMENT_ATTRIBUTE);
document.body.replaceChildren();
});
it("starts when crypto.randomUUID is unavailable", async () => {
const harness = portHarness();
const getRandomValues = vi.fn((bytes: Uint8Array) => {
bytes.fill(0x11);
return bytes;
});
vi.stubGlobal("crypto", { getRandomValues });
vi.stubGlobal("chrome", {
runtime: {
connect: vi.fn(() => harness.port),
sendMessage: vi.fn(),
},
});
const agent = new RecordFrameAgent();
await expect(
agent.start({ type: RECORD_FRAME_START, requestId: "rec-http", startedAtMs: 10 }),
).resolves.toEqual({ ok: true });
expect(getRandomValues).toHaveBeenCalled();
});
it("keeps a failed stop retryable and flushes the final dirty fill", async () => {
const harness = portHarness();
const sendMessage = vi
@@ -1,3 +1,4 @@
import { createRandomUuid } from "@/lib/random-uuid";
import {
isAcceptedRecordStepAck,
RECORD_STEP,
@@ -18,7 +19,7 @@ export class RecordStepDelivery {
constructor(
requestId: string,
send: SendRecordStepMessage = (message) => chrome.runtime.sendMessage(message),
producerId: string = crypto.randomUUID(),
producerId: string = createRandomUuid(),
) {
this.#requestId = requestId;
this.#producerId = producerId;
@@ -1,3 +1,4 @@
import { createRandomUuid } from "@/lib/random-uuid";
import {
markRecordingDocument,
type RecordingDocumentMarker,
@@ -67,7 +68,7 @@ export class RecordFrameAgent {
async #start(
message: RecordFrameStartMessage,
): Promise<{ ok: true } | { ok: false; error: string }> {
const producerId = crypto.randomUUID();
const producerId = createRandomUuid();
const root = await waitForDocumentElement();
if (this.#disposed) return { ok: false, error: "recording document is disposed" };
const marker = markRecordingDocument(producerId, root);
+62 -8
View File
@@ -25,7 +25,7 @@ import { recordFrameCoordinator } from "@/lib/recording/frame-coordinator";
import { attachSessionsLiveFlag } from "@/lib/sessions-live-flag";
import { createDisconnectCleanup } from "@/session-manager/disconnect-cleanup";
import { attachSessionEventHandler } from "@/session-manager/event-handler";
import { SessionManager } from "@/session-manager/manager";
import { isAgentControlledTab, SessionManager } from "@/session-manager/manager";
import {
attachBorrowNotificationButtonHandler,
attachBorrowNotificationClickHandler,
@@ -41,6 +41,8 @@ import {
attachRecordStepListener,
type RecordRuntimeDeps,
} from "@/tools/record";
import { chromeTabsApi } from "@/tools/shared";
import { chromeTabMutationApi } from "@/tools/tabs";
import { detectBrowserMeta } from "@/transport/handshake";
import type { Transport } from "@/transport/transport";
import { WSTransport } from "@/transport/ws-transport";
@@ -49,7 +51,12 @@ export default defineBackground(() => {
const controller = new ConnectionController();
const transport = new WSTransport({ url: __BSK_DAEMON_WS_URL__ });
const sessions = new SessionManager();
const cdp = new ChromiumCdp();
const cdp = new ChromiumCdp(undefined, {
shouldAutoAcceptDialog: async (tabId) => {
const tab = await chrome.tabs.get(tabId);
return sessions.findByWindowId(tab.windowId) !== null;
},
});
const sessionsLive = attachSessionsLiveFlag({ manager: sessions });
let overlayGeneration = 0;
const controlModes = new Map<string, OverlayMode>();
@@ -80,6 +87,24 @@ export default defineBackground(() => {
};
}
/**
* Authoritative overlay state for a specific tab. Agent Window tabs are
* free by default; only tabs explicitly claimed through session startup,
* `tab_create`, or `tab_borrow` receive the control overlay.
*/
function overlayStateForTab(tabId?: number, windowId?: number): OverlayAgentStateMessage {
if (typeof tabId === "number" && typeof windowId === "number") {
const ctx = sessions.findByWindowId(windowId);
if (ctx && isAgentControlledTab(ctx, tabId)) return overlayStateForWindow(windowId);
}
return {
type: OVERLAY_AGENT_STATE,
sessionId: null,
mode: "hidden",
generation: overlayGeneration,
};
}
async function pushOverlayStateToTab(
tabId: number,
state: OverlayAgentStateMessage,
@@ -91,13 +116,18 @@ export default defineBackground(() => {
}
}
async function pushOverlayStateForTab(tabId: number, windowId?: number): Promise<void> {
const state = overlayStateForTab(tabId, windowId);
await pushOverlayStateToTab(tabId, state);
}
async function pushOverlayStateForWindow(windowId: number): Promise<void> {
const state = overlayStateForWindow(windowId);
const tabs = await chrome.tabs.query({ windowId });
await Promise.all(
tabs.map((tab) =>
typeof tab.id === "number" ? pushOverlayStateToTab(tab.id, state) : Promise.resolve(),
),
tabs.map((tab) => {
if (typeof tab.id !== "number") return Promise.resolve();
return pushOverlayStateForTab(tab.id, windowId);
}),
);
}
@@ -136,13 +166,30 @@ export default defineBackground(() => {
if (typeof tab.windowId !== "number") return;
pushOverlayStateForAgentWindow(tab.windowId);
});
// Newly observed tabs are free unless the agent's own creation path has
// already claimed their concrete id. This listener only renders that
// state; it never infers or mutates ownership from event ordering.
chrome.tabs.onCreated.addListener((tab) => {
if (typeof tab.windowId !== "number" || typeof tab.id !== "number") return;
if (!sessions.findByWindowId(tab.windowId)) return;
void pushOverlayStateForTab(tab.id, tab.windowId);
});
chrome.tabs.onRemoved.addListener((tabId, removeInfo) => {
sessions.forgetClosedTab(tabId, { isWindowClosing: removeInfo.isWindowClosing });
});
// Re-sync the storage.session flag on SW startup so a previous SW's
// stale `true` does not keep waking us on every page load until the
// first mutation (review M4/M5 round 3 m-R3-1).
void sessionsLive.refresh();
const cleanupAfterDisconnect = createDisconnectCleanup({
manager: sessions,
sessionStopDeps: { cdp },
// Same contract as the dispatcher's `tool.session_stop`: without these
// deps the agent-tab cleanup and the window-release path are dead code.
sessionStopDeps: {
cdp,
tabManagement: { tabs: chromeTabMutationApi },
tabsQuery: chromeTabsApi,
},
onSessionsChanged: () => {
void sessionsLive.syncFromManager();
},
@@ -176,6 +223,9 @@ export default defineBackground(() => {
recording: recordDeps,
onSessionsChanged: onOverlaySessionStateChanged,
onBrowserControlResumed,
onAgentTabClaimed: (tabId, windowId) => {
void pushOverlayStateForTab(tabId, windowId);
},
approveBorrow: (ctx) =>
requestBorrowConfirmation(ctx.tabId, {
...(ctx.signal !== undefined ? { signal: ctx.signal } : {}),
@@ -309,7 +359,11 @@ export default defineBackground(() => {
}
if (msg.kind === OVERLAY_MSG_READY) {
sendResponse(overlayStateForWindow(sender.tab?.windowId));
// Resolve from explicit tab ownership before sending so free tabs never
// flash the control mask.
if (typeof sender.tab?.id === "number") {
void pushOverlayStateForTab(sender.tab.id, sender.tab.windowId);
}
return false;
}
@@ -0,0 +1,30 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import { createRandomUuid } from "../random-uuid";
describe("createRandomUuid", () => {
afterEach(() => {
vi.unstubAllGlobals();
});
it("uses crypto.randomUUID when it is available", () => {
const randomUUID = vi.fn(() => "00000000-0000-4000-8000-000000000000");
vi.stubGlobal("crypto", { randomUUID, getRandomValues: vi.fn() });
expect(createRandomUuid()).toBe("00000000-0000-4000-8000-000000000000");
expect(randomUUID).toHaveBeenCalledOnce();
});
it("falls back to getRandomValues and sets UUID v4 bits", () => {
const bytes = new Uint8Array(16).fill(0xff);
const getRandomValues = vi.fn((target: Uint8Array) => {
target.set(bytes);
return target;
});
vi.stubGlobal("crypto", { getRandomValues });
const uuid = createRandomUuid();
expect(uuid).toBe("ffffffff-ffff-4fff-bfff-ffffffffffff");
expect(getRandomValues).toHaveBeenCalledOnce();
});
});
+4 -2
View File
@@ -3,8 +3,10 @@
* between the content-script control overlay and the background SW.
*
* Content script → background:
* - `{ kind: "overlay.ready" }` → background replies with the authoritative
* overlay state for the sender's window.
* - `{ kind: "overlay.ready" }` → background decides the per-tab state first
* (see overlayStateForTab) and pushes the authoritative overlay state for
* that tab. It also proactively pushes on tab create and control-mode
* change, so a tab usually receives state before it ever sends ready.
* - `{ kind: "overlay.interrupt", sessionId }` → background asks the
* daemon (via a `session.user_interrupt` WS event) to cancel
* every inflight + queued tool call for that session with
+26
View File
@@ -0,0 +1,26 @@
/**
* Generate a UUID v4 in browser contexts where randomUUID() is unavailable.
*
* Crypto.randomUUID() is restricted to secure contexts, while
* crypto.getRandomValues() is available to content scripts on ordinary HTTP
* origins. BrowserSkill records many intranet pages served over HTTP, so the
* fallback is required for recording to work on those pages.
*/
export function createRandomUuid(): string {
const cryptoApi = globalThis.crypto;
if (typeof cryptoApi.randomUUID === "function") return cryptoApi.randomUUID();
const bytes = new Uint8Array(16);
cryptoApi.getRandomValues(bytes);
bytes[6] = (bytes[6]! & 0x0f) | 0x40;
bytes[8] = (bytes[8]! & 0x3f) | 0x80;
const hex = Array.from(bytes, (byte) => byte.toString(16).padStart(2, "0")).join("");
return [
hex.slice(0, 8),
hex.slice(8, 12),
hex.slice(12, 16),
hex.slice(16, 20),
hex.slice(20),
].join("-");
}
@@ -1,3 +1,4 @@
import { createRandomUuid } from "../random-uuid";
import {
isRecordFrameQueryMessage,
RECORD_FRAME_PORT,
@@ -188,7 +189,7 @@ export class RecordFrameCoordinator {
const recording = this.#recordings.get(requestId);
if (!recording) return true;
recording.finishing = true;
const commandId = crypto.randomUUID();
const commandId = createRandomUuid();
const results = await Promise.all(
[...recording.agents.values()].map((agent) => this.#stopAgent(agent, requestId, commandId)),
);
@@ -9,7 +9,7 @@ describe("disconnect session cleanup", () => {
const manager = new SessionManager({
agentWindow: {
create: vi.fn(async () => nextWindowId++),
ensureActiveTab: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
remove,
},
});
@@ -41,7 +41,7 @@ describe("disconnect session cleanup", () => {
const manager = new SessionManager({
agentWindow: {
create: vi.fn(async () => 100),
ensureActiveTab: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
remove: vi.fn(() => removeGate),
},
});
@@ -39,7 +39,7 @@ describe("attachSessionEventHandler", () => {
agentWindow: {
create: vi.fn(async () => 4242),
remove: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
},
});
await manager.start("aa11");
@@ -73,7 +73,7 @@ describe("attachSessionEventHandler", () => {
agentWindow: {
create: vi.fn(async () => 4242),
remove: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
},
});
const ctx = await manager.start("aa11");
@@ -87,6 +87,7 @@ describe("attachSessionEventHandler", () => {
windowEvents: events.api,
});
manager.forgetClosedTab(7, { isWindowClosing: true });
events.emit(4242);
for (let i = 0; i < 4; i += 1) await Promise.resolve();
@@ -113,7 +114,7 @@ describe("attachSessionEventHandler", () => {
agentWindow: {
create: vi.fn(async () => 1),
remove: vi.fn(),
ensureActiveTab: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
},
});
const transport = fakeTransport();
@@ -129,7 +130,7 @@ describe("attachSessionEventHandler", () => {
agentWindow: {
create: vi.fn(async () => 1),
remove: vi.fn(),
ensureActiveTab: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
},
});
const transport = fakeTransport();
@@ -1,6 +1,6 @@
import { describe, expect, it, vi } from "vitest";
import type { AgentWindowApi, AgentWindowCreateOptions } from "../agent-window";
import { SessionManager } from "../manager";
import { isAgentControlledTab, SessionManager } from "../manager";
function fakeAgentWindow(): AgentWindowApi & {
createMock: ReturnType<typeof vi.fn>;
@@ -13,7 +13,7 @@ function fakeAgentWindow(): AgentWindowApi & {
return id;
});
const removeMock = vi.fn(async (_id: number) => {});
const ensureActiveTabMock = vi.fn(async (_windowId: number, _url: string) => {});
const ensureActiveTabMock = vi.fn(async (_windowId: number, _url: string) => 0);
return {
create: createMock,
remove: removeMock,
@@ -151,6 +151,76 @@ describe("SessionManager", () => {
expect(sm.list()).toEqual([]);
});
describe("explicit tab claims", () => {
it("claims the exact home tab returned by AgentWindowApi", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow() });
const ctx = await sm.start("aa11");
expect(ctx.agentCreatedTabs).toEqual(new Set([0]));
expect(isAgentControlledTab(ctx, 0)).toBe(true);
});
it("keeps every other tab free without classification state", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow() });
const ctx = await sm.start("aa11");
expect(isAgentControlledTab(ctx, 99)).toBe(false);
expect(ctx).not.toHaveProperty("userTabs");
expect(ctx).not.toHaveProperty("pendingAgentTabCount");
});
it("treats a committed borrow as an explicit claim", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow() });
const ctx = await sm.start("aa11");
const reservation = sm.tryReserveBorrow(42, ctx.sessionId);
if ("borrowedBy" in reservation) throw new Error("unexpected borrow conflict");
reservation.commit({ tabId: 42, originalWindowId: 7, originalIndex: 3 });
expect(isAgentControlledTab(ctx, 42)).toBe(true);
});
it("forgets an agent-created tab after Chrome removes it", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow() });
const ctx = await sm.start("aa11");
ctx.agentCreatedTabs.add(42);
sm.forgetClosedTab(42);
expect(ctx.agentCreatedTabs.has(42)).toBe(false);
expect(isAgentControlledTab(ctx, 0)).toBe(true);
});
it("forgets closed borrows idempotently without affecting other tabs or sessions", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow() });
const a = await sm.start("aa11");
const b = await sm.start("bb22");
a.borrowedTabs.set(42, { tabId: 42, originalWindowId: 7, originalIndex: 0 });
a.borrowedTabs.set(43, { tabId: 43, originalWindowId: 7, originalIndex: 1 });
b.borrowedTabs.set(44, { tabId: 44, originalWindowId: 8, originalIndex: 0 });
sm.forgetClosedTab(42);
sm.forgetClosedTab(42);
sm.forgetClosedTab(999);
expect(isAgentControlledTab(a, 42)).toBe(false);
expect(sm.findBorrowingSession(42, null)).toBeNull();
expect([...a.borrowedTabs.keys()]).toEqual([43]);
expect([...b.borrowedTabs.keys()]).toEqual([44]);
expect(sm.list()).toHaveLength(2);
});
it("prevents an in-flight borrow from reclaiming a closed tab", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow() });
const ctx = await sm.start("aa11");
const reservation = sm.tryReserveBorrow(42, ctx.sessionId);
if ("borrowedBy" in reservation) throw new Error("unexpected borrow conflict");
sm.forgetClosedTab(42);
expect(() =>
reservation.commit({ tabId: 42, originalWindowId: 7, originalIndex: 0 }),
).toThrow(/reservation disappeared/);
reservation.release();
expect(sm.findBorrowingSession(42, null)).toBeNull();
expect(ctx.borrowedTabs.has(42)).toBe(false);
});
});
describe("findBorrowingSession", () => {
it("returns null when no session has borrowed the tab", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow() });
@@ -14,8 +14,11 @@ export interface AgentWindowApi {
* Guarantee the Agent Window has an active, CDP-navigable tab.
* `chrome://` pages (including the New Tab page) reject `Page.navigate`,
* so sessions bootstrap with `about:blank` instead.
*
* Resolves with the id of the activated (or newly created) tab, so callers
* can track the session's home tab without re-querying Chrome.
*/
ensureActiveTab(windowId: number, url: string): Promise<void>;
ensureActiveTab(windowId: number, url: string): Promise<number>;
}
/** Creation hints for a new Agent Window. */
@@ -49,15 +52,19 @@ export const chromeAgentWindowApi: AgentWindowApi = {
// cancellation success while the Agent Window remains open.
await chrome.windows.remove(windowId);
},
async ensureActiveTab(windowId: number, url: string): Promise<void> {
async ensureActiveTab(windowId: number, url: string): Promise<number> {
const tabs = await chrome.tabs.query({ windowId });
const first = tabs.find((t) => typeof t.id === "number");
if (first?.id) {
if (first?.id !== undefined) {
if (!first.active) {
await chrome.tabs.update(first.id, { active: true });
}
return;
return first.id;
}
await chrome.tabs.create({ windowId, url, active: true });
const created = await chrome.tabs.create({ windowId, url, active: true });
if (typeof created?.id !== "number") {
throw new Error("[bh] chrome.tabs.create returned no tab id");
}
return created.id;
},
};
+29 -1
View File
@@ -6,9 +6,20 @@ export interface SessionContext {
agentWindowId: number;
refStore: RefStore;
borrowedTabs: Map<number, BorrowedTab>;
/**
* Tabs explicitly claimed by the agent because it created them. This
* includes the Agent Window's home tab and tabs created by `tool.tab_create`.
* Tabs opened through Chrome UI never enter this set.
*/
agentCreatedTabs: Set<number>;
createdAtMs: number;
}
/** Whether this session has explicitly claimed control of `tabId`. */
export function isAgentControlledTab(ctx: SessionContext, tabId: number): boolean {
return ctx.agentCreatedTabs.has(tabId) || ctx.borrowedTabs.has(tabId);
}
export interface BorrowedTab {
tabId: number;
originalWindowId: number;
@@ -106,6 +117,19 @@ export class SessionManager {
return Array.from(this.sessions.values());
}
/**
* Forget a tab Chrome has removed, including any uncommitted borrow.
* Whole-window closures keep committed borrows until the window-removed
* handler reports which user tabs could not be returned.
*/
forgetClosedTab(tabId: number, { isWindowClosing = false } = {}): void {
this.borrowReservations.delete(tabId);
for (const ctx of this.sessions.values()) {
ctx.agentCreatedTabs.delete(tabId);
if (!isWindowClosing) ctx.borrowedTabs.delete(tabId);
}
}
/**
* Look up whether `tabId` is currently borrowed by some *other*
* session than the one calling. Used by M8 `tab_borrow` to refuse
@@ -179,7 +203,7 @@ export class SessionManager {
const { signal: _signal, ...createOptions } = opts;
windowId = await this.agentWindow.create(AGENT_WINDOW_HOME, createOptions);
throwIfSessionStartAborted(opts.signal);
await this.agentWindow.ensureActiveTab(windowId, AGENT_WINDOW_HOME);
const homeTabId = await this.agentWindow.ensureActiveTab(windowId, AGENT_WINDOW_HOME);
throwIfSessionStartAborted(opts.signal);
const ctx: SessionContext = {
@@ -187,6 +211,10 @@ export class SessionManager {
agentWindowId: windowId,
refStore: new RefStore(),
borrowedTabs: new Map(),
// The home tab is the session's first explicit claim. Every other
// tab remains free until `tab_create` or `tab_borrow` identifies it
// by its concrete Chrome tab id.
agentCreatedTabs: new Set([homeTabId]),
createdAtMs: this.now(),
};
this.sessions.set(sessionId, ctx);
@@ -13,7 +13,7 @@ function fakeAgentWindow(ids: number[]) {
return id;
}),
remove: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
};
}
@@ -49,13 +49,54 @@ describe("ToolDispatcher", () => {
vi.unstubAllGlobals();
});
it.each([
"tool.snapshot",
"tool.evaluate",
"tool.get_html",
"tool.console",
])("annotates Chrome extension-access failures from %s", async (method) => {
const tab = { id: 7, windowId: 4242, active: true, url: "https://example.test" };
vi.stubGlobal("chrome", {
tabs: {
get: vi.fn(async () => tab),
query: vi.fn(async () => [tab]),
},
});
const { transport, sent, deliver } = fakeTransport();
const sessions = new SessionManager({
agentWindow: {
create: vi.fn(async () => 4242),
remove: vi.fn(),
ensureActiveTab: vi.fn(),
},
});
await sessions.start("test");
const error = new Error("Cannot access a chrome-extension:// URL of different extension");
const cdp = {
send: vi.fn().mockRejectedValue(error),
ensureConsoleCapture: vi.fn().mockRejectedValue(error),
consoleEntriesSince: vi.fn(),
} as unknown as TestDispatcherCdp;
const dispatcher = new ToolDispatcher({ transport, sessions, cdp });
dispatcher.start();
deliver(makeRequest(method, { session_id: "test", expression: "document.title" }));
await vi.waitFor(() => expect(sent).toHaveLength(1));
expect(sent[0]).toMatchObject({
error: {
code: "cdp_failed",
data: { reason: "cdp_extension_access_denied" },
},
});
dispatcher.stop();
});
it("uses the configured recording runtime for start and stop", async () => {
const { transport, sent, deliver } = fakeTransport();
const sessions = new SessionManager({
agentWindow: {
create: vi.fn(async () => 4242),
remove: vi.fn(),
ensureActiveTab: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
},
});
await sessions.start("aa11");
@@ -117,7 +158,7 @@ describe("ToolDispatcher", () => {
agentWindow: {
create: vi.fn(async () => 4242),
remove: vi.fn(),
ensureActiveTab: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
},
});
const dispatcher = new ToolDispatcher({ transport, sessions });
@@ -140,7 +181,7 @@ describe("ToolDispatcher", () => {
agentWindow: {
create,
remove: vi.fn(),
ensureActiveTab: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
},
});
const dispatcher = new ToolDispatcher({ transport, sessions });
@@ -158,7 +199,7 @@ describe("ToolDispatcher", () => {
agentWindow: {
create: vi.fn(async () => 4242),
remove: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
},
});
await sessions.start("aa11");
@@ -172,6 +213,47 @@ describe("ToolDispatcher", () => {
expect(sent[0]).toEqual({ id: "r-1", result: {} });
});
it("wires the production tab APIs into tool.session_stop so a surviving user tab releases the window", async () => {
// Regression guard for the deps that session_stop reads directly: when
// `tabManagement.tabs` / `tabsQuery` are not injected by the dispatcher,
// the agent-tab cleanup and the release decision silently no-op and the
// Agent Window is closed even though a user tab is still open (issue #57).
const removeTab = vi.fn(async () => {});
const closeWindow = vi.fn(async () => {});
// After cleanup the window still holds one tab (id 99) that is neither
// the home tab nor an agent-created tab — i.e. a genuine user tab.
vi.stubGlobal("chrome", {
tabs: {
remove: removeTab,
query: vi.fn(async () => [{ id: 99, windowId: 4242, active: true }]),
},
});
const { transport, sent, deliver } = fakeTransport();
const sessions = new SessionManager({
agentWindow: {
create: vi.fn(async () => 4242),
remove: closeWindow,
ensureActiveTab: vi.fn(async () => 1),
},
});
await sessions.start("aa11");
// Register the exact tab id returned by the agent's creation path.
sessions.get("aa11")?.agentCreatedTabs.add(7);
const dispatcher = new ToolDispatcher({ transport, sessions });
dispatcher.start();
deliver(makeRequest("tool.session_stop", { session_id: "aa11" }));
await flushMicrotasks();
// The agent tab is closed through the real chrome.tabs surface...
expect(removeTab).toHaveBeenCalledWith(7);
// ...and the window is released rather than closed, so the user's tab survives.
expect(closeWindow).not.toHaveBeenCalled();
expect(sessions.has("aa11")).toBe(false);
expect(sent[0]).toEqual({ id: "r-1", result: { window_released: true } });
});
it("routes tool.console through the CDP console buffer", async () => {
vi.stubGlobal("chrome", {
tabs: {
@@ -183,7 +265,7 @@ describe("ToolDispatcher", () => {
agentWindow: {
create: vi.fn(async () => 4242),
remove: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
},
});
await sessions.start("aa11");
@@ -246,7 +328,7 @@ describe("ToolDispatcher", () => {
agentWindow: {
create: vi.fn(async () => 4242),
remove: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
},
});
const ctx = await sessions.start("aa11");
@@ -339,7 +421,7 @@ describe("ToolDispatcher", () => {
agentWindow: {
create: vi.fn(async () => 4242),
remove: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
},
});
await sessions.start("aa11");
@@ -375,7 +457,7 @@ describe("ToolDispatcher", () => {
agentWindow: {
create: vi.fn(async () => 4242),
remove,
ensureActiveTab: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
},
});
await sessions.start("aa11");
@@ -423,7 +505,7 @@ describe("ToolDispatcher", () => {
agentWindow: {
create: vi.fn(async () => 1),
remove: vi.fn(),
ensureActiveTab: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
},
});
const dispatcher = new ToolDispatcher({ transport, sessions });
@@ -443,7 +525,7 @@ describe("ToolDispatcher", () => {
agentWindow: {
create: vi.fn(async () => 1),
remove: vi.fn(),
ensureActiveTab: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
},
});
const dispatcher = new ToolDispatcher({ transport, sessions });
@@ -467,7 +549,7 @@ describe("ToolDispatcher", () => {
agentWindow: {
create: vi.fn(async () => 1),
remove: vi.fn(),
ensureActiveTab: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
},
});
const dispatcher = new ToolDispatcher({ transport, sessions });
@@ -484,7 +566,7 @@ describe("ToolDispatcher", () => {
agentWindow: {
create: vi.fn(async () => 1),
remove: vi.fn(),
ensureActiveTab: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
},
});
const dispatcher = new ToolDispatcher({ transport, sessions });
@@ -501,7 +583,7 @@ describe("ToolDispatcher", () => {
agentWindow: {
create: vi.fn(async () => 1),
remove: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
},
});
const onSessionsChanged = vi.fn();
@@ -518,16 +600,32 @@ describe("ToolDispatcher", () => {
});
it("invokes onBrowserControlResumed for browser-control tools but not passive reads", async () => {
vi.stubGlobal("chrome", {
tabs: {
create: vi.fn(async (props: chrome.tabs.CreateProperties) => ({
id: 7,
windowId: props.windowId,
url: props.url,
active: props.active,
})),
},
});
const { transport, sent, deliver } = fakeTransport();
const sessions = new SessionManager({
agentWindow: {
create: vi.fn(async () => 1),
remove: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
},
});
const onBrowserControlResumed = vi.fn();
const dispatcher = new ToolDispatcher({ transport, sessions, onBrowserControlResumed });
const onAgentTabClaimed = vi.fn();
const dispatcher = new ToolDispatcher({
transport,
sessions,
onBrowserControlResumed,
onAgentTabClaimed,
});
dispatcher.start();
deliver(makeRequest("tool.session_start", { session_id: "aa11" }));
@@ -544,6 +642,7 @@ describe("ToolDispatcher", () => {
});
await flushMicrotasks();
expect(onBrowserControlResumed).toHaveBeenCalledWith("aa11");
expect(onAgentTabClaimed).toHaveBeenCalledWith(7, 1);
});
it("reasserts remembered hover before follow-up work and releases only after actions", async () => {
@@ -557,7 +656,7 @@ describe("ToolDispatcher", () => {
agentWindow: {
create: vi.fn(async () => 1),
remove: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
},
});
const order: string[] = [];
@@ -639,7 +738,7 @@ describe("ToolDispatcher", () => {
agentWindow: {
create: vi.fn(async () => 1),
remove: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
},
});
const cdp = {
@@ -689,6 +788,135 @@ describe("ToolDispatcher", () => {
);
});
it.each([
{ tab_id: 7, code: "not_found" },
{ tab_id: undefined, code: "invalid_params" },
])("keeps hover when tab_return is rejected with $code", async ({ tab_id, code }) => {
const sendMessage = vi.fn(async () => undefined);
vi.stubGlobal("chrome", {
tabs: {
sendMessage,
get: vi.fn(async () => ({ id: 7, windowId: 100, active: true })),
query: vi.fn(async () => [{ id: 7, windowId: 100, active: true }]),
},
});
const { transport, sent, deliver } = fakeTransport();
const sessions = new SessionManager({
agentWindow: {
create: vi.fn(async () => 100),
remove: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
},
});
await sessions.start("aa11");
const cdp = {
send: vi.fn(async <T>() => ({}) as T),
detachSession: vi.fn(async () => {}),
releaseSessionTab: vi.fn(async () => {}),
};
const dispatcher = new ToolDispatcher({
transport,
sessions,
cdp: cdp as unknown as TestDispatcherCdp,
});
const helpers = dispatcher as unknown as {
rememberHover: (sessionId: string, result: unknown) => unknown;
setHoverBypass: (sessionId: string, tabId: number, enabled: boolean) => Promise<void>;
withHoverReassert: <T>(
params: { session_id: string; tab_id?: number },
work: () => Promise<T>,
) => Promise<T>;
};
helpers.rememberHover("aa11", { tab_id: 7, x: 10, y: 20 });
await helpers.setHoverBypass("aa11", 7, true);
dispatcher.start();
deliver(makeRequest("tool.tab_return", { session_id: "aa11", tab_id }));
await vi.waitFor(() => expect(sent).toHaveLength(1));
expect(sent[0]).toMatchObject({ error: { code } });
expect(cdp.releaseSessionTab).not.toHaveBeenCalled();
expect(sendMessage).not.toHaveBeenCalledWith(7, expect.objectContaining({ enabled: false }));
await helpers.withHoverReassert({ session_id: "aa11", tab_id: 7 }, async () => ({}));
expect(cdp.send).toHaveBeenCalledExactlyOnceWith(7, "Input.dispatchMouseEvent", {
type: "mouseMoved",
x: 10,
y: 20,
});
dispatcher.stop();
});
it("releases the returned tab's hover before moving it and leaves other tabs alone", async () => {
const sendMessage = vi.fn(async () => undefined);
const move = vi.fn(async () => {
expect(sendMessage).toHaveBeenCalledWith(7, expect.objectContaining({ enabled: false }));
return { id: 7, windowId: 200, index: 4 };
});
vi.stubGlobal("chrome", {
tabs: { sendMessage, move },
windows: { get: vi.fn(async () => ({ id: 200 })) },
});
const { transport, sent, deliver } = fakeTransport();
const sessions = new SessionManager({
agentWindow: {
create: vi.fn(async () => 100),
remove: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
},
});
const ctx = await sessions.start("aa11");
ctx.borrowedTabs.set(7, { tabId: 7, originalWindowId: 200, originalIndex: 4 });
const cdp = {
send: vi.fn(async <T>() => ({}) as T),
detachSession: vi.fn(async () => {}),
releaseSessionTab: vi.fn(async () => {}),
};
const dispatcher = new ToolDispatcher({
transport,
sessions,
cdp: cdp as unknown as TestDispatcherCdp,
});
const helpers = dispatcher as unknown as {
rememberHover: (sessionId: string, result: unknown) => unknown;
setHoverBypass: (sessionId: string, tabId: number, enabled: boolean) => Promise<void>;
withHoverReassert: <T>(
params: { session_id: string; tab_id?: number },
work: () => Promise<T>,
) => Promise<T>;
};
for (const [sessionId, tabId] of [
["aa11", 7],
["aa11", 8],
["bb22", 9],
] as const) {
helpers.rememberHover(sessionId, { tab_id: tabId, x: 10, y: 20 });
await helpers.setHoverBypass(sessionId, tabId, true);
}
dispatcher.start();
deliver(makeRequest("tool.tab_return", { session_id: "aa11", tab_id: 7 }));
await vi.waitFor(() => expect(sent).toHaveLength(1));
expect(sent[0]).toMatchObject({ result: { tab_id: 7, returned_to_window_id: 200 } });
expect(cdp.releaseSessionTab).toHaveBeenCalledExactlyOnceWith("aa11", 7);
await helpers.withHoverReassert({ session_id: "aa11", tab_id: 7 }, async () => ({}));
expect(cdp.send).not.toHaveBeenCalled();
for (const [sessionId, tabId] of [
["aa11", 8],
["bb22", 9],
] as const) {
expect(sendMessage).not.toHaveBeenCalledWith(
tabId,
expect.objectContaining({ enabled: false }),
);
await helpers.withHoverReassert({ session_id: sessionId, tab_id: tabId }, async () => ({}));
expect(cdp.send).toHaveBeenLastCalledWith(tabId, "Input.dispatchMouseEvent", {
type: "mouseMoved",
x: 10,
y: 20,
});
}
dispatcher.stop();
});
it("transfers hover overlay bypass ownership between sessions", async () => {
vi.stubGlobal("chrome", {
tabs: {
@@ -700,7 +928,7 @@ describe("ToolDispatcher", () => {
agentWindow: {
create: vi.fn(async () => 1),
remove: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
},
});
const dispatcher = new ToolDispatcher({ transport, sessions });
@@ -737,7 +965,7 @@ describe("ToolDispatcher", () => {
agentWindow: {
create: vi.fn(async () => 4242),
remove: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
},
});
await sessions.start("aa11");
@@ -789,7 +1017,7 @@ describe("ToolDispatcher", () => {
agentWindow: {
create: vi.fn(async () => 1),
remove: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
},
});
const dispatcher = new ToolDispatcher({ transport, sessions });
@@ -823,7 +1051,7 @@ describe("ToolDispatcher", () => {
agentWindow: {
create: vi.fn(async () => 4242),
remove: vi.fn(),
ensureActiveTab: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
},
});
transport.send = () => {
@@ -845,7 +1073,7 @@ describe("ToolDispatcher", () => {
agentWindow: {
create: vi.fn(async () => 5555),
remove,
ensureActiveTab: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
},
});
transport.send = () => {
@@ -873,7 +1101,7 @@ describe("ToolDispatcher", () => {
agentWindow: {
create: vi.fn(() => createPromise),
remove: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
},
});
const dispatcher = new ToolDispatcher({ transport, sessions });
@@ -924,7 +1152,7 @@ describe("ToolDispatcher", () => {
agentWindow: {
create: vi.fn(async () => 1),
remove: vi.fn(),
ensureActiveTab: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
},
});
const dispatcher = new ToolDispatcher({ transport, sessions });
@@ -945,7 +1173,7 @@ describe("ToolDispatcher", () => {
agentWindow: {
create: vi.fn(() => createPromise),
remove: vi.fn(),
ensureActiveTab: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
},
});
const dispatcher = new ToolDispatcher({ transport, sessions });
@@ -20,7 +20,7 @@ function fakeAgentWindow(ids: number[]) {
return id;
}),
remove: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
};
}
@@ -0,0 +1,39 @@
import { describe, expect, it } from "vitest";
import type { RpcError } from "@/transport/types";
import { cdpError, classifyCdpError } from "../errors";
const denied = "Cannot access a chrome-extension:// URL of different extension";
describe("CDP extension access errors", () => {
it("classifies both Chrome Error instances and string failures", () => {
for (const error of [new Error(denied), denied]) {
expect(cdpError(error)).toEqual({
code: "cdp_failed",
message: denied,
data: { reason: "cdp_extension_access_denied" },
});
}
});
it("preserves transfer outcome and cleanup instructions when CDP was denied", () => {
const error: RpcError = {
code: "cdp_failed",
message: denied,
data: {
reason: "transfer_outcome_unknown",
effect_state: "unknown",
cleanup_state: "failed",
},
};
expect(classifyCdpError(error)).toBe(error);
});
it("leaves other error codes and unrelated debugger failures unchanged", () => {
for (const error of [
{ code: "permission_denied", message: denied },
{ code: "cdp_failed", message: "Another debugger is already attached" },
] satisfies RpcError[]) {
expect(classifyCdpError(error)).toBe(error);
}
});
});
@@ -12,7 +12,7 @@ function fakeAgentWindow(ids: number[]) {
return id;
}),
remove: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
};
}
@@ -13,7 +13,7 @@ function sessions() {
agentWindow: {
create: vi.fn(async () => 100),
remove: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
},
});
}
@@ -0,0 +1,573 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import { SessionManager } from "@/session-manager/manager";
import type { CdpRunner } from "@/tools/shared";
import { handleFill } from "../interaction";
interface ScriptParams {
functionDeclaration: string;
arguments?: Array<{ value: unknown }>;
}
async function setup(markup = '<input value="old">') {
document.body.innerHTML = markup;
const element = document.body.firstElementChild as HTMLInputElement;
const manager = new SessionManager({
agentWindow: {
create: async () => 100,
remove: async () => {},
ensureActiveTab: async () => 4,
},
});
const ctx = await manager.start("aa11");
ctx.refStore.set("e1", 12, { tabId: 4 });
const script = vi.fn(async (params: ScriptParams): Promise<unknown> => {
try {
const fn = new Function(`return (${params.functionDeclaration})`)();
return {
result: {
value: await fn.apply(
element,
(params.arguments ?? []).map((arg) => arg.value),
),
},
};
} catch {
return { exceptionDetails: { text: "test page script failed" } };
}
});
// Number/email controls have a browser caret even though their selection
// getters return null. Model that caret instead of assuming it is at the end.
const hiddenCaret = { start: 0, end: 0 };
const moveToEnd = vi.fn(async () => {
if (typeof element.selectionStart === "number") {
element.setSelectionRange(element.value.length, element.value.length);
} else {
hiddenCaret.start = hiddenCaret.end = element.value.length;
}
});
const insert = vi.fn(async (text: string) => {
const focused = document.activeElement as HTMLInputElement;
if (focused.readOnly || focused.disabled) return;
const start = focused.selectionStart ?? hiddenCaret.start;
const end = focused.selectionEnd ?? hiddenCaret.end;
const value = focused.value.slice(0, start) + text + focused.value.slice(end);
focused.value = focused.maxLength >= 0 ? value.slice(0, focused.maxLength) : value;
hiddenCaret.start = hiddenCaret.end = Math.min(start + text.length, focused.value.length);
focused.dispatchEvent(new Event("input", { bubbles: true }));
});
const release = vi.fn(async () => ({}));
const send = vi.fn(async (_tabId: number, method: string, params?: object) => {
switch (method) {
case "DOM.describeNode":
return {
node: {
nodeName: element.tagName,
attributes: Array.from(element.attributes).flatMap((attr) => [attr.name, attr.value]),
},
};
case "DOM.scrollIntoViewIfNeeded":
return {};
case "DOM.focus":
element.focus();
return {};
case "DOM.resolveNode":
return { object: { objectId: "fill-target" } };
case "Runtime.callFunctionOn":
return script(params as ScriptParams);
case "Input.insertText":
await insert((params as { text: string }).text);
return {};
case "Input.dispatchKeyEvent":
if ((params as { commands?: string[] }).commands?.includes("moveToEndOfDocument")) {
await moveToEnd();
}
return {};
case "Runtime.releaseObject":
return release();
default:
throw new Error(`unexpected ${method}`);
}
});
return {
element,
hiddenCaret,
moveToEnd,
script,
insert,
release,
send,
fill: (value = "hello", clearBefore = true, signal?: AbortSignal) =>
handleFill(
manager,
{ session_id: "aa11", ref: "e1", value, clear_before: clearBefore },
{
cdp: { send: send as CdpRunner["send"] },
tabsApi: {
get: async (id) => ({ id, windowId: 100, active: true }) as chrome.tabs.Tab,
query: async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab],
},
signal,
},
),
};
}
afterEach(() => {
vi.restoreAllMocks();
document.body.innerHTML = "";
});
describe("fill result verification", () => {
it.each([
"readonly",
"disabled",
])("preserves a %s field before rejecting it", async (attribute) => {
const h = await setup(`<input ${attribute} value="old">`);
expect(await h.fill()).toMatchObject({
code: "invalid_params",
data: { reason: "target_not_fillable" },
});
expect(h.element.value).toBe("old");
expect(h.insert).not.toHaveBeenCalled();
expect(h.release).toHaveBeenCalledOnce();
});
it.each([
"checkbox",
"radio",
"range",
"date",
"file",
])("rejects an unsupported %s input before clearing", async (type) => {
const h = await setup(`<input type="${type}">`);
const before = h.element.value;
expect(await h.fill()).toMatchObject({
code: "invalid_params",
data: { reason: "target_not_fillable" },
});
expect(h.element.value).toBe(before);
expect(h.insert).not.toHaveBeenCalled();
});
it("rejects an overlong value before clearing", async () => {
const h = await setup('<input maxlength="3" value="old">');
expect(await h.fill()).toMatchObject({
code: "invalid_params",
message: /maxlength/,
data: { reason: "fill_value_invalid" },
});
expect(h.element.value).toBe("old");
expect(h.insert).not.toHaveBeenCalled();
});
it.each([
true,
false,
])("classifies a DOM.focus rejection on a disabled=%s target", async (disabled) => {
const h = await setup(`<input ${disabled ? "disabled" : ""} value="old">`);
const send = h.send.getMockImplementation()!;
h.send.mockImplementation(async (tabId, method, params) => {
if (method === "DOM.focus") throw new Error("Element is not focusable");
return send(tabId, method, params);
});
expect(await h.fill()).toMatchObject({
code: disabled ? "invalid_params" : "cdp_failed",
data: { reason: disabled ? "target_not_fillable" : "fill_failed" },
});
expect(h.element.value).toBe("old");
expect(h.insert).not.toHaveBeenCalled();
expect(h.release).toHaveBeenCalledOnce();
});
it("fills a background document without requiring foreground browser focus", async () => {
const h = await setup();
vi.spyOn(document, "hasFocus").mockReturnValue(false);
expect(await h.fill()).toMatchObject({ value_length: 5 });
expect(h.element.value).toBe("hello");
});
it("restores focus once after clearing without typing into another field", async () => {
const h = await setup();
const other = document.createElement("input");
document.body.append(other);
h.element.addEventListener("input", () => other.focus());
expect(await h.fill()).toMatchObject({ value_length: 5 });
expect(h.element.value).toBe("hello");
expect(other.value).toBe("");
expect(h.send.mock.calls.filter(([, method]) => method === "DOM.focus")).toHaveLength(2);
});
it.each([
"focus",
"value",
])("rechecks %s after restoring focus, without retrying again", async (action) => {
const h = await setup();
const other = document.createElement("input");
document.body.append(other);
h.element.addEventListener(
"input",
() => {
other.focus();
h.element.addEventListener("focus", () => {
if (action === "focus") queueMicrotask(() => other.focus());
else h.element.value = "changed";
});
},
{ once: true },
);
expect(await h.fill()).toMatchObject({
code: "cdp_failed",
data: { reason: action === "focus" ? "fill_focus_lost" : "fill_target_changed" },
});
expect(other.value).toBe("");
expect(h.insert).not.toHaveBeenCalled();
expect(h.send.mock.calls.filter(([, method]) => method === "DOM.focus")).toHaveLength(2);
});
it.each([
"remove",
"readonly",
"restore",
"microtask-restore",
])("stops when clearing causes the page to %s the target", async (action) => {
const h = await setup();
h.element.addEventListener("input", () => {
if (action === "remove") h.element.remove();
if (action === "readonly") h.element.readOnly = true;
if (action === "restore") h.element.value = "restored";
if (action === "microtask-restore")
queueMicrotask(() => {
h.element.value = "restored";
});
});
expect(await h.fill()).toMatchObject({
code: "cdp_failed",
data: { reason: "fill_target_changed" },
});
expect(h.insert).not.toHaveBeenCalled();
});
it("fails when insertText is acknowledged without changing the value", async () => {
const h = await setup();
h.insert.mockImplementation(async () => {});
expect(await h.fill()).toMatchObject({
code: "cdp_failed",
data: { reason: "fill_value_mismatch" },
});
expect(h.release).toHaveBeenCalledOnce();
});
it("detects truncation when maxlength changes after preparation", async () => {
const h = await setup();
h.element.addEventListener(
"input",
() => {
h.element.maxLength = 3;
},
{ once: true },
);
expect(await h.fill()).toMatchObject({
code: "cdp_failed",
data: { reason: "fill_value_mismatch" },
});
expect(h.element.value).toBe("hel");
});
it.each([
"input",
"change",
"microtask-change",
])("detects a value reverted by the %s handler", async (event) => {
const h = await setup();
h.element.addEventListener(event === "microtask-change" ? "change" : event, () => {
if (event === "microtask-change")
queueMicrotask(() => {
h.element.value = "";
});
else h.element.value = "";
});
expect(await h.fill()).toMatchObject({ code: "cdp_failed", message: /expected value/ });
});
it("compares content rather than length without including field values in the error", async () => {
const h = await setup('<input type="password">');
h.element.addEventListener("change", () => {
h.element.value = "other-secret";
});
const result = await h.fill("given-secret");
expect(result).toMatchObject({ code: "cdp_failed", message: /expected value/ });
expect(JSON.stringify(result)).not.toMatch(/given-secret|other-secret/);
});
it("does not report success for a target replaced by a change handler", async () => {
const h = await setup();
h.element.addEventListener("change", () => h.element.replaceWith(h.element.cloneNode()));
expect(await h.fill()).toMatchObject({
code: "cdp_failed",
data: { reason: "fill_target_changed" },
});
});
it.each([
1, 2, 3, 4, 5,
])("rejects script exceptions at phase %s and releases the target", async (phase) => {
const h = await setup();
vi.spyOn(document, "hasFocus").mockReturnValue(false);
const run = h.script.getMockImplementation()!;
let count = 0;
h.script.mockImplementation(async (params) =>
++count === phase ? { exceptionDetails: { text: "page secret" } } : run(params),
);
const result = await h.fill();
expect(result).toMatchObject({ code: "cdp_failed", data: { reason: "fill_failed" } });
expect(JSON.stringify(result)).not.toContain("page secret");
if (phase <= 3) expect(h.insert).not.toHaveBeenCalled();
expect(h.release).toHaveBeenCalledOnce();
});
it.each([1, 2, 3, 5])("rejects missing results at phase %s", async (phase) => {
const h = await setup();
vi.spyOn(document, "hasFocus").mockReturnValue(false);
const run = h.script.getMockImplementation()!;
let count = 0;
h.script.mockImplementation(async (params) =>
++count === phase ? { result: { type: "undefined" } } : run(params),
);
expect(await h.fill()).toMatchObject({ code: "cdp_failed" });
if (phase <= 3) expect(h.insert).not.toHaveBeenCalled();
});
it.each(["input", "change"])("checks state after nested microtasks from %s", async (event) => {
const h = await setup();
h.element.addEventListener(event, () => {
queueMicrotask(() =>
queueMicrotask(() => {
h.element.value = "reverted";
}),
);
});
expect(await h.fill()).toMatchObject({ code: "cdp_failed" });
if (event === "input") expect(h.insert).not.toHaveBeenCalled();
});
it.each([
"<input>",
'<input type="password">',
"<textarea></textarea>",
])("verifies successful text in %s", async (markup) => {
const h = await setup(markup);
expect(await h.fill("你好🙂")).toMatchObject({ value_length: 4 });
expect(h.element.value).toBe("你好🙂");
expect(h.release).toHaveBeenCalledOnce();
});
it("verifies append against the old value plus the requested text", async () => {
const h = await setup();
h.element.focus();
h.element.setSelectionRange(3, 3);
expect(await h.fill("!", false)).toMatchObject({ value_length: 4 });
expect(h.element.value).toBe("old!");
});
it.each([
["<textarea></textarea>", "a\r\nb", "a\nb"],
["<input>", "a\nb", "a b"],
["<input>", "a\r\nb", "a b"],
[
'<input type="email" multiple>',
" a@example.com , b@example.com ",
"a@example.com,b@example.com",
],
])("accepts native normalization for %s", async (markup, requested, normalized) => {
if (markup.startsWith("<textarea")) {
// Happy DOM does not implement Chrome's textarea line-ending normalization.
const setter = Object.getOwnPropertyDescriptor(HTMLTextAreaElement.prototype, "value")!.set!;
vi.spyOn(HTMLTextAreaElement.prototype, "value", "set").mockImplementation(function (
this: HTMLTextAreaElement,
value,
) {
setter.call(this, value.replace(/\r\n?/g, "\n"));
});
}
const h = await setup(markup);
h.insert.mockImplementation(async () => {
h.element.value = normalized;
});
expect(await h.fill(requested)).toMatchObject({ value_length: normalized.length });
});
it("does not count an editable padding break as an extra typed character", async () => {
const h = await setup('<div contenteditable="true" tabindex="0"></div>');
// These two layout/editability properties are missing in Happy DOM.
let innerText = "";
Object.defineProperty(h.element, "isContentEditable", { get: () => true });
Object.defineProperty(h.element, "innerText", { get: () => innerText });
h.insert.mockImplementation(async () => {
h.element.innerHTML = "one<div><br></div>";
innerText = "one\n\n";
});
expect(await h.fill("one\n")).toMatchObject({ value_length: 4 });
});
it.each([
[0, 0],
[1, 1],
[0, 3],
])("appends after moving selection %s:%s to the end", async (start, end) => {
const h = await setup();
h.element.focus();
h.element.setSelectionRange(start, end);
expect(await h.fill("🙂", false)).toMatchObject({ value_length: 5 });
expect(h.element.value).toBe("old🙂");
});
it.each(
["number", "email"].flatMap((type) =>
[
[0, 0],
[1, 1],
[0, 2],
].map(([start, end]) => ({ type, start, end })),
),
)("appends to $type from selection $start:$end without using its unsupported API", async ({
type,
start,
end,
}) => {
const h = await setup(`<input type="${type}" value="12">`);
Object.assign(h.hiddenCaret, { start, end });
vi.spyOn(h.element, "selectionStart", "get").mockReturnValue(null);
vi.spyOn(h.element, "selectionEnd", "get").mockReturnValue(null);
const selection = vi.spyOn(h.element, "setSelectionRange").mockImplementation(() => {
throw new DOMException("unsupported", "InvalidStateError");
});
expect(await h.fill("3", false)).toMatchObject({ value_length: 3 });
expect(h.element.value).toBe("123");
expect(selection).not.toHaveBeenCalled();
});
it.each([
"number",
"email",
])("does not report successful %s append when the caret command is ignored", async (type) => {
const h = await setup(`<input type="${type}" value="12">`);
vi.spyOn(h.element, "selectionStart", "get").mockReturnValue(null);
vi.spyOn(h.element, "selectionEnd", "get").mockReturnValue(null);
h.moveToEnd.mockResolvedValueOnce(undefined);
expect(await h.fill("3", false)).toMatchObject({
code: "cdp_failed",
data: { reason: "fill_value_mismatch" },
});
expect(h.element.value).toBe("312");
expect(h.insert).toHaveBeenCalledOnce();
});
it("rejects invalid number text before clearing", async () => {
const h = await setup('<input type="number" value="12">');
expect(await h.fill("abc")).toMatchObject({
code: "invalid_params",
data: { reason: "fill_value_invalid" },
});
expect(h.element.value).toBe("12");
expect(h.insert).not.toHaveBeenCalled();
});
it("reports page formatting as unconfirmed without retrying the write", async () => {
const h = await setup();
h.element.addEventListener("change", () => {
h.element.value = h.element.value.toUpperCase();
});
expect(await h.fill()).toMatchObject({
code: "cdp_failed",
data: { reason: "fill_value_mismatch" },
});
expect(h.element.value).toBe("HELLO");
expect(h.insert).toHaveBeenCalledOnce();
});
it.each([
["focus", false],
["focus", true],
["value", false],
["value", true],
["cancel", false],
["cancel", true],
["throw", false],
["throw", true],
] as const)("stops when positioning triggers %s (clear=%s) and releases the key", async (action, clearBefore) => {
const h = await setup();
vi.spyOn(document, "hasFocus").mockReturnValue(false);
const controller = new AbortController();
const other = document.createElement("input");
document.body.append(other);
const send = h.send.getMockImplementation()!;
h.send.mockImplementation(async (tabId, method, params) => {
const reply = await send(tabId, method, params);
if (
method === "Input.dispatchKeyEvent" &&
(params as { type: string }).type === "rawKeyDown"
) {
if (action === "focus") other.focus();
if (action === "value") h.element.value = "changed";
if (action === "cancel") controller.abort();
if (action === "throw") throw new Error("dispatch failed");
}
return reply;
});
const result = await h.fill("!", clearBefore, controller.signal);
expect(result).toMatchObject({ code: action === "cancel" ? "cancelled" : "cdp_failed" });
expect(h.insert).not.toHaveBeenCalled();
expect(other.value).toBe("");
expect(h.send).toHaveBeenCalledWith(
4,
"Input.dispatchKeyEvent",
expect.objectContaining({ type: "keyUp" }),
);
expect(h.release).toHaveBeenCalledOnce();
});
it.each([true, false])("verifies an empty request with clear_before=%s", async (clearBefore) => {
const h = await setup();
expect(await h.fill("", clearBefore)).toMatchObject({ value_length: clearBefore ? 0 : 3 });
expect(h.element.value).toBe(clearBefore ? "" : "old");
expect(h.insert).not.toHaveBeenCalled();
});
it("cancels after insertion without sending change events", async () => {
const h = await setup();
const controller = new AbortController();
h.insert.mockImplementation(async () => controller.abort());
expect(await h.fill("hello", true, controller.signal)).toMatchObject({ code: "cancelled" });
expect(h.script).toHaveBeenCalledTimes(2);
expect(h.release).toHaveBeenCalledOnce();
});
it("cancels after resolving the remote object without clearing the field", async () => {
const h = await setup();
const controller = new AbortController();
const send = h.send.getMockImplementation()!;
h.send.mockImplementation(async (tabId, method, params) => {
const result = await send(tabId, method, params);
if (method === "DOM.resolveNode") controller.abort();
return result;
});
expect(await h.fill("hello", true, controller.signal)).toMatchObject({ code: "cancelled" });
expect(h.element.value).toBe("old");
expect(h.script).not.toHaveBeenCalled();
expect(h.release).toHaveBeenCalledOnce();
});
it("honors cancellation during verification", async () => {
const h = await setup();
const controller = new AbortController();
h.element.addEventListener("change", () => controller.abort());
expect(await h.fill("hello", true, controller.signal)).toMatchObject({ code: "cancelled" });
expect(h.release).toHaveBeenCalledOnce();
});
it("keeps the verified outcome when releasing the remote object fails", async () => {
const h = await setup();
h.release.mockRejectedValueOnce(new Error("target closed"));
expect(await h.fill()).toMatchObject({ value_length: 5 });
});
});
@@ -1,5 +1,5 @@
import { describe, expect, it, vi } from "vitest";
import { resolveFrameProjection, resolveNodeGeometry } from "../frame-geometry";
import { resolveNodeGeometry } from "../frame-geometry";
import {
clipPolygon,
polygonArea,
@@ -10,6 +10,7 @@ import {
rectPolygon,
regionBounds,
} from "../geometry";
import { GeometryContext } from "../geometry/frame-context";
import type { CdpRunner } from "../shared";
describe("frame geometry projection", () => {
@@ -82,28 +83,24 @@ describe("frame geometry projection", () => {
throw new Error(`unexpected child command ${method}`);
}) as CdpRunner["sendToTarget"],
};
const projection = await resolveFrameProjection(
cdp,
{
rootFrameId: "main",
frames: [
{ frameId: "main", target: { tabId: 4 } },
{
frameId: "same-process-parent",
parentFrameId: "main",
ownerBackendNodeId: 10,
target: { tabId: 4 },
},
{
frameId: "oopif",
parentFrameId: "same-process-parent",
ownerBackendNodeId: 20,
target: { tabId: 4, sessionId: "oopif-session" },
},
],
},
"oopif",
);
const projection = await new GeometryContext(cdp, 4, {
rootFrameId: "main",
frames: [
{ frameId: "main", target: { tabId: 4 } },
{
frameId: "same-process-parent",
parentFrameId: "main",
ownerBackendNodeId: 10,
target: { tabId: 4 },
},
{
frameId: "oopif",
parentFrameId: "same-process-parent",
ownerBackendNodeId: 20,
target: { tabId: 4, sessionId: "oopif-session" },
},
],
}).targetProjection("oopif");
expect(projection?.edges).toEqual([
{
@@ -0,0 +1,464 @@
import { describe, expect, it, vi } from "vitest";
import type { CdpFrameGraph } from "@/browser-driver/frame-graph";
import { resolveNodeGeometry } from "../frame-geometry";
import { rectPolygon } from "../geometry";
import {
projectSnapshotRect,
screenshotPageRect,
snapshotCoordinates,
snapshotViewportRect,
} from "../geometry/coordinate-types";
import { GeometryContext, snapshotLayoutScale } from "../geometry/frame-context";
import type { CdpRunner } from "../shared";
const target = { tabId: 4 };
const graph: CdpFrameGraph = {
rootFrameId: "main",
frames: [
{ frameId: "main", target },
{ frameId: "child", parentFrameId: "main", ownerBackendNodeId: 10, target },
],
};
function driver() {
const send = vi.fn(async (_tab: number, method: string): Promise<object> => {
if (method === "Page.getLayoutMetrics")
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 1000 },
};
if (method === "DOM.getBoxModel")
return { model: { content: [52.5, 612.5, 427.5, 612.5, 427.5, 862.5, 52.5, 862.5] } };
if (method === "DOM.resolveNode") return { object: { objectId: "owner" } };
if (method === "Runtime.callFunctionOn")
return { result: { value: { width: 300, height: 200 } } };
if (method === "Runtime.releaseObject") return {};
throw new Error(method);
});
return { send: send as CdpRunner["send"], getFrameGraph: vi.fn(async () => graph), calls: send };
}
describe("measurement geometry context", () => {
it.each([0.8, 1, 1.25, 2])("normalizes snapshot layout units at scale %s", (scale) => {
const measured = snapshotLayoutScale({
visualViewport: { clientWidth: 997.25, zoom: 1.1 },
cssVisualViewport: { clientWidth: 997.25 / scale, zoom: 1.1 },
// Rounded layout dimensions must not supply the scale.
layoutViewport: { clientWidth: 997 },
cssLayoutViewport: { clientWidth: Math.floor(997.25 / scale) },
});
expect(measured).toBeCloseTo(scale, 12);
const coordinates = snapshotCoordinates(
{ scrollOffsetX: 80 * scale, scrollOffsetY: 200 * scale },
measured,
{ x: 999, y: 999 },
);
expect(
snapshotViewportRect(
[160 * scale, 320 * scale, 120 * scale, 40 * scale],
{ target },
coordinates,
)?.rect,
).toEqual({ x: 80, y: 120, width: 120, height: 40 });
});
it("uses CSS root scroll only for missing snapshot offsets, without scaling it twice", () => {
const coordinates = snapshotCoordinates({ scrollOffsetX: 80 }, 2, { x: 999, y: 100 });
expect(snapshotViewportRect([200, 800, 240, 80], { target }, coordinates)?.rect).toEqual({
x: 60,
y: 300,
width: 120,
height: 40,
});
expect(snapshotCoordinates({ scrollOffsetX: 0 }, 2)).toBeNull();
expect(
snapshotCoordinates({ scrollOffsetX: NaN, scrollOffsetY: 0 }, 2, { x: 0, y: 0 }),
).toBeNull();
});
it("does not infer snapshot units from missing, invalid, or CSS-only metrics", () => {
expect(snapshotLayoutScale({ cssLayoutViewport: { clientWidth: 800 } })).toBeNull();
for (const raw of [0, -1, NaN, Infinity]) {
expect(
snapshotLayoutScale({
visualViewport: { clientWidth: raw },
cssVisualViewport: { clientWidth: 800 },
}),
).toBeNull();
}
expect(
snapshotLayoutScale({
visualViewport: { clientWidth: 0, clientHeight: 600 },
cssVisualViewport: { clientWidth: 0, clientHeight: 300 },
}),
).toBe(2);
expect(snapshotCoordinates({ scrollOffsetX: 0, scrollOffsetY: 0 }, null)).toBeNull();
expect(snapshotViewportRect([10, 20, 120, 40], { target }, null)).toBeNull();
});
it("shares in-flight frame measurements, but never shares them with the next context", async () => {
const cdp = driver();
const context = new GeometryContext(cdp, 4);
const results = await Promise.all(
Array.from({ length: 20 }, () => context.targetProjection("child")),
);
expect(results.every((value) => value === results[0])).toBe(true);
expect(cdp.getFrameGraph).toHaveBeenCalledTimes(1);
expect(cdp.calls.mock.calls.map(([, method]) => method)).toEqual([
"Page.getLayoutMetrics",
"DOM.getBoxModel",
]);
await new GeometryContext(cdp, 4).targetProjection("child");
expect(cdp.getFrameGraph).toHaveBeenCalledTimes(2);
expect(cdp.calls.mock.calls).toHaveLength(4);
});
it("projects frame-local snapshot CSS bounds through the scaled owner content quad", async () => {
const cdp = driver();
const context = new GeometryContext(cdp, 4);
const source = { target, frameId: "child" };
const projection = await context.snapshotProjection(source, 10, [], {
width: 1000,
height: 1000,
});
if (projection.status !== "available") throw new Error("expected available projection");
const input = snapshotViewportRect([17, 23, 120, 40], source, {
layoutUnitsPerCssPixel: 1,
scrollCss: { x: 0, y: 0 },
});
expect(projectSnapshotRect(input!, projection.projection)).toEqual({
x: 73.75,
y: 641.25,
width: 150,
height: 50,
});
const scrolled = snapshotViewportRect([17, 23, 120, 40], source, {
layoutUnitsPerCssPixel: 1,
scrollCss: { x: 0, y: 10 },
});
expect(projectSnapshotRect(scrolled!, projection.projection)?.y).toBe(628.75);
await context.snapshotProjection(source, 10, [], { width: 1000, height: 1000 });
expect(cdp.calls.mock.calls.filter(([, method]) => method === "DOM.resolveNode")).toHaveLength(
1,
);
expect(
cdp.calls.mock.calls.filter(([, method]) => method === "Runtime.releaseObject"),
).toHaveLength(1);
});
it("falls back only for missing or invalid batch entries and does not reuse sizes across contexts", async () => {
const cdp = driver();
const original = cdp.send;
const calls: string[] = [];
cdp.send = async (tabId, method, params) => {
calls.push(method);
if (method === "Runtime.evaluate")
return {
result: {
deepSerializedValue: {
type: "array",
value: [
{
type: "array",
value: [
{ type: "node", value: { backendNodeId: 10 } },
{ type: "string", value: '{"width":300,"height":200}' },
],
},
{
type: "array",
value: [
{ type: "node", value: { backendNodeId: 11 } },
{ type: "string", value: '{"width":0,"height":200}' },
],
},
],
},
},
} as never;
if (method === "Runtime.releaseObjectGroup") return {} as never;
return original(tabId, method, params);
};
const context = new GeometryContext(cdp, 4);
context.registerSnapshotOwners(target, new Set([10, 11]));
const values = await Promise.all(
[10, 11].map((id) =>
context.snapshotProjection({ target, frameId: String(id) }, id, [], {
width: 1000,
height: 1000,
}),
),
);
expect(values.every((value) => value.status === "available")).toBe(true);
expect(calls.filter((method) => method === "DOM.resolveNode")).toHaveLength(1);
expect(calls.filter((method) => method === "Runtime.evaluate")).toHaveLength(1);
await new GeometryContext(cdp, 4).snapshotProjection({ target, frameId: "child" }, 10, [], {
width: 1000,
height: 1000,
});
expect(calls.filter((method) => method === "DOM.resolveNode")).toHaveLength(2);
});
it("waits for batch object cleanup and propagates cancellation", async () => {
const cdp = driver();
const original = cdp.send;
const controller = new AbortController();
let release!: () => void;
let cleanup!: () => void;
const started = new Promise<void>((resolve) => {
cleanup = resolve;
});
cdp.send = async (tabId, method, params) => {
if (method === "Runtime.evaluate") return {} as never;
if (method === "Runtime.releaseObjectGroup") {
cleanup();
await new Promise<void>((resolve) => {
release = resolve;
});
return {} as never;
}
return original(tabId, method, params);
};
const context = new GeometryContext(cdp, 4, undefined, controller.signal);
context.registerSnapshotOwners(target, new Set([10, 11]));
let settled = false;
const operation = context
.snapshotProjection({ target, frameId: "child" }, 10, [], { width: 1000, height: 1000 })
.finally(() => {
settled = true;
});
const rejected = expect(operation).rejects.toMatchObject({ name: "AbortError" });
await started;
controller.abort();
expect(settled).toBe(false);
release();
await rejected;
expect(cdp.calls.mock.calls.some(([, method]) => method === "DOM.resolveNode")).toBe(false);
});
it("rejects owner mismatches and invalid numeric bounds", () => {
const input = snapshotViewportRect(
[30, 50, 120, 40],
{ target, frameId: "main" },
{ layoutUnitsPerCssPixel: 1, scrollCss: { x: 10, y: 20 } },
)!;
const projection = {
source: { target, frameId: "main" },
geometry: { sourceClips: [], edges: [], topViewport: { width: 800, height: 600 } },
};
expect(projectSnapshotRect(input, projection)).toEqual({
x: 20,
y: 30,
width: 120,
height: 40,
});
expect(
projectSnapshotRect(input, { ...projection, source: { target, frameId: "other" } }),
).toBeNull();
expect(
projectSnapshotRect(input, {
...projection,
source: { target: { tabId: 5 }, frameId: "main" },
}),
).toBeNull();
expect(
snapshotViewportRect(
[0, 0, NaN, 40],
{ target },
{ layoutUnitsPerCssPixel: 1, scrollCss: { x: 0, y: 0 } },
),
).toBeNull();
expect(
snapshotViewportRect(
[0, 0, 10, 40],
{ target },
{ layoutUnitsPerCssPixel: 1, scrollCss: { x: Infinity, y: 0 } },
),
).toBeNull();
});
it("does not turn a border quad or a failed owner read into a content projection", async () => {
const cdp: CdpRunner = {
send: vi.fn(async () => ({
model: { border: [0, 0, 100, 0, 100, 100, 0, 100] },
})) as CdpRunner["send"],
};
expect(
await new GeometryContext(cdp, 4).snapshotProjection({ target, frameId: "child" }, 10, [], {
width: 100,
height: 100,
}),
).toEqual({
status: "unavailable",
source: { target, frameId: "child" },
ownerBackendNodeId: 10,
});
});
it("keeps ancestor clips in target coordinates instead of applying nested offsets twice", async () => {
const cdp = driver();
const projection = await new GeometryContext(cdp, 4).snapshotProjection(
{ target, frameId: "nested" },
10,
[rectPolygon({ x: 60, y: 620, w: 100, h: 100 })],
{ width: 1000, height: 1000 },
);
if (projection.status !== "available") throw new Error("expected available projection");
const input = snapshotViewportRect(
[0, 0, 300, 200],
{ target, frameId: "nested" },
{ layoutUnitsPerCssPixel: 1, scrollCss: { x: 0, y: 0 } },
);
expect(projectSnapshotRect(input!, projection.projection)).toEqual({
x: 60,
y: 620,
width: 100,
height: 100,
});
});
it("rejects malformed frame ancestry without looping", async () => {
const cdp = driver();
const cycle: CdpFrameGraph = {
rootFrameId: "a",
frames: [
{ frameId: "a", parentFrameId: "b", target },
{ frameId: "b", parentFrameId: "a", target },
],
};
expect(await new GeometryContext(cdp, 4, cycle).targetProjection("a")).toBeNull();
expect(cdp.calls).not.toHaveBeenCalled();
const orphan: CdpFrameGraph = {
rootFrameId: "a",
frames: [{ frameId: "a", parentFrameId: "missing", target }],
};
expect(await new GeometryContext(cdp, 4, orphan).targetProjection("a")).toBeNull();
});
it("rejects a live node whose frame belongs to another target before any DOM input", async () => {
const cdp = driver();
expect(
await resolveNodeGeometry(
cdp,
4,
{ target: { tabId: 4, sessionId: "wrong" }, frameId: "child", backendNodeId: 20 },
{ scrollIntoView: true },
),
).toMatchObject({ code: "cdp_failed" });
expect(cdp.calls).not.toHaveBeenCalled();
});
it("bounds concurrent measurements and does not dispatch queued reads after cancellation", async () => {
let active = 0;
let peak = 0;
const release: Array<() => void> = [];
const send = vi.fn(async () => {
active++;
peak = Math.max(peak, active);
await new Promise<void>((resolve) => release.push(resolve));
active--;
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 100, clientHeight: 100 },
};
});
const controller = new AbortController();
const context = new GeometryContext(
{ send: send as CdpRunner["send"] },
4,
undefined,
controller.signal,
);
const tasks = Array.from({ length: 8 }, (_, index) =>
context.viewport({ tabId: 4, sessionId: String(index) }),
);
const settled = Promise.allSettled(tasks);
expect(send).toHaveBeenCalledTimes(4);
controller.abort();
for (const done of release) done();
const results = await settled;
expect(peak).toBe(4);
expect(send).toHaveBeenCalledTimes(4);
expect(results.filter((item) => item.status === "rejected")).toHaveLength(4);
});
it("releases an owner object even when cancellation arrives before its viewport read", async () => {
const controller = new AbortController();
const cdp = driver();
const original = cdp.calls.getMockImplementation()!;
cdp.calls.mockImplementation(async (tab, method) => {
const result = await original(tab, method);
if (method === "DOM.resolveNode") controller.abort();
return result;
});
const context = new GeometryContext(cdp, 4, undefined, controller.signal);
await expect(
context.snapshotProjection({ target, frameId: "child" }, 10, [], {
width: 1000,
height: 1000,
}),
).rejects.toMatchObject({ name: "AbortError" });
expect(cdp.calls.mock.calls.map(([, method]) => method)).toEqual([
"DOM.getBoxModel",
"DOM.resolveNode",
"Runtime.releaseObject",
]);
});
it("reads live owner geometry only after scrolling", async () => {
const cdp = driver();
const original = cdp.calls.getMockImplementation()!;
let scrolled = false;
cdp.calls.mockImplementation(async (tab, method) => {
if (method === "DOM.scrollIntoViewIfNeeded") {
scrolled = true;
return {};
}
if (method === "DOM.getContentQuads") {
expect(scrolled).toBe(true);
return { quads: [[60, 620, 100, 620, 100, 640, 60, 640]] };
}
if (method === "DOM.getBoxModel") expect(scrolled).toBe(true);
return original(tab, method);
});
expect(
await resolveNodeGeometry(
cdp,
4,
{ target, frameId: "child", backendNodeId: 20 },
{ scrollIntoView: true },
),
).toMatchObject({ topBounds: { x: 60, y: 620, width: 40, height: 20 } });
expect(cdp.getFrameGraph).toHaveBeenCalledTimes(1);
expect(cdp.calls.mock.calls.filter(([, method]) => method === "DOM.getBoxModel")).toHaveLength(
1,
);
});
it("adapts viewport CSS bounds to page DIP with scroll and browser zoom, never raster DPR", () => {
const viewport = { width: 1600, height: 1000, scrollX: 10, scrollY: 100, cssToDip: 0.9 };
const clip = screenshotPageRect({ x: 20, y: 30, width: 120, height: 40 }, viewport);
expect(clip).toEqual({ space: "page-dip", rect: { x: 27, y: 117, width: 108, height: 36 } });
expect(
screenshotPageRect({ x: 20, y: 30, width: 120, height: 40 }, { ...viewport, cssToDip: NaN }),
).toBeNull();
expect(
screenshotPageRect({ x: 20, y: 30, width: 120, height: 40 }, { ...viewport, cssToDip: 0 }),
).toBeNull();
});
it("retries a rejected measurement only in a new operation", async () => {
const cdp = driver();
cdp.calls.mockRejectedValueOnce(new Error("unavailable"));
const context = new GeometryContext(cdp, 4);
await expect(context.viewport(target)).rejects.toThrow("unavailable");
await expect(context.viewport(target)).rejects.toThrow("unavailable");
expect(cdp.calls).toHaveBeenCalledTimes(1);
expect(await new GeometryContext(cdp, 4).viewport(target)).toEqual({
width: 1000,
height: 1000,
});
});
});
@@ -21,7 +21,7 @@ function fakeAgentWindow(ids: number[]) {
return id;
}),
remove: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
};
}
@@ -540,6 +540,21 @@ describe("handleHover", () => {
});
});
function successfulFillScript(params: unknown) {
const script = params as { arguments?: Array<{ value: unknown }>; functionDeclaration: string };
const args = script.arguments ?? [];
return {
result: {
value:
args.length === 2
? { before: "", expected: args[0].value }
: script.functionDeclaration.startsWith("function(expected)")
? { connected: true, matches: true, valueLength: String(args[0].value).length }
: "ready",
},
};
}
describe("handleFill", () => {
it("returns not_found for unknown ref", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
@@ -586,7 +601,8 @@ describe("handleFill", () => {
"DOM.scrollIntoViewIfNeeded": () => ({}),
"DOM.focus": () => ({}),
"DOM.resolveNode": () => ({ object: { objectId: "obj-1" } }),
"Runtime.callFunctionOn": () => ({ result: { type: "undefined" } }),
"Runtime.callFunctionOn": successfulFillScript,
"Runtime.releaseObject": () => ({}),
"Input.insertText": () => ({}),
});
const res = await handleFill(
@@ -600,13 +616,12 @@ describe("handleFill", () => {
expect(res.used_ref).toBe("e1");
const insert = fake.sent.find((c) => c.method === "Input.insertText");
expect(insert?.params).toEqual({ text: "hello" });
// clear_before defaults to true → callFunctionOn invoked once to
// clear, once to fire input/change after typing.
// Foreground replacement needs no extra caret-positioning round trip.
const callFns = fake.sent.filter((c) => c.method === "Runtime.callFunctionOn");
expect(callFns.length).toBeGreaterThanOrEqual(2);
expect(callFns).toHaveLength(4);
});
it("clear_before=false skips the wipe call", async () => {
it("passes clear_before=false to preparation and verifies the result", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
ctx.refStore.set("e1", 1, { tabId: 4 });
@@ -618,12 +633,12 @@ describe("handleFill", () => {
"DOM.focus": () => ({}),
"DOM.resolveNode": () => ({ object: { objectId: "obj-2" } }),
"Runtime.callFunctionOn": (p) => {
const fn = (p as { functionDeclaration?: string }).functionDeclaration ?? "";
// No "this.value = ''" clearing on a no-clear path; only the
// post-input dispatchEvent.
expect(fn).not.toMatch(/this\.value\s*=\s*''/);
return { result: { type: "undefined" } };
const args = (p as { arguments?: Array<{ value: unknown }> }).arguments ?? [];
if (args.length === 2) expect(args[1].value).toBe(false);
return successfulFillScript(p);
},
"Input.dispatchKeyEvent": () => ({}),
"Runtime.releaseObject": () => ({}),
"Input.insertText": () => ({}),
});
const res = await handleFill(
@@ -650,7 +665,8 @@ describe("handleFill", () => {
"DOM.scrollIntoViewIfNeeded": () => ({}),
"DOM.focus": () => ({}),
"DOM.resolveNode": () => ({ object: { objectId: "obj-3" } }),
"Runtime.callFunctionOn": () => ({ result: { type: "undefined" } }),
"Runtime.callFunctionOn": successfulFillScript,
"Runtime.releaseObject": () => ({}),
"Input.insertText": () => ({}),
});
const res = await handleFill(
@@ -0,0 +1,266 @@
import { describe, expect, it, vi } from "vitest";
import { SessionManager } from "@/session-manager/manager";
import type { BrowserNavigationApi } from "../browser-navigation";
import { handleNavigate, handleReload } from "../navigation";
import type { CdpRunner } from "../shared";
const denied = "Cannot access a chrome-extension:// URL of different extension";
const url = "https://example.test/recovered";
function event<T extends (...args: never[]) => void>() {
const listeners = new Set<T>();
return {
listeners,
addListener: (listener: T) => listeners.add(listener),
removeListener: (listener: T) => listeners.delete(listener),
fire: (...args: Parameters<T>) => {
for (const listener of [...listeners]) listener(...args);
},
};
}
async function fixture() {
const manager = new SessionManager({
agentWindow: {
create: vi.fn(async () => 100),
remove: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 4),
},
});
await manager.start("test");
type Listener = Parameters<BrowserNavigationApi["onCommitted"]["addListener"]>[0];
const events = {
onCommitted: event<Listener>(),
onDOMContentLoaded: event<Listener>(),
onCompleted: event<Listener>(),
onErrorOccurred: event<Listener>(),
};
const cdpEvents = event<Parameters<NonNullable<CdpRunner["onEvent"]>>[0]>();
const state = { blocked: true, destinationBlocked: false, emitIdle: true };
const send = vi.fn(async (_tabId: number, method: string) => {
if (state.blocked) throw new Error(denied);
if (method === "Page.getFrameTree") {
return { frameTree: { frame: { id: "main", loaderId: "new-loader" } } };
}
if (method === "Runtime.evaluate") return { result: { value: "complete" } };
if (method === "Page.setLifecycleEventsEnabled" && state.emitIdle) {
cdpEvents.fire({ tabId: 4 }, "Page.lifecycleEvent", {
frameId: "main",
loaderId: "new-loader",
name: "networkIdle",
});
}
if (method === "Page.navigate") return { frameId: "main", loaderId: "new-loader" };
return {};
});
const cdp: CdpRunner = {
send: send as CdpRunner["send"],
onEvent: (listener) => {
cdpEvents.addListener(listener);
return { dispose: () => cdpEvents.removeListener(listener) };
},
};
const main = { tabId: 4, frameId: 0, documentId: "new-document" };
const navigate = async () => {
state.blocked = state.destinationBlocked;
events.onCommitted.fire(main);
events.onDOMContentLoaded.fire(main);
events.onCompleted.fire(main);
};
const browserNavigation = { ...events, update: vi.fn(navigate), reload: vi.fn(navigate) };
const tab = { id: 4, windowId: 100, active: true, url } as chrome.tabs.Tab;
const tabsApi = { get: vi.fn(async () => tab), query: vi.fn(async () => [tab]) };
const deps = { cdp, tabsApi, browserNavigation, defaultTimeoutMs: 1000 };
const expectCleanedUp = () => {
for (const entry of Object.values(events)) expect(entry.listeners.size).toBe(0);
expect(cdpEvents.listeners.size).toBe(0);
};
return { manager, deps, state, send, events, main, cdpEvents, expectCleanedUp };
}
describe("navigation after Chrome denies extension-frame access", () => {
it.each([
"load",
"commit",
"domcontentloaded",
"networkidle",
] as const)("recovers the same tab with the requested %s checkpoint", async (waitUntil) => {
const f = await fixture();
const result = await handleNavigate(
f.manager,
{
session_id: "test",
url,
wait_until: waitUntil,
},
f.deps,
);
expect(result).toMatchObject({ tab_id: 4, final_url: url, reached: waitUntil });
expect(f.deps.browserNavigation.update).toHaveBeenCalledExactlyOnceWith(4, { url });
expect(f.send.mock.calls.some(([, method]) => method === "Page.navigate")).toBe(false);
expect(f.send).toHaveBeenCalledWith(4, "Page.enable", {});
f.expectCleanedUp();
});
it("recovers reload through tabs.reload and preserves hard reload", async () => {
const f = await fixture();
const result = await handleReload(f.manager, { session_id: "test", hard: true }, f.deps);
expect(result).toMatchObject({ tab_id: 4, reached: "load" });
expect(f.deps.browserNavigation.reload).toHaveBeenCalledExactlyOnceWith(4, {
bypassCache: true,
});
expect(f.deps.browserNavigation.update).not.toHaveBeenCalled();
f.expectCleanedUp();
});
it("reports the restriction when reload leaves the conflicting frame present", async () => {
const f = await fixture();
f.state.destinationBlocked = true;
const result = await handleReload(f.manager, { session_id: "test" }, f.deps);
expect(result).toMatchObject({
code: "cdp_failed",
data: { reason: "cdp_extension_access_denied" },
});
expect(f.deps.browserNavigation.reload).toHaveBeenCalledOnce();
f.expectCleanedUp();
});
it("does not fall back for unrelated debugger failures", async () => {
const f = await fixture();
f.send.mockRejectedValue(new Error("Another debugger is already attached"));
const result = await handleNavigate(f.manager, { session_id: "test", url }, f.deps);
expect(result).toMatchObject({
code: "cdp_failed",
message: "Another debugger is already attached",
});
expect(f.deps.browserNavigation.update).not.toHaveBeenCalled();
f.expectCleanedUp();
});
it("does not replay navigation when a dispatched CDP command fails", async () => {
const f = await fixture();
f.state.blocked = false;
const send = f.send.getMockImplementation()!;
f.send.mockImplementation(async (tabId, method) => {
if (method === "Page.navigate") throw new Error(denied);
return send(tabId, method);
});
const result = await handleNavigate(f.manager, { session_id: "test", url }, f.deps);
expect(result).toMatchObject({
code: "cdp_failed",
data: { reason: "cdp_extension_access_denied" },
});
expect(f.deps.browserNavigation.update).not.toHaveBeenCalled();
f.expectCleanedUp();
});
it("keeps the Agent Window guard in front of the recovery", async () => {
const f = await fixture();
f.deps.tabsApi.get.mockResolvedValue({
id: 4,
windowId: 200,
active: true,
url,
} as chrome.tabs.Tab);
const result = await handleNavigate(f.manager, { session_id: "test", tab_id: 4, url }, f.deps);
expect(result).toMatchObject({ code: "permission_denied" });
expect(f.send).not.toHaveBeenCalled();
expect(f.deps.browserNavigation.update).not.toHaveBeenCalled();
});
it("ignores old-document, subframe, and other-tab completion events", async () => {
const f = await fixture();
f.deps.browserNavigation.update.mockImplementation(async () => {});
let settled = false;
const work = handleNavigate(f.manager, { session_id: "test", url }, f.deps).then((result) => {
settled = true;
return result;
});
await vi.waitFor(() => expect(f.deps.browserNavigation.update).toHaveBeenCalledOnce());
f.events.onCompleted.fire(f.main);
f.events.onCommitted.fire({ ...f.main, tabId: 9 });
f.events.onCommitted.fire({ ...f.main, frameId: 1 });
f.events.onCompleted.fire(f.main);
await Promise.resolve();
expect(settled).toBe(false);
f.events.onCommitted.fire(f.main);
f.events.onCompleted.fire({ ...f.main, documentId: "old-document" });
await Promise.resolve();
expect(settled).toBe(false);
f.state.blocked = false;
f.events.onCompleted.fire(f.main);
expect(await work).toMatchObject({ reached: "load" });
f.expectCleanedUp();
});
it("does not infer network idle from a browser load event", async () => {
const f = await fixture();
f.state.emitIdle = false;
const result = await handleNavigate(
f.manager,
{
session_id: "test",
url,
wait_until: "networkidle",
timeout_ms: 10,
},
f.deps,
);
expect(result).toMatchObject({ reached: "timeout" });
f.expectCleanedUp();
});
it("cleans up browser listeners on timeout", async () => {
const f = await fixture();
f.deps.browserNavigation.update.mockImplementation(async () => {});
const result = await handleNavigate(
f.manager,
{ session_id: "test", url, timeout_ms: 10 },
f.deps,
);
expect(result).toMatchObject({ reached: "timeout" });
f.expectCleanedUp();
});
it("does not navigate an already-cancelled request", async () => {
const f = await fixture();
const result = await handleNavigate(
f.manager,
{ session_id: "test", url },
{
...f.deps,
signal: AbortSignal.abort(),
},
);
expect(result).toMatchObject({ code: "cancelled" });
expect(f.deps.browserNavigation.update).not.toHaveBeenCalled();
f.expectCleanedUp();
});
it("cleans up browser listeners when cancelled after dispatch", async () => {
const f = await fixture();
const ac = new AbortController();
f.deps.browserNavigation.update.mockImplementation(async () => {
ac.abort();
});
const result = await handleNavigate(
f.manager,
{ session_id: "test", url },
{ ...f.deps, signal: ac.signal },
);
expect(result).toMatchObject({ code: "cancelled" });
expect(f.deps.browserNavigation.update).toHaveBeenCalledOnce();
f.expectCleanedUp();
});
it("reports browser navigation errors and releases listeners", async () => {
const f = await fixture();
f.deps.browserNavigation.update.mockImplementation(async () => {
f.events.onErrorOccurred.fire({ ...f.main, error: "net::ERR_NAME_NOT_RESOLVED" });
});
const result = await handleNavigate(f.manager, { session_id: "test", url }, f.deps);
expect(result).toMatchObject({ code: "cdp_failed", message: "net::ERR_NAME_NOT_RESOLVED" });
f.expectCleanedUp();
});
});
@@ -21,7 +21,7 @@ function fakeAgentWindow(ids: number[]) {
return id;
}),
remove: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
};
}
@@ -12,7 +12,7 @@ function fakeAgentWindow(ids: number[]) {
return id;
}),
remove: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
};
}
@@ -28,7 +28,7 @@ function fakeAgentWindow(ids: number[]) {
return id;
}),
remove: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
};
}
@@ -65,7 +65,11 @@ function makeFakeCdp(handlers: Record<string, (params?: object) => unknown>) {
sent.push({ method, params });
const handler = handlers[method];
if (!handler && method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 },
};
}
if (!handler) throw new Error(`unexpected CDP call ${method}`);
return handler(params);
@@ -266,7 +270,15 @@ describe("handleScreenshot", () => {
expect(sent.find((c) => c.method === "Page.captureScreenshot")).toBeUndefined();
});
it("captures a clipped PNG when ref is given", async () => {
it.each([
{ zoom: 1, scroll: 0, expected: { x: 10, y: 20, width: 100, height: 40 } },
{ zoom: 0.9, scroll: 100, expected: { x: 9, y: 108, width: 90, height: 36 } },
{ zoom: 1.25, scroll: 100, expected: { x: 12.5, y: 150, width: 125, height: 50 } },
])("captures a ref in page DIP at zoom $zoom and scroll $scroll", async ({
zoom,
scroll,
expected,
}) => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
ctx.refStore.set("e5", 999, { tabId: 7 });
@@ -274,6 +286,10 @@ describe("handleScreenshot", () => {
"DOM.scrollIntoViewIfNeeded": () => ({}),
"DOM.getContentQuads": () => ({ quads: [[10, 20, 110, 20, 110, 60, 10, 60]] }),
"Page.captureScreenshot": () => ({ data: TINY_PNG }),
"Page.getLayoutMetrics": () => ({
cssLayoutViewport: { clientWidth: 1280, clientHeight: 720, pageX: 0, pageY: scroll },
cssVisualViewport: { zoom },
}),
});
const captureVisibleTab = vi.fn();
const res = await handleScreenshot(
@@ -297,7 +313,8 @@ describe("handleScreenshot", () => {
clip?: { x: number; y: number; width: number; height: number };
}
)?.clip;
expect(clip).toMatchObject({ x: 10, y: 20, width: 100, height: 40 });
expect(clip).toMatchObject(expected);
expect(sent.filter((c) => c.method === "Page.getLayoutMetrics")).toHaveLength(1);
});
it("returns not_found for unknown ref", async () => {
@@ -655,6 +672,60 @@ describe("buildVomScene", () => {
expect(nestedInput?.parentId).toBe(result.nodes.find((item) => item.backendNodeId === 210)?.id);
});
it("excludes an overlay only in its owning document when target backend ids collide", () => {
const owner = (id: number): CapturedNode => ({
backendNodeId: id,
parentBackendNodeId: null,
frameId: "main",
tag: "iframe",
attrs: {},
rect: { x: id * 10, y: 0, w: 200, h: 100 },
paintOrder: 0,
position: "static",
pointerEvents: "auto",
});
const owners = [owner(10), owner(20)];
const result = buildFrameVomScene(
[
{
frameId: "main",
contextScopeId: "main",
target: { tabId: 7 },
domNodes: owners,
axNodes: [],
},
...["left", "right"].map((frameId, i) => ({
frameId,
contextScopeId: frameId,
parentFrameId: "main",
ownerBackendNodeId: owners[i].backendNodeId,
target: { tabId: 7, sessionId: frameId },
excludedBackendNodeIds: new Set(i === 0 ? [7] : []),
domNodes: [],
axNodes: [
{
nodeId: "button",
frameId,
backendDOMNodeId: 7,
role: { type: "role", value: "button" },
name: { type: "computedString", value: frameId + " action" },
},
],
})),
],
{
nodes: owners,
viewport: { width: 800, height: 600 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(),
},
);
const rendered = renderVom(result);
expect(rendered.text).not.toContain("left action");
expect(rendered.text).toContain("right action");
expect(rendered.refs.find((ref) => ref.backendNodeId === 7)?.frameId).toBe("right");
});
it("does not place a child document at the page root when its frame boundary is unresolved", () => {
const childNode: CapturedNode = {
backendNodeId: 101,
@@ -2481,7 +2552,11 @@ describe("handleSnapshot", () => {
if (method === "Accessibility.enable") return {};
if (method === "Accessibility.getFullAXTree") return { nodes };
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 } };
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
};
}
if (method === "DOMSnapshot.enable") return {};
if (method === "DOMSnapshot.captureSnapshot") throw new Error("snapshot unsupported");
@@ -2568,7 +2643,11 @@ describe("handleSnapshot", () => {
if (method === "Accessibility.enable") return {};
if (method === "Accessibility.getFullAXTree") return { nodes: [root, button] };
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 } };
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
};
}
if (method === "DOMSnapshot.enable") return {};
if (method === "DOMSnapshot.captureSnapshot") {
@@ -2659,7 +2738,11 @@ describe("handleSnapshot", () => {
if (method === "Accessibility.enable") return {};
if (method === "Accessibility.getFullAXTree") return { nodes: [root, button] };
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 } };
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
};
}
if (method === "DOMSnapshot.enable") return {};
if (method === "DOMSnapshot.captureSnapshot") {
@@ -2753,7 +2836,11 @@ describe("handleSnapshot", () => {
if (method === "Accessibility.enable") return {};
if (method === "Accessibility.getFullAXTree") return { nodes: [root, button] };
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 } };
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
};
}
if (method === "DOMSnapshot.enable") return {};
if (method === "DOMSnapshot.captureSnapshot") {
@@ -2918,7 +3005,11 @@ describe("handleSnapshot", () => {
};
}
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } };
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 },
};
}
if (method === "DOMSnapshot.enable") return {};
if (method === "DOMSnapshot.captureSnapshot") {
@@ -3131,10 +3222,14 @@ describe("handleSnapshot", () => {
}
const VP_METRICS = {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
};
function makeFrameAwareDeps() {
function makeFrameAwareDeps(
ownerGeometry: "unavailable" | "available" | "clipped" = "unavailable",
) {
const strings = ["html", "body", "iframe", "button", "title", "Remote", "static", "auto"];
const i = (value: string) => strings.indexOf(value);
const styles = [i("static"), i("auto"), i("auto")];
@@ -3142,6 +3237,8 @@ describe("handleSnapshot", () => {
strings,
documents: [
{
scrollOffsetX: 0,
scrollOffsetY: 0,
frameId: "main",
nodes: {
parentIndex: [-1, 0, 1],
@@ -3162,6 +3259,8 @@ describe("handleSnapshot", () => {
},
},
{
scrollOffsetX: 0,
scrollOffsetY: 0,
frameId: "child",
nodes: {
parentIndex: [-1, 0, 1],
@@ -3214,6 +3313,16 @@ describe("handleSnapshot", () => {
];
const send = vi.fn(async (_tabId: number, method: string) => {
if (method === "Page.getLayoutMetrics") return VP_METRICS;
if (ownerGeometry !== "unavailable") {
if (method === "DOM.getBoxModel") {
const x = ownerGeometry === "clipped" ? 10000 : 100;
return { model: { content: [x, 100, x + 400, 100, x + 400, 400, x, 400] } };
}
if (method === "DOM.resolveNode") return { object: { objectId: "owner" } };
if (method === "Runtime.callFunctionOn")
return { result: { value: { width: 400, height: 300 } } };
if (method === "Runtime.releaseObject") return {};
}
if (method === "DOMSnapshot.enable" || method === "Accessibility.enable") return {};
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
if (method === "Accessibility.getFullAXTree") return { nodes: mainAx };
@@ -3454,6 +3563,43 @@ describe("handleSnapshot", () => {
expect(result.text).toContain("•••");
});
it.each([
undefined,
100,
0,
])("reports unavailable iframe geometry within the observation budget %s", async (maxTokens) => {
const result = await captureVomObservation(makeFrameAwareDeps().cdp, 4, "https://example.com", {
maxTokens,
});
expect(result.text.match(/@warning/g)).toHaveLength(1);
expect(result.text).toContain("geometry incomplete");
expect(result.text.startsWith("@vom 1\n")).toBe(true);
if (maxTokens === undefined) {
expect(result.truncated).toBe(false);
expect(result.text).toContain("Frame action");
expect(result.matchNodes.find((node) => node.backendNodeId === 22)).toMatchObject({
rect: null,
localRect: { x: 20, y: 30, w: 120, h: 40 },
});
}
if (maxTokens === 100) expect(Math.ceil(result.text.length / 4)).toBeLessThanOrEqual(maxTokens);
});
it.each([
"available",
"clipped",
] as const)("does not report successful %s geometry as an observation failure", async (ownerGeometry) => {
const result = await captureVomObservation(
makeFrameAwareDeps(ownerGeometry).cdp,
4,
"https://example.com",
);
expect(result.text).not.toContain("@warning");
const node = result.matchNodes.find((item) => item.backendNodeId === 22);
if (ownerGeometry === "clipped") expect(node?.rect).toBeNull();
else expect(node?.rect).toEqual({ x: 120, y: 130, w: 120, h: 40 });
});
it("keeps frames and rendered refs on the same frame identity", async () => {
const deps = makeFrameAwareDeps();
@@ -1,8 +1,26 @@
import { describe, expect, it, vi } from "vitest";
import { SessionManager } from "@/session-manager/manager";
import { handleSessionStop } from "../session";
import type { ChromeTabsApi } from "../shared";
import { type AgentOverlayResetApi, type ChromeWindowsApi, type TabMutationApi } from "../tabs";
/** Build a read-only query api over a FakeState. */
function makeQuery(state: FakeState): ChromeTabsApi {
return {
get: vi.fn(async (id) => {
const t = state.tabs.get(id);
if (!t) throw new Error(`tab ${id} not found`);
return t;
}),
query: vi.fn(async (q: chrome.tabs.QueryInfo) => {
const w = q.windowId;
return Array.from(state.tabs.values()).filter(
(t) => typeof w !== "number" || t.windowId === w,
);
}),
};
}
function fakeAgentWindow(ids: number[]) {
let i = 0;
const create = vi.fn(async () => {
@@ -11,7 +29,7 @@ function fakeAgentWindow(ids: number[]) {
return id;
});
const remove = vi.fn(async () => {});
const ensureActiveTab = vi.fn(async () => {});
const ensureActiveTab = vi.fn(async () => 0);
return { create, remove, ensureActiveTab };
}
@@ -30,7 +48,9 @@ function makeApis(
} {
const tabs: TabMutationApi = {
create: vi.fn(),
remove: vi.fn(async () => {}),
remove: vi.fn(async (id: number) => {
state.tabs.delete(id);
}),
update: vi.fn(async (_id, _p) => undefined),
get: vi.fn(async (id) => {
const t = state.tabs.get(id);
@@ -237,11 +257,15 @@ describe("handleSessionStop with auto-return", () => {
moves: [],
};
const { tabs, windows } = makeApis(state, { moveThrowsFor: new Set([1]) });
const cdp = {
detachSession: vi.fn(async () => {}),
releaseSessionTab: vi.fn(async () => {}),
};
const res = await handleSessionStop(
sm,
{ session_id: "aa11" },
{ tabManagement: { tabs, windows } },
{ cdp, tabManagement: { tabs, windows } },
);
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
expect(res.return_failures?.map((f) => f.tab_id)).toEqual([1]);
@@ -249,6 +273,8 @@ describe("handleSessionStop with auto-return", () => {
expect(sm.has("aa11")).toBe(true);
expect(ctx.borrowedTabs.has(1)).toBe(true);
expect(ctx.borrowedTabs.has(2)).toBe(false);
expect(cdp.releaseSessionTab).toHaveBeenCalledExactlyOnceWith("aa11", 2);
expect(cdp.detachSession).not.toHaveBeenCalled();
expect(aw.remove).not.toHaveBeenCalled();
});
@@ -267,4 +293,275 @@ describe("handleSessionStop with auto-return", () => {
await handleSessionStop(sm, { session_id: "aa11" }, { tabManagement: { tabs, windows } });
expect(ctx.refStore.isEmpty()).toBe(true);
});
it.each([
"before teardown",
"during its return",
"after its return failed",
"before its turn in the return loop",
])("finishes stopping when a borrowed tab closes %s", async (timing) => {
const aw = fakeAgentWindow([100]);
const sm = new SessionManager({ agentWindow: aw });
const ctx = await sm.start("aa11");
ctx.borrowedTabs.set(1, { tabId: 1, originalWindowId: 200, originalIndex: 0 });
ctx.borrowedTabs.set(2, { tabId: 2, originalWindowId: 200, originalIndex: 1 });
const state: FakeState = {
tabs: new Map([
[1, { id: 1, windowId: 100 } as chrome.tabs.Tab],
[2, { id: 2, windowId: 100 } as chrome.tabs.Tab],
]),
windowsClosed: new Set(),
moves: [],
};
const { tabs, windows } = makeApis(state);
const cdp = { detachSession: vi.fn(async () => {}) };
const closeTab = (tabId: number) => {
state.tabs.delete(tabId);
sm.forgetClosedTab(tabId);
};
const baseMove = tabs.move;
tabs.move = vi.fn(async (tabId, props) => {
if (tabId === 1 && timing === "during its return") closeTab(1);
if (tabId === 1 && timing === "after its return failed") {
throw new Error("simulated move failure");
}
if (tabId === 2 && timing === "after its return failed") closeTab(1);
if (tabId === 1 && timing === "before its turn in the return loop") closeTab(2);
if (!state.tabs.has(tabId)) throw new Error(`tab ${tabId} not found`);
return baseMove(tabId, props);
});
if (timing === "before teardown") closeTab(1);
const res = await handleSessionStop(
sm,
{ session_id: "aa11" },
{
cdp,
tabManagement: {
tabs,
windows,
agentOverlayReset: { resetAgentOverlays: vi.fn(async () => {}) },
},
tabsQuery: makeQuery(state),
},
);
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
const closedTabId = timing === "before its turn in the return loop" ? 2 : 1;
expect(res.returned_tab_ids).toEqual([closedTabId === 1 ? 2 : 1]);
expect(res.return_failures).toBeUndefined();
expect(ctx.borrowedTabs.size).toBe(0);
expect(sm.has("aa11")).toBe(false);
expect(cdp.detachSession).toHaveBeenCalledWith("aa11");
expect(aw.remove).toHaveBeenCalledWith(100);
if (timing === "before teardown" || timing === "before its turn in the return loop") {
expect(tabs.move).not.toHaveBeenCalledWith(closedTabId, expect.anything());
}
});
});
describe("handleSessionStop window release (issue #57)", () => {
const agentWindowId = 100;
it("releases the window when user-created tabs remain", async () => {
const aw = fakeAgentWindow([agentWindowId]);
// ensureActiveTab returns the home tab id (10). We override the fake to
// return a known id.
aw.ensureActiveTab = vi.fn(async () => 10);
const sm = new SessionManager({ agentWindow: aw });
const ctx = await sm.start("aa11");
ctx.agentCreatedTabs.add(11);
ctx.agentCreatedTabs.add(12);
// user tab 99 (created via Chrome UI, NOT in agentCreatedTabs)
const state: FakeState = {
tabs: new Map([
[10, { id: 10, windowId: agentWindowId } as chrome.tabs.Tab],
[11, { id: 11, windowId: agentWindowId } as chrome.tabs.Tab],
[12, { id: 12, windowId: agentWindowId } as chrome.tabs.Tab],
[99, { id: 99, windowId: agentWindowId } as chrome.tabs.Tab],
]),
windowsClosed: new Set(),
moves: [],
};
const { tabs, windows } = makeApis(state);
const query = makeQuery(state);
const res = await handleSessionStop(
sm,
{ session_id: "aa11" },
{ tabManagement: { tabs, windows }, tabsQuery: query },
);
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
expect(res.window_released).toBe(true);
// agent tabs + home removed, user tab 99 kept
expect((tabs.remove as ReturnType<typeof vi.fn>).mock.calls.map((c) => c[0]).sort()).toEqual([
10, 11, 12,
]);
expect(state.tabs.has(99)).toBe(true);
// window released (dropOnly), not closed
expect(aw.remove).not.toHaveBeenCalled();
expect(sm.has("aa11")).toBe(false);
});
it("closes the window (not release) when no user tabs remain", async () => {
const aw = fakeAgentWindow([agentWindowId]);
aw.ensureActiveTab = vi.fn(async () => 10);
const sm = new SessionManager({ agentWindow: aw });
const ctx = await sm.start("aa11");
ctx.agentCreatedTabs.add(11);
const state: FakeState = {
tabs: new Map([
[10, { id: 10, windowId: agentWindowId } as chrome.tabs.Tab],
[11, { id: 11, windowId: agentWindowId } as chrome.tabs.Tab],
]),
windowsClosed: new Set(),
moves: [],
};
const { tabs, windows } = makeApis(state);
const query = makeQuery(state);
const res = await handleSessionStop(
sm,
{ session_id: "aa11" },
{ tabManagement: { tabs, windows }, tabsQuery: query },
);
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
expect(res.window_released).toBeFalsy();
expect(aw.remove).toHaveBeenCalledWith(agentWindowId);
expect(sm.has("aa11")).toBe(false);
});
it("is non-fatal when an agent tab is already gone", async () => {
const aw = fakeAgentWindow([agentWindowId]);
aw.ensureActiveTab = vi.fn(async () => 10);
const sm = new SessionManager({ agentWindow: aw });
const ctx = await sm.start("aa11");
ctx.agentCreatedTabs.add(11); // no longer in state → remove throws
const state: FakeState = {
tabs: new Map([[10, { id: 10, windowId: agentWindowId } as chrome.tabs.Tab]]),
windowsClosed: new Set(),
moves: [],
};
const { tabs, windows } = makeApis(state);
(tabs.remove as ReturnType<typeof vi.fn>).mockImplementation(async (id: number) => {
if (id === 11) throw new Error("tab 11 already closed");
state.tabs.delete(id); // home (10) removed normally
});
const query = makeQuery(state);
const res = await handleSessionStop(
sm,
{ session_id: "aa11" },
{ tabManagement: { tabs, windows }, tabsQuery: query },
);
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
// user tab 10? no — 10 is home, removed. window would be empty → close.
expect(res.window_released).toBeFalsy();
expect(aw.remove).toHaveBeenCalled();
});
it("degrades to close-window when tabsQuery is missing", async () => {
const aw = fakeAgentWindow([agentWindowId]);
aw.ensureActiveTab = vi.fn(async () => 10);
const sm = new SessionManager({ agentWindow: aw });
const ctx = await sm.start("aa11");
ctx.agentCreatedTabs.add(11);
const state: FakeState = {
tabs: new Map([
[10, { id: 10, windowId: agentWindowId } as chrome.tabs.Tab],
[11, { id: 11, windowId: agentWindowId } as chrome.tabs.Tab],
]),
windowsClosed: new Set(),
moves: [],
};
const { tabs, windows } = makeApis(state);
const res = await handleSessionStop(
sm,
{ session_id: "aa11" },
{ tabManagement: { tabs, windows } }, // no tabsQuery
);
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
// No query available → conservatively close the window.
expect(res.window_released).toBeFalsy();
expect(aw.remove).toHaveBeenCalledWith(agentWindowId);
});
it("degrades to close-window when tabsQuery.query throws", async () => {
const aw = fakeAgentWindow([agentWindowId]);
aw.ensureActiveTab = vi.fn(async () => 10);
const sm = new SessionManager({ agentWindow: aw });
const ctx = await sm.start("aa11");
ctx.agentCreatedTabs.add(11);
const state: FakeState = {
tabs: new Map([[11, { id: 11, windowId: agentWindowId } as chrome.tabs.Tab]]),
windowsClosed: new Set(),
moves: [],
};
const { tabs, windows } = makeApis(state);
const query = makeQuery(state);
(query.query as ReturnType<typeof vi.fn>).mockImplementation(async () => {
throw new Error("query failed");
});
const res = await handleSessionStop(
sm,
{ session_id: "aa11" },
{ tabManagement: { tabs, windows }, tabsQuery: query },
);
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
expect(res.window_released).toBeFalsy();
expect(aw.remove).toHaveBeenCalled();
});
it("tracks agentCreatedTabs across tab_create / tab_close", async () => {
const aw = fakeAgentWindow([agentWindowId]);
aw.ensureActiveTab = vi.fn(async () => 10);
const sm = new SessionManager({ agentWindow: aw });
const ctx = await sm.start("aa11");
// Simulate handleTabCreate adding, then handleTabClose removing.
ctx.agentCreatedTabs.add(11);
expect(ctx.agentCreatedTabs.has(11)).toBe(true);
ctx.agentCreatedTabs.delete(11); // as handleTabClose does on success
expect(ctx.agentCreatedTabs.has(11)).toBe(false);
// Unknown id delete is a safe no-op (user-closed tab).
expect(() => ctx.agentCreatedTabs.delete(999)).not.toThrow();
});
it("closes the window (not release) when an agent tab fails to close", async () => {
// Regression: a leaked agent tab (still in agentCreatedTabs, but
// still present because Step 4 remove() threw) must NOT be mistaken
// for a user tab. Otherwise the window would be released (dropOnly)
// and the agent tab would leak (issue #57 regression).
const aw = fakeAgentWindow([agentWindowId]);
aw.ensureActiveTab = vi.fn(async () => 10);
const sm = new SessionManager({ agentWindow: aw });
const ctx = await sm.start("aa11");
ctx.agentCreatedTabs.add(11); // Step 4 remove() will throw for this tab
const state: FakeState = {
tabs: new Map([
[10, { id: 10, windowId: agentWindowId } as chrome.tabs.Tab],
[11, { id: 11, windowId: agentWindowId } as chrome.tabs.Tab],
]),
windowsClosed: new Set(),
moves: [],
};
const { tabs, windows } = makeApis(state);
(tabs.remove as ReturnType<typeof vi.fn>).mockImplementation(async (id: number) => {
if (id === 11) throw new Error("tab 11 failed to close");
state.tabs.delete(id); // home (10) removed normally; 11 lingers
});
const query = makeQuery(state);
const res = await handleSessionStop(
sm,
{ session_id: "aa11" },
{ tabManagement: { tabs, windows }, tabsQuery: query },
);
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
// The leaked agent tab must NOT keep the window open.
expect(res.window_released).toBeFalsy();
expect(aw.remove).toHaveBeenCalledWith(agentWindowId);
expect(sm.has("aa11")).toBe(false);
});
});
@@ -12,7 +12,7 @@ function fakeAgentWindow() {
return {
create: vi.fn(async () => 100),
remove: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
};
}
@@ -0,0 +1,237 @@
// @vitest-environment node
import { describe, expect, it } from "vitest";
import type { CdpFrame, CdpFrameGraph, CdpTarget } from "@/browser-driver/frame-graph";
import { GeometryContext } from "../geometry/frame-context";
import type { CdpRunner } from "../shared";
import { captureViewModel } from "../vom/capture";
import { captureFrameData } from "../vom/frame-capture";
type Send = <T = Record<string, unknown>>(
method: string,
params?: object,
sessionId?: string,
) => Promise<T>;
type Tree = { frame: { id: string; parentId?: string; name?: string }; childFrames?: Tree[] };
type Rect = { x: number; y: number; w: number; h: number };
type Oracle = { probe: Rect; owners: Record<string, { x: number; y: number; scale: number }> };
// The independent oracle uses DOM border boxes and this fixture's axis-aligned
// iframe transforms. No production snapshot conversion/projection builds expectations.
const oracleExpression = `(() => {
const box = document.querySelector('#probe').getBoundingClientRect();
const owners = {};
for (const frame of document.querySelectorAll('iframe')) {
const rect = frame.getBoundingClientRect();
const style = getComputedStyle(frame);
const scale = rect.width / frame.offsetWidth;
owners[frame.name] = {
x: rect.x + (frame.clientLeft + parseFloat(style.paddingLeft)) * scale,
y: rect.y + (frame.clientTop + parseFloat(style.paddingTop)) * scale,
scale,
};
}
return { probe: { x: box.x, y: box.y, w: box.width, h: box.height }, owners };
})()`;
describe.skipIf(!process.env.BSK_GEOMETRY_CHROME)("real DOMSnapshot coordinate contract", () => {
it.each([
{ deviceScale: 1, zoom: 1 },
{ deviceScale: 0.8, zoom: 1 },
{ deviceScale: 1, zoom: 1.25 },
{ deviceScale: 2, zoom: 1 },
{ deviceScale: 2, zoom: 0.8 },
])("matches DOM geometry at device scale $deviceScale and zoom $zoom", async (configuration) => {
const evalRoot = new URL("../../../../../evals/browser/", import.meta.url);
const { createEvalServer } = await import(new URL("lib/server.mjs", evalRoot).href);
const { withChrome } = await import(
new URL("cases/regression/snapshot-coordinates/chrome.mjs", evalRoot).href
);
const server = createEvalServer();
const { baseUrl } = await server.start();
try {
await withChrome(
{ executable: process.env.BSK_GEOMETRY_CHROME, ...configuration },
async (send: Send) => {
const { targetId } = await send<{ targetId: string }>("Target.createTarget", {
url: "about:blank",
});
const { sessionId: rootSession } = await send<{ sessionId: string }>(
"Target.attachToTarget",
{ targetId, flatten: true },
);
await send(
"Page.navigate",
{ url: `${baseUrl}/snapshot-coordinates?run=coordinates` },
rootSession,
);
await expect
.poll(
() =>
server
.snapshot("coordinates")
.events.some(
(event: { type: string; data: { root?: boolean } }) =>
event.type === "geometry.ready" && event.data.root,
),
{ timeout: 10_000 },
)
.toBe(true);
const targets = await send<{ targetInfos: { targetId: string; type: string }[] }>(
"Target.getTargets",
);
const sessions = [rootSession];
for (const target of targets.targetInfos.filter((target) => target.type === "iframe")) {
const { sessionId } = await send<{ sessionId: string }>("Target.attachToTarget", {
targetId: target.targetId,
flatten: true,
});
sessions.push(sessionId);
}
expect(sessions.length).toBe(2); // The cross-site fixture must actually be an OOPIF.
const frames: CdpFrame[] = [];
const names = new Map<string, string>();
const sessionFor = (target: CdpTarget) => target.sessionId ?? rootSession;
for (const sessionId of sessions) {
const { frameTree } = await send<{ frameTree: Tree }>(
"Page.getFrameTree",
{},
sessionId,
);
const target: CdpTarget = {
tabId: 1,
...(sessionId === rootSession ? {} : { sessionId }),
};
const visit = (tree: Tree, parentFrameId = tree.frame.parentId) => {
frames.push({ frameId: tree.frame.id, parentFrameId, target });
names.set(tree.frame.id, tree.frame.name ?? "");
for (const child of tree.childFrames ?? []) visit(child, tree.frame.id);
};
visit(frameTree);
}
expect(frames).toHaveLength(5);
for (const frame of frames) {
if (!frame.parentFrameId) continue;
const parent = frames.find((item) => item.frameId === frame.parentFrameId)!;
const owner = await send<{ backendNodeId: number }>(
"DOM.getFrameOwner",
{ frameId: frame.frameId },
sessionFor(parent.target),
);
frame.ownerBackendNodeId = owner.backendNodeId;
}
const graph: CdpFrameGraph = { rootFrameId: frames[0].frameId, frames };
const calls: string[] = [];
const cdp: CdpRunner = {
send: (tabId, method, params) => {
calls.push(`${tabId}:${method}`);
return send(method, params, rootSession);
},
sendToTarget: (target, method, params) => {
calls.push(`${target.sessionId ?? target.tabId}:${method}`);
return send(method, params, sessionFor(target));
},
getFrameGraph: async () => graph,
};
const oracles = new Map<string, Oracle>();
for (const frame of frames) {
const session = sessionFor(frame.target);
const { executionContextId } = await send<{ executionContextId: number }>(
"Page.createIsolatedWorld",
{ frameId: frame.frameId, worldName: "coordinate-oracle" },
session,
);
await expect
.poll(
async () => {
const ready = await send<{ result: { value: boolean } }>(
"Runtime.evaluate",
{
expression: 'document.documentElement.dataset.geometryReady === "true"',
contextId: executionContextId,
returnByValue: true,
},
session,
);
return ready.result.value;
},
{ timeout: 5_000 },
)
.toBe(true);
// Establish scroll after cross-process target creation/layout settles.
const scroll = frame.parentFrameId
? names.get(frame.frameId) === "nested"
? [10, 20]
: [40, 100]
: [80, 240];
await send(
"Runtime.evaluate",
{
expression: `(async () => { scrollTo(${scroll.join(",")}); await new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve))); })()`,
contextId: executionContextId,
awaitPromise: true,
},
session,
);
const reply = await send<{ result: { value: Oracle } }>(
"Runtime.evaluate",
{ expression: oracleExpression, contextId: executionContextId, returnByValue: true },
session,
);
oracles.set(frame.frameId, reply.result.value);
}
const geometry = new GeometryContext(cdp, 1, graph);
const captured = await captureViewModel(cdp, 1, { geometry });
await captureFrameData(cdp, 1, captured, undefined, geometry);
expect(captured.frameGeometryIssues).toEqual([]);
for (const frame of frames) {
const node = captured.frameNodes
?.get(frame.frameId)
?.find((node) => node.attrs.id === "probe");
expect(node, `missing probe in ${names.get(frame.frameId) || "root"}`).toBeDefined();
const local = oracles.get(frame.frameId)!.probe;
const top = { ...local };
let current = frame;
while (current.parentFrameId) {
const owner = oracles.get(current.parentFrameId)!.owners[names.get(current.frameId)!];
top.x = owner.x + top.x * owner.scale;
top.y = owner.y + top.y * owner.scale;
top.w *= owner.scale;
top.h *= owner.scale;
current = frames.find((item) => item.frameId === current.parentFrameId)!;
}
expect(
node!.rect,
JSON.stringify({
frame: names.get(frame.frameId),
local,
top,
viewport: captured.viewport,
}),
).not.toBeNull();
for (const key of ["x", "y", "w", "h"] as const) {
expect(
Math.abs(node!.localRect![key] - local[key]),
`${names.get(frame.frameId)} local ${key}`,
).toBeLessThan(2);
expect(
Math.abs(node!.rect![key] - top[key]),
`${names.get(frame.frameId)} top ${key}`,
).toBeLessThan(2);
}
}
expect(calls.filter((call) => call.endsWith(":Page.getLayoutMetrics"))).toHaveLength(
sessions.length,
);
const metrics = await send<{
cssVisualViewport: { zoom: number };
visualViewport: { clientWidth: number };
}>("Page.getLayoutMetrics", {}, rootSession);
expect(metrics.cssVisualViewport.zoom).toBeCloseTo(configuration.zoom, 4);
},
);
} finally {
await server.stop();
}
}, 30_000);
});
@@ -11,7 +11,7 @@ function fakeAgentWindow(ids: number[]) {
return id;
},
remove: async () => {},
ensureActiveTab: async () => {},
ensureActiveTab: async () => 1,
};
}
+181 -1
View File
@@ -1,5 +1,7 @@
import { describe, expect, it, vi } from "vitest";
import { type CdpDebuggerApi, ChromiumCdp } from "@/browser-driver/chromium-cdp";
import { SessionManager } from "@/session-manager/manager";
import { handleHover } from "../interaction";
import {
type AgentOverlayResetApi,
type ChromeWindowsApi,
@@ -23,7 +25,7 @@ function fakeAgentWindow(ids: number[]) {
return id;
}),
remove: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
};
}
@@ -221,6 +223,7 @@ describe("handleTabCreate", () => {
active: true,
});
expect(res).toMatchObject({ tab_id: 10, window_id: 100 });
expect(sm.get("aa11")?.agentCreatedTabs.has(10)).toBe(true);
});
it("forwards explicit url + index + active=false", async () => {
@@ -256,6 +259,41 @@ describe("handleTabCreate", () => {
const res = await handleTabCreate(sm, { session_id: "aa11", index: -1 }, { tabs: api });
expect(res).toMatchObject({ code: "invalid_params" });
});
it("does not claim any tab when chrome.tabs.create fails", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
await sm.start("aa11");
const state: FakeTabState = { tabs: new Map(), nextTabId: 10, windowsClosed: new Set() };
const { api, spies } = makeTabMutationApi(state);
spies.create.mockRejectedValueOnce(new Error("chrome.tabs.create failed"));
const res = await handleTabCreate(sm, { session_id: "aa11" }, { tabs: api });
expect(res).toMatchObject({ code: "protocol_error" });
expect(sm.get("aa11")?.agentCreatedTabs).toEqual(new Set([1]));
});
it("claims the returned id even when a user tab appears during tab creation", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
const state: FakeTabState = { tabs: new Map(), nextTabId: 10, windowsClosed: new Set() };
const { api, spies } = makeTabMutationApi(state);
let resolveCreate!: (tab: chrome.tabs.Tab) => void;
const createResult = new Promise<chrome.tabs.Tab>((resolve) => {
resolveCreate = resolve;
});
spies.create.mockImplementationOnce(() => createResult);
const pending = handleTabCreate(sm, { session_id: "aa11" }, { tabs: api });
// A user can create an unrelated same-window tab while Chrome is still
// resolving the agent request. There is no pending slot for it to steal.
expect(ctx.agentCreatedTabs.has(99)).toBe(false);
resolveCreate({ id: 10, windowId: 100, active: true } as chrome.tabs.Tab);
await expect(pending).resolves.toMatchObject({ tab_id: 10 });
expect(ctx.agentCreatedTabs.has(10)).toBe(true);
expect(ctx.agentCreatedTabs.has(99)).toBe(false);
});
});
describe("handleTabClose", () => {
@@ -511,6 +549,148 @@ describe("handleTabBorrow", () => {
});
describe("handleTabReturn", () => {
it.each([
"not_found",
"cdp_failed",
])("detaches after selector resolution fails with %s", async (code) => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
ctx.borrowedTabs.set(7, { tabId: 7, originalWindowId: 200, originalIndex: 4 });
const state: FakeTabState = {
tabs: new Map([[7, { id: 7, windowId: 100 } as chrome.tabs.Tab]]),
nextTabId: 50,
windowsClosed: new Set(),
};
const { api } = makeTabMutationApi(state);
const { api: windows } = makeWindowsApi(state);
const debuggerApi = {
attach: vi.fn(async () => {}),
detach: vi.fn(async () => {}),
sendCommand: vi.fn(async (_target: unknown, method: string) => {
if (method === "DOM.getDocument") return { root: { nodeId: 1 } };
if (method === "DOM.querySelector") {
if (code === "cdp_failed") throw new Error("selector query failed");
return { nodeId: 0 };
}
return {};
}),
onEvent: { addListener: vi.fn(), removeListener: vi.fn() },
onDetach: { addListener: vi.fn(), removeListener: vi.fn() },
};
const cdp = new ChromiumCdp(debuggerApi as unknown as CdpDebuggerApi);
const hover = await handleHover(
sm,
{ session_id: "aa11", tab_id: 7, selector: "#missing" },
{
cdp,
tabsApi: { get: api.get, query: vi.fn(async () => []) },
},
);
expect(hover).toMatchObject({ code });
expect(cdp.isAttached(7)).toBe(true);
const returned = await handleTabReturn(
sm,
{ session_id: "aa11", tab_id: 7 },
{
tabs: api,
windows,
cdp,
agentOverlayReset: { resetAgentOverlays: vi.fn(async () => {}) },
},
);
expect(returned).toMatchObject({ returned_to_window_id: 200 });
expect(ctx.borrowedTabs.has(7)).toBe(false);
expect(cdp.isAttached(7)).toBe(false);
expect(debuggerApi.detach).toHaveBeenCalledExactlyOnceWith({ tabId: 7 });
cdp.dispose();
});
it("honours cancellation while preparing a validated return", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
ctx.borrowedTabs.set(7, { tabId: 7, originalWindowId: 200, originalIndex: 4 });
const state: FakeTabState = {
tabs: new Map([[7, { id: 7, windowId: 100 } as chrome.tabs.Tab]]),
nextTabId: 50,
windowsClosed: new Set(),
};
const { api, spies } = makeTabMutationApi(state);
const { api: windows } = makeWindowsApi(state);
const controller = new AbortController();
const beforeReturn = vi.fn(async () => {
controller.abort();
});
const result = await handleTabReturn(
sm,
{ session_id: "aa11", tab_id: 7 },
{
tabs: api,
windows,
signal: controller.signal,
beforeReturn,
},
);
expect(result).toMatchObject({ code: "cancelled" });
expect(beforeReturn).toHaveBeenCalledExactlyOnceWith("aa11", 7);
expect(spies.move).not.toHaveBeenCalled();
expect(ctx.borrowedTabs.has(7)).toBe(true);
});
it.each([
"original",
"fallback",
"cleanup_failed",
"failed",
"cancelled",
])("releases CDP only after a successful return (%s)", async (mode) => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
ctx.borrowedTabs.set(7, { tabId: 7, originalWindowId: 200, originalIndex: 4 });
const state: FakeTabState = {
tabs: new Map([[7, { id: 7, windowId: 100 } as chrome.tabs.Tab]]),
nextTabId: 50,
windowsClosed: new Set(),
};
const { api, spies } = makeTabMutationApi(state);
const { api: windowsApi } = makeWindowsApi(state, { lastFocused: 500 });
const success = mode !== "failed" && mode !== "cancelled";
if (mode === "fallback") spies.move.mockRejectedValueOnce(new Error("original move failed"));
if (mode === "failed") spies.move.mockRejectedValue(new Error("move failed"));
let windowAtRelease: number | undefined;
const cdp = {
releaseSessionTab: vi.fn(async () => {
windowAtRelease = state.tabs.get(7)?.windowId;
if (mode === "cleanup_failed") throw new Error("debugger release failed");
}),
};
const controller = new AbortController();
if (mode === "cancelled") controller.abort();
const result = await handleTabReturn(
sm,
{ session_id: "aa11", tab_id: 7 },
{
tabs: api,
windows: windowsApi,
cdp,
signal: controller.signal,
agentOverlayReset: { resetAgentOverlays: vi.fn(async () => {}) },
},
);
expect("code" in result).toBe(!success);
expect(ctx.borrowedTabs.has(7)).toBe(!success);
expect(cdp.releaseSessionTab).toHaveBeenCalledTimes(success ? 1 : 0);
if (success) {
expect(cdp.releaseSessionTab).toHaveBeenCalledWith("aa11", 7);
expect(windowAtRelease).toBe(mode === "fallback" ? 500 : 200);
expect(spies.move).toHaveBeenCalledTimes(mode === "fallback" ? 2 : 1);
}
});
it("returns the tab to its original window/index and clears borrowedTabs", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
@@ -12,7 +12,7 @@ function fakeAgentWindow(ids: number[]) {
return id;
}),
remove: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => 1),
};
}
@@ -11,7 +11,7 @@ function fakeAgentWindow(ids: number[]) {
return id;
});
const remove = vi.fn(async () => {});
const ensureActiveTab = vi.fn(async () => {});
const ensureActiveTab = vi.fn(async () => 1);
return { create, remove, ensureActiveTab };
}
@@ -0,0 +1,110 @@
import type { RpcError, WaitUntil } from "@/transport/types";
import { cdpError } from "./errors";
interface NavigationEvent {
tabId: number;
frameId: number;
documentId?: string;
error?: string;
}
interface NavigationEvents {
addListener(listener: (details: NavigationEvent) => void): void;
removeListener(listener: (details: NavigationEvent) => void): void;
}
/** Normal browser navigation remains available when chrome.debugger is denied. */
export interface BrowserNavigationApi {
update(tabId: number, props: { url: string }): Promise<unknown>;
reload(tabId: number, props: { bypassCache: boolean }): Promise<void>;
onCommitted: NavigationEvents;
onDOMContentLoaded: NavigationEvents;
onCompleted: NavigationEvents;
onErrorOccurred: NavigationEvents;
}
export const chromeBrowserNavigationApi: BrowserNavigationApi = {
update: (tabId, props) => chrome.tabs.update(tabId, props),
reload: (tabId, props) => chrome.tabs.reload(tabId, props),
get onCommitted() {
return chrome.webNavigation.onCommitted;
},
get onDOMContentLoaded() {
return chrome.webNavigation.onDOMContentLoaded;
},
get onCompleted() {
return chrome.webNavigation.onCompleted;
},
get onErrorOccurred() {
return chrome.webNavigation.onErrorOccurred;
},
};
export type NavigationOutcome =
| { reached: "match" | "timeout" | "cancelled"; lastLifecycle?: string }
| { reached: "failed"; error: RpcError };
/** Subscribe before dispatch and ignore the departing document's load events. */
export async function navigateWithBrowserApi(
api: BrowserNavigationApi,
tabId: number,
action: () => Promise<unknown>,
waitUntil: Exclude<WaitUntil, "networkidle">,
timeoutMs: number,
signal?: AbortSignal,
): Promise<NavigationOutcome> {
if (signal?.aborted) return { reached: "cancelled" };
if (timeoutMs <= 0) return { reached: "timeout" };
let committed = false;
let documentId: string | undefined;
let lastLifecycle: string | undefined;
let finish!: (outcome: NavigationOutcome) => void;
const outcome = new Promise<NavigationOutcome>((resolve) => {
finish = resolve;
});
const isMainFrame = (details: NavigationEvent) =>
details.tabId === tabId && details.frameId === 0;
const onCommitted = (details: NavigationEvent) => {
if (!isMainFrame(details)) return;
committed = true;
documentId = details.documentId;
lastLifecycle = "commit";
if (waitUntil === "commit") finish({ reached: "match", lastLifecycle });
};
const onDOMContentLoaded = (details: NavigationEvent) => {
if (!isMainFrame(details) || !committed || (documentId && details.documentId !== documentId))
return;
lastLifecycle = "DOMContentLoaded";
if (waitUntil === "domcontentloaded") finish({ reached: "match", lastLifecycle });
};
const onCompleted = (details: NavigationEvent) => {
if (!isMainFrame(details) || !committed || (documentId && details.documentId !== documentId))
return;
finish({ reached: "match", lastLifecycle: "load" });
};
const onError = (details: NavigationEvent) => {
if (!isMainFrame(details)) return;
finish({ reached: "failed", error: cdpError(details.error ?? "browser navigation failed") });
};
const onAbort = () => finish({ reached: "cancelled", lastLifecycle });
const subscriptions = [
[api.onCommitted, onCommitted],
[api.onDOMContentLoaded, onDOMContentLoaded],
[api.onCompleted, onCompleted],
[api.onErrorOccurred, onError],
] as const;
for (const [event, listener] of subscriptions) event.addListener(listener);
signal?.addEventListener("abort", onAbort, { once: true });
const timer = setTimeout(() => finish({ reached: "timeout", lastLifecycle }), timeoutMs);
try {
if (signal?.aborted) return { reached: "cancelled" };
await action();
return await outcome;
} catch (error) {
return { reached: "failed", error: cdpError(error) };
} finally {
clearTimeout(timer);
signal?.removeEventListener("abort", onAbort);
for (const [event, listener] of subscriptions) event.removeListener(listener);
}
}
+33 -6
View File
@@ -36,6 +36,7 @@ import { isRequestFrame } from "@/transport/types";
import { handleConsole } from "./console";
import { handleDownload } from "./download";
import { type EmulateCdpRunner, handleEmulate } from "./emulate";
import { classifyCdpError } from "./errors";
import { handleEvaluate } from "./evaluate";
import { handleRequestHelp } from "./human-loop";
import { handleClick, handleFill, handleHover, handlePress, handleSelect } from "./interaction";
@@ -69,6 +70,7 @@ import {
import { chromeTabsApi, lookupSession, resolveTargetTab } from "./shared";
import {
type BorrowConfirmationApprover,
chromeTabMutationApi,
handleTabBorrow,
handleTabClose,
handleTabCreate,
@@ -118,6 +120,8 @@ export interface DispatcherDeps {
onSessionsChanged?: () => void;
/** Invoked before a tool that dispatches page input or mutates browser state is forwarded. */
onBrowserControlResumed?: (sessionId: string) => void;
/** Invoked after a tab is explicitly claimed so its overlay can be refreshed immediately. */
onAgentTabClaimed?: (tabId: number, windowId: number) => void;
/** User approval for `tool.tab_borrow` (overlay in content script). */
approveBorrow?: BorrowConfirmationApprover;
/** i18n notification copy for `tool.request_help` (resolved per-call). */
@@ -147,6 +151,7 @@ export class ToolDispatcher {
private readonly recording?: RecordRuntimeDeps;
private readonly onSessionsChanged?: () => void;
private readonly onBrowserControlResumed?: (sessionId: string) => void;
private readonly onAgentTabClaimed?: (tabId: number, windowId: number) => void;
private readonly approveBorrow?: BorrowConfirmationApprover;
private readonly helpNotificationCopy?: () => { title: string; body: string };
private subscription: { dispose(): void } | null = null;
@@ -167,6 +172,7 @@ export class ToolDispatcher {
this.recording = deps.recording;
this.onSessionsChanged = deps.onSessionsChanged;
this.onBrowserControlResumed = deps.onBrowserControlResumed;
this.onAgentTabClaimed = deps.onAgentTabClaimed;
this.approveBorrow = deps.approveBorrow;
this.helpNotificationCopy = deps.helpNotificationCopy;
}
@@ -234,7 +240,7 @@ export class ToolDispatcher {
if (sessionId) this.onBrowserControlResumed?.(sessionId);
const result = await this.invoke(req, ac.signal);
if (isRpcError(result)) {
body = { id: req.id, error: result };
body = { id: req.id, error: classifyCdpError(result) };
} else {
body = { id: req.id, result };
if (req.method === "tool.session_start") {
@@ -302,13 +308,25 @@ export class ToolDispatcher {
await this.releaseHoverLatch((req.params as SessionStopParams).session_id);
return handleSessionStop(this.sessions, req.params as SessionStopParams, {
cdp: this.cdp,
// Must be wired in production: the agent-tab cleanup and the
// window-release decision (issue #57) read these deps directly
// and silently no-op when they are absent.
tabManagement: { tabs: chromeTabMutationApi },
tabsQuery: chromeTabsApi,
signal,
});
}
case "tool.tab_list":
return handleTabList(this.sessions, req.params as TabListParams, chromeTabsApi, signal);
case "tool.tab_create":
return handleTabCreate(this.sessions, req.params as TabCreateParams, { signal });
case "tool.tab_create": {
const result = await handleTabCreate(this.sessions, req.params as TabCreateParams, {
signal,
});
if (!isRpcError(result)) {
this.onAgentTabClaimed?.(result.tab_id, result.window_id);
}
return result;
}
case "tool.tab_close":
return this.withHoverReleaseForRequest(
req.params as TabCloseParams,
@@ -317,13 +335,22 @@ export class ToolDispatcher {
);
case "tool.tab_select":
return handleTabSelect(this.sessions, req.params as TabSelectParams, { signal });
case "tool.tab_borrow":
return handleTabBorrow(this.sessions, req.params as TabBorrowParams, {
case "tool.tab_borrow": {
const result = await handleTabBorrow(this.sessions, req.params as TabBorrowParams, {
signal,
approveBorrow: this.approveBorrow,
});
if (!isRpcError(result)) {
this.onAgentTabClaimed?.(result.tab_id, result.agent_window_id);
}
return result;
}
case "tool.tab_return":
return handleTabReturn(this.sessions, req.params as TabReturnParams, { signal });
return handleTabReturn(this.sessions, req.params as TabReturnParams, {
signal,
cdp: this.cdp,
beforeReturn: (sessionId, tabId) => this.releaseHoverLatch(sessionId, tabId),
});
case "tool.window_resize":
return handleWindowResize(
this.sessions,
+28
View File
@@ -13,6 +13,34 @@ import type {
export type { RpcErrorData, RpcErrorReason };
/** Chrome checks the whole tab's frame tree, including other extensions' frames. */
export function isCdpExtensionAccessDenied(error: unknown): boolean {
const message = error instanceof Error ? error.message : String(error);
return message.includes("Cannot access a chrome-extension:// URL of different extension");
}
/** Preserve more specific reasons, especially file-transfer outcome/cleanup details. */
export function classifyCdpError(error: RpcError): RpcError {
if (
error.code !== "cdp_failed" ||
error.data?.reason ||
!isCdpExtensionAccessDenied(error.message)
) {
return error;
}
return {
...error,
data: { ...error.data, reason: "cdp_extension_access_denied" },
};
}
export function cdpError(error: unknown): RpcError {
return classifyCdpError({
code: "cdp_failed",
message: error instanceof Error ? error.message : String(error),
});
}
export interface TransferErrorOptions {
effectState: TransferEffectState;
phase: string;
+46 -151
View File
@@ -1,24 +1,21 @@
import { type CdpFrame, type CdpFrameGraph, type CdpTarget } from "@/browser-driver/frame-graph";
import { type CdpTarget, cdpTargetKey } from "@/browser-driver/frame-graph";
import type { RpcError } from "@/transport/types";
import { nodeContentRegion, scrollNodeIntoView } from "./element-geometry";
import {
clipPolygon,
type GeometryProjection,
type Point,
type Polygon,
type ProjectiveEdge,
parseCdpQuad,
polygonArea,
polygonCentroid,
projectRegionToViewport,
type Quad,
type Region,
rectPolygon,
regionBounds,
type Size,
type ViewportRect,
} from "./geometry";
import { type CdpRunner, cdpRunnerForTarget, isRpcError, sendToCdpTarget } from "./shared";
import type { CssViewport } from "./geometry/coordinate-types";
import { cssViewport, GeometryContext } from "./geometry/frame-context";
import { type CdpRunner, cdpRunnerForTarget, isRpcError } from "./shared";
export interface NodeAddress {
target: CdpTarget;
@@ -27,6 +24,8 @@ export interface NodeAddress {
}
export interface ResolvedNodeGeometry {
/** The same top-level measurement used by projection, for screenshot coordinate adaptation. */
topViewport: CssViewport;
topVisibleRegions: Region;
topBounds: ViewportRect;
/** Point in the top-level tab viewport, used by root-target input events. */
@@ -39,145 +38,16 @@ function geometryError(message: string): RpcError {
return { code: "cdp_failed", message };
}
function frameMap(graph: CdpFrameGraph): Map<string, CdpFrame> {
return new Map(graph.frames.map((frame) => [frame.frameId, frame]));
}
function targetRootFrame(byId: Map<string, CdpFrame>, frame: CdpFrame): CdpFrame | null {
const seen = new Set<string>();
let current = frame;
while (current.parentFrameId) {
if (seen.has(current.frameId)) return null;
seen.add(current.frameId);
const parent = byId.get(current.parentFrameId);
if (!parent || parent.target.sessionId !== current.target.sessionId) break;
current = parent;
}
return current;
}
function frameAncestry(graph: CdpFrameGraph, frameId: string): CdpFrame[] | null {
const byId = frameMap(graph);
const path: CdpFrame[] = [];
const seen = new Set<string>();
let current = byId.get(frameId);
if (!current) return null;
while (current.parentFrameId) {
if (seen.has(current.frameId)) return null;
seen.add(current.frameId);
path.push(current);
const parent = byId.get(current.parentFrameId);
if (!parent) return null;
current = parent;
}
return path;
}
async function targetViewport(cdp: CdpRunner, target: CdpTarget): Promise<Size | null> {
const metrics = await sendToCdpTarget<{
cssLayoutViewport?: { clientWidth?: number; clientHeight?: number };
layoutViewport?: { clientWidth?: number; clientHeight?: number };
}>(cdp, target, "Page.getLayoutMetrics", {});
const viewport = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
const width = viewport.clientWidth ?? 0;
const height = viewport.clientHeight ?? 0;
return width > 0 && height > 0 ? { width, height } : null;
}
async function ownerContentQuad(
cdp: CdpRunner,
parent: CdpFrame,
ownerBackendNodeId: number,
): Promise<Quad | null> {
const result = await sendToCdpTarget<{ model?: { content?: number[] } }>(
cdp,
parent.target,
"DOM.getBoxModel",
{ backendNodeId: ownerBackendNodeId },
);
return parseCdpQuad(result.model?.content);
}
async function sameTargetFrameClips(
cdp: CdpRunner,
byId: Map<string, CdpFrame>,
frame: CdpFrame,
targetRoot: CdpFrame,
): Promise<Polygon[] | null> {
const clips: Polygon[] = [];
let current = frame;
const seen = new Set<string>();
while (current.frameId !== targetRoot.frameId) {
if (seen.has(current.frameId) || !current.parentFrameId) return null;
seen.add(current.frameId);
const parent = byId.get(current.parentFrameId);
if (!parent || current.ownerBackendNodeId === undefined) return null;
const clip = await ownerContentQuad(cdp, parent, current.ownerBackendNodeId);
if (!clip) return null;
clips.push(clip);
current = parent;
}
return clips;
}
export async function resolveFrameProjection(
cdp: CdpRunner,
graph: CdpFrameGraph,
frameId: string,
): Promise<GeometryProjection | null> {
const byId = frameMap(graph);
const frame = byId.get(frameId);
if (!frame) return null;
let targetRoot = targetRootFrame(byId, frame);
if (!targetRoot) return null;
const sourceViewport = await targetViewport(cdp, targetRoot.target);
if (!sourceViewport) return null;
const sourceClips = await sameTargetFrameClips(cdp, byId, frame, targetRoot);
if (!sourceClips) return null;
const edges: ProjectiveEdge[] = [];
while (targetRoot.parentFrameId) {
const parent = byId.get(targetRoot.parentFrameId);
if (!parent || targetRoot.ownerBackendNodeId === undefined) return null;
const destinationQuad = await ownerContentQuad(cdp, parent, targetRoot.ownerBackendNodeId);
if (!destinationQuad) return null;
const source =
edges.length === 0 ? sourceViewport : await targetViewport(cdp, targetRoot.target);
if (!source) return null;
const parentTargetRoot = targetRootFrame(byId, parent);
if (!parentTargetRoot) return null;
const destinationClips = await sameTargetFrameClips(cdp, byId, parent, parentTargetRoot);
if (!destinationClips) return null;
edges.push({ sourceViewport: source, destinationQuad, destinationClips });
targetRoot = parentTargetRoot;
}
const topViewport =
edges.length === 0 ? sourceViewport : await targetViewport(cdp, targetRoot.target);
return topViewport ? { sourceClips, edges, topViewport } : null;
}
async function loadFrameGraph(cdp: CdpRunner, tabId: number): Promise<CdpFrameGraph | null> {
if (!cdp.getFrameGraph) return null;
try {
return await cdp.getFrameGraph(tabId);
} catch (error) {
console.debug("[bsk frame-geometry] frame graph resolution failed", error);
return null;
}
}
async function scrollFrameOwners(
cdp: CdpRunner,
tabId: number,
graph: CdpFrameGraph,
context: GeometryContext,
frameId: string,
): Promise<RpcError | null> {
const byId = frameMap(graph);
const ancestry = frameAncestry(graph, frameId);
const ancestry = await context.ancestry(frameId);
if (!ancestry) return geometryError(`could not resolve frame ancestry for ${frameId}`);
for (const child of [...ancestry].reverse()) {
const parent = child.parentFrameId ? byId.get(child.parentFrameId) : undefined;
const parent = child.parentFrameId ? await context.frame(child.parentFrameId) : undefined;
if (!parent || child.ownerBackendNodeId === undefined) {
return geometryError(`could not resolve frame owner for ${child.frameId}`);
}
@@ -191,17 +61,20 @@ async function scrollFrameOwners(
return null;
}
async function scrollElementWithFrameGraph(
async function scrollElementWithContext(
cdp: CdpRunner,
tabId: number,
target: CdpTarget,
backendNodeId: number,
frameId: string | undefined,
graph: CdpFrameGraph | null,
context: GeometryContext,
): Promise<RpcError | null> {
if (frameId) {
if (!graph) return geometryError(`could not resolve frame graph for ${frameId}`);
const error = await scrollFrameOwners(cdp, tabId, graph, frameId);
const frame = await context.frame(frameId);
if (!frame || cdpTargetKey(frame.target) !== cdpTargetKey(target)) {
return geometryError("node frame does not belong to its target");
}
const error = await scrollFrameOwners(cdp, tabId, context, frameId);
if (error) return error;
} else if (target.sessionId) {
return geometryError("an OOPIF node address requires frameId");
@@ -216,8 +89,15 @@ export async function scrollElementAndFramesIntoView(
backendNodeId: number,
frameId?: string,
): Promise<RpcError | null> {
const graph = frameId ? await loadFrameGraph(cdp, tabId) : null;
return scrollElementWithFrameGraph(cdp, tabId, target, backendNodeId, frameId, graph);
if (target.tabId !== tabId) return geometryError("node target belongs to another tab");
return scrollElementWithContext(
cdp,
tabId,
target,
backendNodeId,
frameId,
new GeometryContext(cdp, tabId),
);
}
function largestRegion(regions: Region): Polygon | null {
@@ -240,23 +120,32 @@ export async function resolveNodeGeometry(
if (address.target.sessionId && !address.frameId) {
return geometryError("an OOPIF node address requires frameId");
}
const graph = address.frameId ? await loadFrameGraph(cdp, tabId) : null;
if (address.target.tabId !== tabId) return geometryError("node target belongs to another tab");
const context = new GeometryContext(cdp, tabId);
const graph = address.frameId ? await context.graph() : null;
if (address.frameId && graph) {
const frame = await context.frame(address.frameId);
if (!frame || cdpTargetKey(frame.target) !== cdpTargetKey(address.target)) {
return geometryError("node frame does not belong to its target");
}
}
if (address.frameId && !graph) {
return geometryError(`could not resolve frame graph for ${address.frameId}`);
}
if (options.scrollIntoView) {
const scrollError = await scrollElementWithFrameGraph(
const scrollError = await scrollElementWithContext(
cdp,
tabId,
address.target,
address.backendNodeId,
address.frameId,
graph,
context,
);
if (scrollError) return scrollError;
}
// Only topology was read above. Begin live measurements after scrolling.
const localRegion = await nodeContentRegion(
cdpRunnerForTarget(cdp, address.target),
tabId,
@@ -267,7 +156,7 @@ export async function resolveNodeGeometry(
let topVisibleRegions: Region;
let targetActionPoint: Point | null = null;
if (address.frameId && graph) {
const projection = await resolveFrameProjection(cdp, graph, address.frameId);
const projection = await context.targetProjection(address.frameId);
if (!projection)
return geometryError(`could not resolve frame geometry for ${address.frameId}`);
topVisibleRegions = projectRegionToViewport(localRegion, projection);
@@ -290,7 +179,7 @@ export async function resolveNodeGeometry(
targetActionPoint = localActionRegion ? polygonCentroid(localActionRegion) : null;
}
} else {
const viewport = await targetViewport(cdp, address.target);
const viewport = await context.viewport(address.target);
if (!viewport) return geometryError("could not resolve top viewport geometry");
topVisibleRegions = localRegion
.map((polygon) =>
@@ -309,7 +198,13 @@ export async function resolveNodeGeometry(
return { code: "permission_denied", message: "element not visible in its target" };
}
targetActionPoint ??= actionPoint;
return { topVisibleRegions, topBounds, actionPoint, targetActionPoint };
return {
topVisibleRegions,
topBounds,
actionPoint,
targetActionPoint,
topViewport: cssViewport(await context.layoutMetrics({ tabId })),
};
} catch (error) {
return geometryError(error instanceof Error ? error.message : String(error));
}
-19
View File
@@ -245,22 +245,3 @@ export function projectRectToViewport(
if (!rect) return null;
return regionBounds(projectRegionToViewport([rectPolygon(rect)], projection));
}
export function childFrameProjection(
parent: GeometryProjection,
ownerRectInParent: { x: number; y: number; w: number; h: number },
): GeometryProjection {
const destinationQuad = rectPolygon(ownerRectInParent);
return {
sourceClips: [],
edges: [
{
sourceViewport: { width: ownerRectInParent.w, height: ownerRectInParent.h },
destinationQuad,
destinationClips: parent.sourceClips,
},
...parent.edges,
],
topViewport: parent.topViewport,
};
}
@@ -0,0 +1,157 @@
import { type CdpTarget, cdpTargetKey } from "@/browser-driver/frame-graph";
import type { GeometryProjection, ViewportRect } from "../geometry";
import { projectRectToViewport } from "../geometry";
export interface CoordinateOwner {
target: CdpTarget;
frameId?: string;
}
export interface FrameViewportRect {
space: "frame-viewport-css";
owner: CoordinateOwner;
rect: ViewportRect;
}
export interface SnapshotProjection {
source: CoordinateOwner;
geometry: GeometryProjection;
}
/** A failed owner read is different from a successfully projected, clipped-out rect. */
export interface UnavailableFrameProjection {
status: "unavailable";
source: CoordinateOwner;
ownerBackendNodeId: number;
}
export interface UnavailableSnapshotCoordinates {
status: "unavailable";
source: CoordinateOwner;
reason: "snapshot-coordinates-unavailable";
}
export type SnapshotProjectionResult =
| { status: "available"; projection: SnapshotProjection }
| UnavailableFrameProjection;
/** Descendants inherit the failed boundary without attempting another owner read. */
export type FrameProjectionIssue =
| UnavailableFrameProjection
| UnavailableSnapshotCoordinates
| {
status: "blocked";
source: CoordinateOwner;
cause: UnavailableFrameProjection | UnavailableSnapshotCoordinates;
};
export type FrameProjectionState = SnapshotProjectionResult | FrameProjectionIssue;
export interface SnapshotCoordinates {
layoutUnitsPerCssPixel: number;
scrollCss: { x: number; y: number };
}
/** Snapshot scroll offsets share the raw layout units of its bounds. Only a
* target's root document may fall back to that target's CSS layout viewport. */
export function snapshotCoordinates(
document: { scrollOffsetX?: number; scrollOffsetY?: number },
layoutUnitsPerCssPixel: number | null,
rootScrollCss?: { x?: number; y?: number },
): SnapshotCoordinates | null {
if (
layoutUnitsPerCssPixel === null ||
!Number.isFinite(layoutUnitsPerCssPixel) ||
layoutUnitsPerCssPixel <= 0
)
return null;
const x =
document.scrollOffsetX === undefined
? rootScrollCss?.x
: document.scrollOffsetX / layoutUnitsPerCssPixel;
const y =
document.scrollOffsetY === undefined
? rootScrollCss?.y
: document.scrollOffsetY / layoutUnitsPerCssPixel;
if (x === undefined || y === undefined || !Number.isFinite(x) || !Number.isFinite(y)) return null;
return { layoutUnitsPerCssPixel, scrollCss: { x, y } };
}
/** Convert raw document-relative Blink layout units exactly once, before any
* CSS viewport clipping or frame projection. This scale is not screenshot zoom. */
export function snapshotViewportRect(
bounds: number[],
owner: CoordinateOwner,
coordinates: SnapshotCoordinates | null,
): FrameViewportRect | null {
if (!coordinates || bounds.length < 4 || !bounds.slice(0, 4).every(Number.isFinite)) return null;
const { layoutUnitsPerCssPixel: scale, scrollCss: scroll } = coordinates;
const [x, y, width, height] = bounds;
if (
!Number.isFinite(scale) ||
scale <= 0 ||
width <= 0 ||
height <= 0 ||
!Number.isFinite(scroll.x) ||
!Number.isFinite(scroll.y)
) {
return null;
}
return {
space: "frame-viewport-css",
owner,
rect: {
x: x / scale - scroll.x,
y: y / scale - scroll.y,
width: width / scale,
height: height / scale,
},
};
}
export function projectSnapshotRect(
input: FrameViewportRect,
projection: SnapshotProjection,
): ViewportRect | null {
if (
input.owner.frameId !== projection.source.frameId ||
cdpTargetKey(input.owner.target) !== cdpTargetKey(projection.source.target)
) {
return null;
}
const { x, y, width: w, height: h } = input.rect;
return projectRectToViewport({ x, y, w, h }, projection.geometry);
}
/** CSS viewport metadata; browser zoom is distinct from raster devicePixelRatio. */
export interface CssViewport {
width: number;
height: number;
scrollX: number;
scrollY: number;
cssToDip: number;
}
/** Page.captureScreenshot takes page-relative DIP, not viewport-relative CSS pixels. */
export function screenshotPageRect(
rect: ViewportRect,
viewport: CssViewport,
): { space: "page-dip"; rect: ViewportRect } | null {
const { cssToDip, scrollX, scrollY } = viewport;
if (
![rect.x, rect.y, rect.width, rect.height, cssToDip, scrollX, scrollY].every(Number.isFinite) ||
cssToDip <= 0 ||
rect.width <= 0 ||
rect.height <= 0
)
return null;
return {
space: "page-dip",
rect: {
x: (rect.x + scrollX) * cssToDip,
y: (rect.y + scrollY) * cssToDip,
width: rect.width * cssToDip,
height: rect.height * cssToDip,
},
};
}
@@ -0,0 +1,356 @@
import {
type CdpFrame,
type CdpFrameGraph,
type CdpTarget,
cdpTargetKey,
} from "@/browser-driver/frame-graph";
import {
type GeometryProjection,
type Polygon,
type ProjectiveEdge,
parseCdpQuad,
type Quad,
type Size,
} from "../geometry";
import { type CdpRunner, sendToCdpTarget } from "../shared";
import type { CoordinateOwner, CssViewport, SnapshotProjectionResult } from "./coordinate-types";
import { readSnapshotOwnerSizes } from "./snapshot-owner-sizes";
export interface LayoutMetrics {
cssVisualViewport?: { zoom?: number; clientWidth?: number; clientHeight?: number };
visualViewport?: { zoom?: number; clientWidth?: number; clientHeight?: number };
cssLayoutViewport?: {
clientWidth?: number;
clientHeight?: number;
pageX?: number;
pageY?: number;
};
layoutViewport?: { clientWidth?: number; clientHeight?: number; pageX?: number; pageY?: number };
}
/** Chromium emits these paired floating-point dimensions before/after dividing
* by LayoutZoomFactor. Integer layout viewport dimensions lose precision at
* fractional zoom. Neither visualViewport.scale nor .zoom is this conversion.
* See InspectorPageAgent::getLayoutMetrics in Chromium. */
export function snapshotLayoutScale(metrics: LayoutMetrics): number | null {
for (const dimension of ["clientWidth", "clientHeight"] as const) {
const raw = metrics.visualViewport?.[dimension];
const css = metrics.cssVisualViewport?.[dimension];
if (
raw !== undefined &&
css !== undefined &&
Number.isFinite(raw) &&
Number.isFinite(css) &&
raw > 0 &&
css > 0
) {
const scale = raw / css;
if (Number.isFinite(scale) && scale > 0) return scale;
}
}
return null;
}
/** The legacy field is a compatibility fallback, never a source of raster DPR. */
export function cssViewport(metrics: LayoutMetrics): CssViewport {
const value = metrics.cssLayoutViewport ?? metrics.layoutViewport;
return {
width: value?.clientWidth ?? 0,
height: value?.clientHeight ?? 0,
scrollX: value?.pageX ?? 0,
scrollY: value?.pageY ?? 0,
cssToDip: metrics.cssVisualViewport?.zoom ?? metrics.visualViewport?.zoom ?? 1,
};
}
/** One read-only measurement phase. Discard after scrolling or any later operation. */
export class GeometryContext {
private readonly metrics = new Map<string, Promise<LayoutMetrics>>();
private readonly owners = new Map<string, Promise<Quad | null>>();
private readonly projections = new Map<string, Promise<GeometryProjection | null>>();
private readonly snapshotOwners = new Map<string, Promise<{ quad: Quad; size: Size } | null>>();
private readonly sizeBatches = new Map<string, () => Promise<Map<number, Size>>>();
private active = 0;
private readonly waiting: Array<() => void> = [];
private graphPromise?: Promise<CdpFrameGraph | null>;
private framesPromise?: Promise<Map<string, CdpFrame>>;
constructor(
private readonly cdp: CdpRunner,
private readonly tabId: number,
graph?: CdpFrameGraph,
private readonly signal?: AbortSignal,
) {
if (graph) this.graphPromise = Promise.resolve(graph);
}
private async request<T>(
target: CdpTarget,
method: string,
params: object,
cleanup = false,
): Promise<T> {
if (!cleanup && this.signal?.aborted) throw new DOMException("geometry aborted", "AbortError");
if (this.active >= 4) await new Promise<void>((resolve) => this.waiting.push(resolve));
else this.active++;
try {
if (!cleanup && this.signal?.aborted)
throw new DOMException("geometry aborted", "AbortError");
return await sendToCdpTarget<T>(this.cdp, target, method, params);
} finally {
const next = this.waiting.shift();
if (next) next();
else this.active--;
}
}
graph(): Promise<CdpFrameGraph | null> {
this.graphPromise ??=
this.cdp.getFrameGraph?.(this.tabId).catch(() => null) ?? Promise.resolve(null);
return this.graphPromise;
}
private frames(): Promise<Map<string, CdpFrame>> {
this.framesPromise ??= this.graph().then(
(graph) => new Map(graph?.frames.map((frame) => [frame.frameId, frame]) ?? []),
);
return this.framesPromise;
}
async frame(frameId: string): Promise<CdpFrame | undefined> {
return (await this.frames()).get(frameId);
}
/** Topology only: safe to resolve before the caller scrolls frame owners. */
async ancestry(frameId: string): Promise<CdpFrame[] | null> {
const frames = await this.frames();
const path: CdpFrame[] = [];
const seen = new Set<string>();
let current = frames.get(frameId);
if (!current) return null;
while (current.parentFrameId) {
if (seen.has(current.frameId)) return null;
seen.add(current.frameId);
path.push(current);
const parent = frames.get(current.parentFrameId);
if (!parent) return null;
current = parent;
}
return path;
}
layoutMetrics(target: CdpTarget): Promise<LayoutMetrics> {
const key = cdpTargetKey(target);
let promise = this.metrics.get(key);
if (!promise) {
promise = this.request<LayoutMetrics>(target, "Page.getLayoutMetrics", {});
this.metrics.set(key, promise);
}
return promise;
}
async viewport(target: CdpTarget): Promise<Size | null> {
const { width, height } = cssViewport(await this.layoutMetrics(target));
return Number.isFinite(width) && Number.isFinite(height) && width > 0 && height > 0
? { width, height }
: null;
}
ownerContent(target: CdpTarget, backendNodeId: number): Promise<Quad | null> {
const key = `${cdpTargetKey(target)}:${backendNodeId}`;
let promise = this.owners.get(key);
if (!promise) {
promise = this.request<{ model?: { content?: number[] } }>(target, "DOM.getBoxModel", {
backendNodeId,
}).then((reply) => parseCdpQuad(reply.model?.content));
this.owners.set(key, promise);
}
return promise;
}
/** Same-target owner quads are already target-relative, including nested owners. */
async snapshotProjection(
source: CoordinateOwner,
ownerBackendNodeId: number,
ancestorClips: Polygon[],
viewport: Size,
): Promise<SnapshotProjectionResult> {
const key = `${cdpTargetKey(source.target)}:${ownerBackendNodeId}`;
let promise = this.snapshotOwners.get(key);
if (!promise) {
promise = this.snapshotOwner(source.target, ownerBackendNodeId);
this.snapshotOwners.set(key, promise);
}
let owner: { quad: Quad; size: Size } | null;
try {
owner = await promise;
} catch (error) {
if (error && typeof error === "object" && "name" in error && error.name === "AbortError")
throw error;
console.debug("[bsk geometry] frame owner unavailable", error);
owner = null;
}
return owner
? {
status: "available",
projection: {
source,
geometry: {
sourceClips: [],
edges: [
{
sourceViewport: owner.size,
destinationQuad: owner.quad,
destinationClips: ancestorClips,
},
],
topViewport: viewport,
},
},
}
: { status: "unavailable", source, ownerBackendNodeId };
}
/** Register one target's snapshot owners; the batch starts only when a
* projection actually needs a size. Single owners keep the cheaper old path. */
registerSnapshotOwners(target: CdpTarget, owners: ReadonlySet<number>): void {
const key = cdpTargetKey(target);
if (owners.size < 2 || this.sizeBatches.has(key)) return;
let pending: Promise<Map<number, Size>> | undefined;
this.sizeBatches.set(key, () => {
pending ??= readSnapshotOwnerSizes(
(method, params, cleanup) => this.request(target, method, params, cleanup),
owners,
).catch((error) => {
if (error && typeof error === "object" && "name" in error && error.name === "AbortError")
throw error;
return new Map<number, Size>();
});
return pending;
});
}
private async snapshotOwner(
target: CdpTarget,
backendNodeId: number,
): Promise<{ quad: Quad; size: Size } | null> {
const quad = await this.ownerContent(target, backendNodeId);
if (!quad) return null;
const batchSize = (await this.sizeBatches.get(cdpTargetKey(target))?.())?.get(backendNodeId);
if (this.signal?.aborted) throw new DOMException("geometry aborted", "AbortError");
if (batchSize) return { quad, size: batchSize };
const resolved = await this.request<{ object?: { objectId?: string } }>(
target,
"DOM.resolveNode",
{ backendNodeId },
);
const objectId = resolved.object?.objectId;
if (!objectId) return null;
try {
const reply = await this.request<{ result?: { value?: Size }; exceptionDetails?: unknown }>(
target,
"Runtime.callFunctionOn",
{
objectId,
functionDeclaration: `function() {
if (!this.isConnected) return null;
const style = getComputedStyle(this);
return {
width: this.clientWidth - parseFloat(style.paddingLeft) - parseFloat(style.paddingRight),
height: this.clientHeight - parseFloat(style.paddingTop) - parseFloat(style.paddingBottom)
};
}`,
returnByValue: true,
},
);
const size = reply.result?.value;
return !reply.exceptionDetails &&
size &&
Number.isFinite(size.width) &&
Number.isFinite(size.height) &&
size.width > 0 &&
size.height > 0
? { quad, size }
: null;
} finally {
// Cleanup must still run if cancellation arrived after resolving the object.
await this.request(target, "Runtime.releaseObject", { objectId }, true).catch(() => {});
}
}
private targetRoot(frames: Map<string, CdpFrame>, frame: CdpFrame): CdpFrame | null {
const seen = new Set<string>();
let current = frame;
while (current.parentFrameId) {
if (seen.has(current.frameId)) return null;
seen.add(current.frameId);
const parent = frames.get(current.parentFrameId);
if (!parent) return null;
if (cdpTargetKey(parent.target) !== cdpTargetKey(current.target)) break;
current = parent;
}
return current;
}
private async clips(
frames: Map<string, CdpFrame>,
frame: CdpFrame,
root: CdpFrame,
): Promise<Polygon[] | null> {
const clips: Polygon[] = [];
let current = frame;
const seen = new Set<string>();
while (current.frameId !== root.frameId) {
if (seen.has(current.frameId) || !current.parentFrameId) return null;
seen.add(current.frameId);
const parent = frames.get(current.parentFrameId);
if (!parent || current.ownerBackendNodeId === undefined) return null;
const quad = await this.ownerContent(parent.target, current.ownerBackendNodeId);
if (!quad) return null;
clips.push(quad);
current = parent;
}
return clips;
}
/** Input is already in the frame's CDP target viewport, not its local viewport. */
targetProjection(frameId: string): Promise<GeometryProjection | null> {
let promise = this.projections.get(frameId);
if (!promise) {
promise = this.buildTargetProjection(frameId);
this.projections.set(frameId, promise);
}
return promise;
}
private async buildTargetProjection(frameId: string): Promise<GeometryProjection | null> {
const frames = await this.frames();
const frame = frames.get(frameId);
if (!frame) return null;
let root = this.targetRoot(frames, frame);
if (!root) return null;
const sourceViewport = await this.viewport(root.target);
if (!sourceViewport) return null;
const sourceClips = await this.clips(frames, frame, root);
if (!sourceClips) return null;
const edges: ProjectiveEdge[] = [];
const seen = new Set<string>();
while (root.parentFrameId) {
if (seen.has(root.frameId)) return null;
seen.add(root.frameId);
const parent = frames.get(root.parentFrameId);
if (!parent || root.ownerBackendNodeId === undefined) return null;
const destinationQuad = await this.ownerContent(parent.target, root.ownerBackendNodeId);
const source = await this.viewport(root.target);
const parentRoot = this.targetRoot(frames, parent);
if (!destinationQuad || !source || !parentRoot) return null;
const destinationClips = await this.clips(frames, parent, parentRoot);
if (!destinationClips) return null;
edges.push({ sourceViewport: source, destinationQuad, destinationClips });
root = parentRoot;
}
const topViewport = await this.viewport(root.target);
return topViewport ? { sourceClips, edges, topViewport } : null;
}
}
@@ -0,0 +1,74 @@
import type { Size } from "../geometry";
interface SerializedValue {
type: string;
value?: unknown;
}
type Send = <T>(method: string, params: object, cleanup?: boolean) => Promise<T>;
/** Batch only accessible frame owners. Missing/shadow/cross-origin owners keep
* the existing node-scoped read; no page-wide style columns are requested. */
export async function readSnapshotOwnerSizes(
send: Send,
owners: ReadonlySet<number>,
): Promise<Map<number, Size>> {
const sizes = new Map<number, Size>();
const objectGroup = `bsk-frame-sizes-${crypto.randomUUID()}`;
try {
const reply = await send<{ result?: { deepSerializedValue?: SerializedValue } }>(
"Runtime.evaluate",
{
expression: `(() => {
const result = [];
const documents = [document];
for (let i = 0; i < documents.length && result.length < ${owners.size}; i++) {
for (const owner of documents[i].querySelectorAll('iframe,frame')) {
if (result.length >= ${owners.size}) break;
if (!owner.isConnected) continue;
const style = getComputedStyle(owner);
result.push([owner, JSON.stringify({
width: owner.clientWidth - parseFloat(style.paddingLeft) - parseFloat(style.paddingRight),
height: owner.clientHeight - parseFloat(style.paddingTop) - parseFloat(style.paddingBottom)
})]);
try { if (owner.contentDocument) documents.push(owner.contentDocument); } catch {}
}
}
return result;
})()`,
objectGroup,
serializationOptions: {
serialization: "deep",
maxDepth: 3,
additionalParameters: { maxNodeDepth: 0, includeShadowTree: "none" },
},
},
);
const result = reply.result?.deepSerializedValue;
if (result?.type !== "array" || !Array.isArray(result.value)) return sizes;
for (const entry of result.value as SerializedValue[]) {
if (entry.type !== "array" || !Array.isArray(entry.value)) continue;
const [node, data] = entry.value as SerializedValue[];
if (node?.type !== "node" || data?.type !== "string" || typeof data.value !== "string")
continue;
const id = (node.value as { backendNodeId?: number } | undefined)?.backendNodeId;
if (id === undefined || !owners.has(id)) continue;
try {
const size = JSON.parse(data.value) as Size;
if (
size &&
Number.isFinite(size.width) &&
Number.isFinite(size.height) &&
size.width > 0 &&
size.height > 0
)
sizes.set(id, size);
} catch {
/* An invalid entry uses the existing individual read. */
}
}
} finally {
await send("Runtime.releaseObjectGroup", { objectGroup }, true).catch(() => {});
}
return sizes;
}
+282 -42
View File
@@ -183,6 +183,8 @@ export async function resolveBackendNode(
}
// selector path
try {
// Selector lookup itself attaches CDP, even when no element is found.
cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
const doc = await cdp.send<{ root?: { nodeId?: number } }>(target.tabId, "DOM.getDocument", {
depth: 0,
});
@@ -607,6 +609,53 @@ type SelectMutationResult =
// tool.fill
// ---------------------------------------------------------------------------
type FillFailureReason =
| "target_not_fillable"
| "fill_value_invalid"
| "fill_target_changed"
| "fill_focus_lost"
| "fill_value_mismatch"
| "fill_failed";
type FillPreparation =
| { before: string; expected: string }
| { reason: FillFailureReason; error: string };
type FillReadiness = "ready" | "background" | "fill_target_changed" | "fill_focus_lost";
function fillError(reason: FillFailureReason, message: string): RpcError {
return rpcError(
reason === "target_not_fillable" || reason === "fill_value_invalid"
? "invalid_params"
: "cdp_failed",
reason,
message,
);
}
interface FillScriptReply<T> {
result?: { value?: T };
exceptionDetails?: unknown;
}
const FILL_EDITABLE_FUNCTION = `function() {
const tag = this.tagName.toLowerCase();
const supported = tag === 'input'
? ['text', 'search', 'tel', 'url', 'email', 'password', 'number'].includes(this.type)
: tag === 'textarea' || this.isContentEditable;
return this.isConnected && supported && !this.readOnly && !this.matches(':disabled');
}`;
// Chrome renders a trailing editable newline with an empty <div><br></div>.
// innerText includes the padding break; it is not an extra typed character.
const FILL_VALUE_FUNCTION = `function() {
if (!this.isContentEditable) return this.value;
const value = this.innerText;
const tail = this.lastChild;
const padding = tail && tail.nodeName === 'DIV' && tail.childNodes.length === 1 && tail.firstChild.nodeName === 'BR';
return padding && value.endsWith('\\n\\n') ? value.slice(0, -1) : value;
}`;
export async function handleFill(
manager: SessionManager,
params: FillParams,
@@ -658,12 +707,8 @@ export async function handleFill(
if (throwIfAborted(deps.signal)) {
return { code: "cancelled", message: "fill aborted" };
}
await nodeCdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
} catch (err) {
return {
code: "cdp_failed",
message: err instanceof Error ? err.message : String(err),
};
return fillError("fill_failed", err instanceof Error ? err.message : String(err));
}
if (throwIfAborted(deps.signal)) {
@@ -676,57 +721,252 @@ export async function handleFill(
const clearBefore = params.clear_before ?? true;
try {
if (clearBefore) {
// Clear input/textarea value or wipe contenteditable innerText,
// then fire `input` so frameworks observe the empty state.
await nodeCdp.send(target.tabId, "Runtime.callFunctionOn", {
objectId,
functionDeclaration: `function() {
if (this.isContentEditable) { this.textContent = ''; }
else {
const proto = this instanceof HTMLTextAreaElement
? HTMLTextAreaElement.prototype
: HTMLInputElement.prototype;
const descriptor = Object.getOwnPropertyDescriptor(proto, 'value');
if (descriptor && descriptor.set) descriptor.set.call(this, '');
else this.value = '';
}
this.dispatchEvent(new Event('input', { bubbles: true }));
}`,
returnByValue: true,
});
if (throwIfAborted(deps.signal)) {
return { code: "cancelled", message: "fill aborted" };
}
try {
await nodeCdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
} catch (error) {
// Chrome rejects DOM.focus for disabled controls, including inherited
// fieldset state. Classify the live target before treating this as a
// browser failure; the normal path needs no extra round trip.
const editable = await nodeCdp.send<FillScriptReply<boolean>>(
target.tabId,
"Runtime.callFunctionOn",
{ objectId, functionDeclaration: FILL_EDITABLE_FUNCTION, returnByValue: true },
);
if (throwIfAborted(deps.signal)) {
return { code: "cancelled", message: "fill aborted" };
}
if (!editable.exceptionDetails && editable.result?.value === false) {
return fillError(
"target_not_fillable",
"fill target is not editable or its input type is unsupported",
);
}
throw error;
}
if (throwIfAborted(deps.signal)) {
return { code: "cancelled", message: "fill aborted" };
}
// Check editability before clearing. Keep the expected result tied to
// this object, including the existing value on the append path.
const prepared = await nodeCdp.send<FillScriptReply<FillPreparation>>(
target.tabId,
"Runtime.callFunctionOn",
{
objectId,
functionDeclaration: `function(value, clearBefore) {
const tag = this.tagName.toLowerCase();
const native = tag === 'input' || tag === 'textarea';
if (!(${FILL_EDITABLE_FUNCTION}).call(this)) return { reason: 'target_not_fillable', error: 'fill target is not editable or its input type is unsupported' };
if (this.getRootNode().activeElement !== this) return { reason: 'fill_focus_lost', error: 'fill target does not have focus' };
const before = clearBefore ? '' : (${FILL_VALUE_FUNCTION}).call(this);
let expected = before + value;
if (native) {
// Use the browser's own value sanitization (e.g. textarea
// line endings) without changing the live control.
const normalizer = this.ownerDocument.createElement(tag);
if (tag === 'input') {
normalizer.type = this.type;
normalizer.multiple = this.multiple;
}
// insertText treats line breaks in a single-line input as spaces.
if (tag === 'input') expected = expected.replace(/\\r\\n|\\r|\\n/g, ' ');
normalizer.value = expected;
if (expected !== '' && normalizer.value === '' && this.type === 'number') {
return { reason: 'fill_value_invalid', error: 'fill value is not valid for a number input' };
}
expected = normalizer.value;
if (this.type !== 'number' && this.maxLength >= 0 && expected.length > this.maxLength) {
return { reason: 'fill_value_invalid', error: 'fill value exceeds the target maxlength' };
}
} else {
expected = expected.replace(/\\r\\n?/g, '\\n');
}
if (clearBefore) {
if (native) {
const proto = tag === 'textarea' ? HTMLTextAreaElement.prototype : HTMLInputElement.prototype;
Object.getOwnPropertyDescriptor(proto, 'value').set.call(this, '');
} else {
this.textContent = '';
}
this.dispatchEvent(new Event('input', { bubbles: true }));
}
return { before, expected };
}`,
arguments: [{ value: params.value }, { value: clearBefore }],
returnByValue: true,
},
);
if (throwIfAborted(deps.signal)) {
return { code: "cancelled", message: "fill aborted" };
}
if (prepared.exceptionDetails) {
return fillError("fill_failed", "fill preparation script failed");
}
const preparation = prepared.result?.value;
if (preparation && "error" in preparation) {
return fillError(preparation.reason, preparation.error);
}
if (typeof preparation?.before !== "string" || typeof preparation?.expected !== "string") {
return fillError("fill_failed", "fill preparation returned an unexpected result");
}
// A separate call runs after the clearing event's microtasks drain.
// DOM focus works in background tabs. Use document.hasFocus() only to
// detect deferred focus events, never to reject background input.
const checkReady = async (): Promise<FillReadiness | RpcError> => {
const reply = await nodeCdp.send<FillScriptReply<FillReadiness>>(
target.tabId,
"Runtime.callFunctionOn",
{
objectId,
functionDeclaration: `function(before) {
if (!(${FILL_EDITABLE_FUNCTION}).call(this) ||
(${FILL_VALUE_FUNCTION}).call(this) !== before) return 'fill_target_changed';
if (this.getRootNode().activeElement !== this) return 'fill_focus_lost';
return this.ownerDocument.hasFocus() ? 'ready' : 'background';
}`,
arguments: [{ value: preparation.before }],
returnByValue: true,
},
);
const state = reply.result?.value;
if (
reply.exceptionDetails ||
(state !== "ready" &&
state !== "background" &&
state !== "fill_target_changed" &&
state !== "fill_focus_lost")
) {
return fillError(
"fill_failed",
"fill readiness script failed or returned an unexpected result",
);
}
return state;
};
let ready = await checkReady();
if (throwIfAborted(deps.signal)) {
return { code: "cancelled", message: "fill aborted" };
}
// Restore focus once only if clearing left this same target editable
// and unchanged. Recheck after focus handlers have run.
if (clearBefore && ready === "fill_focus_lost") {
await nodeCdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
if (throwIfAborted(deps.signal)) {
return { code: "cancelled", message: "fill aborted" };
}
ready = await checkReady();
}
if (throwIfAborted(deps.signal)) {
return { code: "cancelled", message: "fill aborted" };
}
if (typeof ready !== "string") return ready;
if (ready !== "ready" && ready !== "background") {
return fillError(ready, "fill target changed or lost focus before typing");
}
if (params.value !== "" && (!clearBefore || ready === "background")) {
// Native editing commands also support number/email inputs, whose
// selection APIs cannot set a caret, and multiline contenteditables.
// Also do this for background replacement: CDP input focuses the
// renderer, delivering deferred focus events before it inserts text.
// Use a command-only event so this does not invoke an End shortcut.
try {
await deps.cdp.send(target.tabId, "Input.dispatchKeyEvent", {
type: "rawKeyDown",
commands: ["moveToEndOfDocument"],
});
} finally {
await deps.cdp.send(target.tabId, "Input.dispatchKeyEvent", { type: "keyUp" });
}
if (throwIfAborted(deps.signal)) {
return { code: "cancelled", message: "fill aborted" };
}
// Focus and key handlers can change the target or move focus too.
ready = await checkReady();
if (throwIfAborted(deps.signal)) {
return { code: "cancelled", message: "fill aborted" };
}
if (typeof ready !== "string") return ready;
if (ready !== "ready" && ready !== "background") {
return fillError(ready, "fill target changed or lost focus while positioning the caret");
}
}
// CDP `Input.insertText` handles IME / multi-byte input out of the
// box, much more reliably than per-key `dispatchKeyEvent`.
await deps.cdp.send(target.tabId, "Input.insertText", { text: params.value });
if (params.value !== "") {
await deps.cdp.send(target.tabId, "Input.insertText", { text: params.value });
}
if (throwIfAborted(deps.signal)) {
return { code: "cancelled", message: "fill aborted" };
}
// Fire `input` + `change` so React / Vue controlled inputs commit.
await nodeCdp.send(target.tabId, "Runtime.callFunctionOn", {
// Keep the existing notifications, then read in a separate call so
// nested microtasks queued by the page cannot race the verification.
const notified = await nodeCdp.send<FillScriptReply<unknown>>(
target.tabId,
"Runtime.callFunctionOn",
{
objectId,
functionDeclaration: `function() {
this.dispatchEvent(new Event('input', { bubbles: true }));
this.dispatchEvent(new Event('change', { bubbles: true }));
}`,
returnByValue: true,
},
);
if (throwIfAborted(deps.signal)) {
return { code: "cancelled", message: "fill aborted" };
}
if (notified.exceptionDetails) {
return fillError("fill_failed", "fill notification script failed");
}
const verified = await nodeCdp.send<
FillScriptReply<{ connected: boolean; matches: boolean; valueLength: number }>
>(target.tabId, "Runtime.callFunctionOn", {
objectId,
functionDeclaration: `function() {
this.dispatchEvent(new Event('input', { bubbles: true }));
this.dispatchEvent(new Event('change', { bubbles: true }));
}`,
functionDeclaration: `function(expected) {
const value = (${FILL_VALUE_FUNCTION}).call(this);
return { connected: this.isConnected, matches: value === expected, valueLength: value.length };
}`,
arguments: [{ value: preparation.expected }],
returnByValue: true,
});
if (throwIfAborted(deps.signal)) {
return { code: "cancelled", message: "fill aborted" };
}
const verification = verified.result?.value;
if (
verified.exceptionDetails ||
typeof verification?.connected !== "boolean" ||
typeof verification?.matches !== "boolean" ||
typeof verification?.valueLength !== "number"
) {
return fillError(
"fill_failed",
"fill verification script failed or returned an unexpected result",
);
}
if (!verification.connected) {
return fillError("fill_target_changed", "fill target was removed or replaced during input");
}
if (!verification.matches) {
return fillError(
"fill_value_mismatch",
"fill could not verify the expected value; observe the page before deciding whether to retry",
);
}
return attachDialogs(deps.cdp, target.tabId, dialogCursor, {
tab_id: target.tabId,
used_ref: node.usedRef,
used_selector: node.usedSelector,
value_length: verification.valueLength,
});
} catch (err) {
return {
code: "cdp_failed",
message: err instanceof Error ? err.message : String(err),
};
return fillError("fill_failed", err instanceof Error ? err.message : String(err));
} finally {
await nodeCdp.send(target.tabId, "Runtime.releaseObject", { objectId }).catch(() => undefined);
}
return attachDialogs(deps.cdp, target.tabId, dialogCursor, {
tab_id: target.tabId,
used_ref: node.usedRef,
used_selector: node.usedSelector,
value_length: params.value.length,
});
}
// ---------------------------------------------------------------------------
+116 -10
View File
@@ -28,7 +28,13 @@ import type {
RpcError,
WaitUntil,
} from "@/transport/types";
import {
type BrowserNavigationApi,
chromeBrowserNavigationApi,
navigateWithBrowserApi,
} from "./browser-navigation";
import { attachDialogs, markDialogCursor } from "./dialogs";
import { cdpError, isCdpExtensionAccessDenied } from "./errors";
import {
type CdpRunner,
type ChromeTabsApi,
@@ -42,6 +48,7 @@ import {
export interface NavigationDeps {
cdp: CdpRunner;
tabsApi: ChromeTabsApi;
browserNavigation?: BrowserNavigationApi;
/** Optional AbortSignal — M7 abort hook (M10.2 will wire the full chain). */
signal?: AbortSignal;
/** Override default timeout when the caller omits `timeout_ms`. */
@@ -479,6 +486,81 @@ async function readTabUrl(api: ChromeTabsApi, tabId: number): Promise<string | u
}
}
/** Recover only a failed preflight; never replay an already-dispatched navigation. */
async function recoverBrowserNavigation(
deps: NavigationDeps,
tabId: number,
action: (api: BrowserNavigationApi) => Promise<unknown>,
waitUntil: WaitUntil,
timeoutMs: number,
): Promise<ReloadResult | RpcError> {
const abort = linkedAbortSignal(deps.signal);
const deadline = Date.now() + timeoutMs;
const api = deps.browserNavigation ?? chromeBrowserNavigationApi;
try {
let outcome = await navigateWithBrowserApi(
api,
tabId,
() => action(api),
waitUntil === "networkidle" ? "commit" : waitUntil,
timeoutMs,
abort.signal,
);
if (outcome.reached === "failed") return outcome.error;
if (outcome.reached === "cancelled" || abort.signal.aborted) {
return { code: "cancelled", message: "navigation recovery aborted" };
}
if (outcome.reached === "match") {
// A completed reload is not a recovery if the restricted frame remains.
await deps.cdp.send(tabId, "Page.enable", {});
if (abort.signal.aborted)
return { code: "cancelled", message: "navigation recovery aborted" };
if (waitUntil === "networkidle") {
// webNavigation cannot prove network idle. Subscribe on the new
// document before enabling CDP lifecycle events after its commit.
const frame = await readMainFrameInfo(deps.cdp, tabId);
if (!frame.frameId) return cdpError("no main frame after navigation");
const remaining = deadline - Date.now();
if (remaining <= 0) {
outcome = { reached: "timeout", lastLifecycle: "commit" };
} else {
const wait = startLifecycleWait(
deps.cdp,
tabId,
frame.frameId,
"networkIdle",
remaining,
abort.signal,
{ loaderId: frame.loaderId ?? undefined },
);
await deps.cdp.send(tabId, "Page.setLifecycleEventsEnabled", { enabled: true });
outcome = await wait.promise;
}
}
}
if (outcome.reached === "cancelled" || abort.signal.aborted) {
return { code: "cancelled", message: "navigation recovery aborted" };
}
return {
tab_id: tabId,
final_url: await readTabUrl(deps.tabsApi, tabId),
reached: outcome.reached === "match" ? waitUntil : "timeout",
...(outcome.reached === "timeout"
? {
error_text: `timed out waiting for ${waitUntil} during navigation recovery after ${timeoutMs}ms`,
}
: {}),
};
} catch (error) {
return abort.signal.aborted
? { code: "cancelled", message: "navigation recovery aborted" }
: cdpError(error);
} finally {
abort.abort();
abort.cleanup();
}
}
// ---------------------------------------------------------------------------
// tool.navigate
// ---------------------------------------------------------------------------
@@ -505,7 +587,20 @@ export async function handleNavigate(
try {
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
await ensureCdpReady(deps.cdp, target.tabId);
try {
await ensureCdpReady(deps.cdp, target.tabId);
} catch (error) {
if (!isCdpExtensionAccessDenied(error)) throw error;
const recovered = await recoverBrowserNavigation(
deps,
target.tabId,
(api) => api.update(target.tabId, { url: params.url }),
waitUntil,
timeoutMs,
);
if (isRpcError(recovered)) return recovered;
return attachDialogs(deps.cdp, target.tabId, dialogCursor, { ...recovered, url: params.url });
}
const expected = cdpLifecycleName(waitUntil);
const beforeReadyState = await probeMainFrameReadyState(deps.cdp, target.tabId);
const beforeFrame = await readMainFrameInfo(deps.cdp, target.tabId);
@@ -574,10 +669,7 @@ export async function handleNavigate(
}`,
});
} catch (err) {
return {
code: "cdp_failed",
message: err instanceof Error ? err.message : String(err),
};
return cdpError(err);
}
}
@@ -727,7 +819,24 @@ export async function handleReload(
try {
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
await ensureCdpReady(deps.cdp, target.tabId);
try {
await ensureCdpReady(deps.cdp, target.tabId);
} catch (error) {
if (!isCdpExtensionAccessDenied(error)) throw error;
const previousUrl = await readTabUrl(deps.tabsApi, target.tabId);
const recovered = await recoverBrowserNavigation(
deps,
target.tabId,
(api) => api.reload(target.tabId, { bypassCache: ignoreCache }),
waitUntil,
timeoutMs,
);
if (isRpcError(recovered)) return recovered;
return attachDialogs(deps.cdp, target.tabId, dialogCursor, {
...recovered,
previous_url: previousUrl,
});
}
const previousUrl = await readTabUrl(deps.tabsApi, target.tabId);
const beforeFrame = await readMainFrameInfo(deps.cdp, target.tabId);
const expected = cdpLifecycleName(waitUntil);
@@ -776,10 +885,7 @@ export async function handleReload(
}`,
});
} catch (err) {
return {
code: "cdp_failed",
message: err instanceof Error ? err.message : String(err),
};
return cdpError(err);
}
}
+33 -20
View File
@@ -34,6 +34,8 @@ import type {
import { attachDialogs, markDialogCursor } from "./dialogs";
import { rpcError } from "./errors";
import { resolveNodeGeometry } from "./frame-geometry";
import { screenshotPageRect } from "./geometry/coordinate-types";
import { cssViewport, GeometryContext } from "./geometry/frame-context";
import {
type ChromeTabsApi,
enforceToolTargetScope,
@@ -197,15 +199,14 @@ async function captureElementScreenshot(
if (isRpcError(geometry)) return geometry;
if (signal?.aborted) return cancelled("screenshot");
const rect = geometry.topBounds;
const clip = screenshotPageRect(rect, geometry.topViewport);
if (!clip) return { code: "cdp_failed", message: "invalid screenshot coordinate space" };
try {
const shot = await cdp.send<{ data?: string }>(tabId, "Page.captureScreenshot", {
format: "png",
clip: {
x: rect.x,
y: rect.y,
width: rect.width,
height: rect.height,
...clip.rect,
scale: 1,
},
});
@@ -675,6 +676,7 @@ function legacyFrameDocuments(
...(pageUrl ? { url: pageUrl } : {}),
axNodes: [],
domNodes: captured.nodes.map((node) => ({ ...node, frameId: node.frameId ?? rootFrameId })),
excludedBackendNodeIds: captured.frameExcludedBackendNodeIds?.get(rootFrameId),
},
];
const pending = [...captured.iframeNodes.entries()];
@@ -698,6 +700,7 @@ function legacyFrameDocuments(
target: parent.target,
axNodes: [],
domNodes: domNodes.map((node) => ({ ...node, frameId: node.frameId ?? frameId })),
excludedBackendNodeIds: captured.frameExcludedBackendNodeIds?.get(frameId),
});
pending.splice(index, 1);
progress = true;
@@ -984,26 +987,19 @@ function emptyCapturedViewModel(viewport = { width: 0, height: 0 }): CapturedVie
return { viewport, nodes: [], iframeNodes: new Map(), excludedBackendNodeIds: new Set() };
}
interface LayoutMetricsViewportReply {
cssLayoutViewport?: { clientWidth?: number; clientHeight?: number };
layoutViewport?: { clientWidth?: number; clientHeight?: number };
}
async function fallbackCapturedViewModel(
cdp: CdpRunner,
tabId: number,
geometry: GeometryContext,
signal?: AbortSignal,
): Promise<CapturedViewModel> {
throwIfAborted(signal, "observation");
let viewport = { width: 0, height: 0 };
try {
const metrics = await cdp.send<LayoutMetricsViewportReply>(tabId, "Page.getLayoutMetrics", {});
const metrics = await geometry.layoutMetrics({ tabId });
throwIfAborted(signal, "observation");
const source = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
viewport = {
width: source.clientWidth ?? 0,
height: source.clientHeight ?? 0,
};
const source = cssViewport(metrics);
viewport = { width: source.width, height: source.height };
} catch (error) {
if (isAbortError(error)) throw error;
}
@@ -1016,12 +1012,13 @@ async function captureForVom(
cdp: CdpRunner,
tabId: number,
options: CaptureVomObservationOptions,
geometry: GeometryContext,
): Promise<CapturedViewModel> {
try {
return await captureViewModel(cdp, tabId, { signal: options.signal });
return await captureViewModel(cdp, tabId, { signal: options.signal, geometry });
} catch (error) {
if (isAbortError(error)) throw error;
return fallbackCapturedViewModel(cdp, tabId, options.signal);
return fallbackCapturedViewModel(cdp, tabId, geometry, options.signal);
}
}
@@ -1096,9 +1093,16 @@ export async function captureVomObservation(
throwIfAborted(options.signal, "observation");
await cdp.ensureAttachedToUrl?.(tabId, url);
throwIfAborted(options.signal, "observation");
const captured = await captureForVom(cdp, tabId, options);
const geometry = new GeometryContext(cdp, tabId, undefined, options.signal);
const captured = await captureForVom(cdp, tabId, options, geometry);
throwIfAborted(options.signal, "observation");
const documents = await captureFrameData<CdpAxNode>(cdp, tabId, captured, options.signal);
const documents = await captureFrameData<CdpAxNode>(
cdp,
tabId,
captured,
options.signal,
geometry,
);
throwIfAborted(options.signal, "observation");
const normalizedDocuments =
documents.length === 1 && captured.iframeNodes.size > 0
@@ -1131,12 +1135,21 @@ export async function captureVomObservation(
captured,
hoverProbes.surfaceProbes,
);
// Reserve space using the renderer's character-based token estimate. Like VOM
// headers, this integrity notice remains visible even under a tiny token budget.
const geometryNotice = captured.frameGeometryIssues?.length
? "@warning geometry incomplete: some page or frame content has no top-level coordinates.\n"
: "";
const rendered = renderVom(decoratedScene, {
maxDepth: options.maxDepth,
maxTokens: options.maxTokens,
maxTokens:
!geometryNotice || options.maxTokens === undefined
? options.maxTokens
: Math.max(0, options.maxTokens - Math.ceil(geometryNotice.length / 4)),
redactValues: options.redactValues,
activeRegionPolicy: options.activeRegionPolicy,
});
if (geometryNotice) rendered.text += `\n${geometryNotice.trimEnd()}`;
throwIfAborted(options.signal, "observation");
return projectRecordSafeObservation({
rootFrameId: captured.rootFrameId ?? normalizedDocuments[0]?.frameId ?? "root",
+94 -8
View File
@@ -2,6 +2,7 @@ import { type SessionManager, SessionStartCleanupError } from "@/session-manager
import type { RpcError } from "@/transport/types";
import { rpcError } from "./errors";
import { clearRecordingForSession } from "./record";
import type { CdpRunner, ChromeTabsApi } from "./shared";
import { isRpcError } from "./shared";
import { returnBorrowedTab, type TabManagementDeps } from "./tabs";
@@ -73,11 +74,17 @@ export interface SessionStopResult {
/** Tab ids whose return path failed; those entries remain borrowed so
* shutdown can be retried without closing the Agent Window. */
return_failures?: Array<{ tab_id: number; code: string; message: string }>;
/**
* True when the Agent Window was *released* to the user (instead of being
* closed) because it still contained user-created tabs after the agent's own
* tabs were closed. Single-session scope.
*/
window_released?: boolean;
}
export interface SessionStopDeps {
signal?: AbortSignal;
cdp?: {
cdp?: Pick<CdpRunner, "releaseSessionTab"> & {
detachSession(sessionId: string): Promise<void>;
};
/**
@@ -87,6 +94,13 @@ export interface SessionStopDeps {
* without a real browser.
*/
tabManagement?: TabManagementDeps;
/**
* Read-only Chrome tabs API used to *query* remaining tabs before deciding
* whether to release the window. Kept separate from `tabManagement.tabs`
* (a `TabMutationApi` that has no `query`) so the mutation interface stays
* pure. Tests can inject a fake `ChromeTabsApi` too.
*/
tabsQuery?: ChromeTabsApi;
}
/**
@@ -152,8 +166,10 @@ export async function handleSessionStart(
* cleanly (review parity with M6).
* 3. Detach CDP sessions the extension still holds for this
* session (no-op if M6/M7 didn't attach to any tab).
* 4. Close the Agent Window. SessionManager.stop() removes the
* Chrome window and forgets the context.
* 4. Close the agent-created tabs and the home tab (design §3.2).
* 5. If any tab remains — those are user-created tabs — release the
* window to the user (dropOnly) instead of closing it, so user
* tabs survive. Otherwise close the window as before.
*
* Failures in step 1 keep the Agent Window open: a failed borrowed tab
* may still be there, so closing the window would risk losing user
@@ -188,15 +204,19 @@ export async function handleSessionStop(
const returnFailures: SessionStopResult["return_failures"] = [];
const borrowedIds = Array.from(ctx.borrowedTabs.keys());
for (const tabId of borrowedIds) {
// A close event may have removed this borrow while another tab returned.
if (!ctx.borrowedTabs.has(tabId)) continue;
try {
const tabManagement = {
...(deps.tabManagement ?? {}),
cdp: deps.cdp ?? deps.tabManagement?.cdp,
signal: deps.signal,
isAgentWindowId:
deps.tabManagement?.isAgentWindowId ??
((windowId: number) => manager.findByWindowId(windowId) !== null),
};
const outcome = await returnBorrowedTab(ctx, tabId, tabManagement);
if (!ctx.borrowedTabs.has(tabId)) continue;
if (typeof outcome === "object" && "code" in outcome) {
console.warn(`[bsk session_stop] auto-return failed for tab ${tabId}`, outcome);
returnFailures?.push({
@@ -209,6 +229,7 @@ export async function handleSessionStop(
ctx.borrowedTabs.delete(tabId);
}
} catch (err) {
if (!ctx.borrowedTabs.has(tabId)) continue;
const message = err instanceof Error ? err.message : String(err);
console.warn(`[bsk session_stop] auto-return threw for tab ${tabId}: ${message}`);
returnFailures?.push({
@@ -219,8 +240,15 @@ export async function handleSessionStop(
}
}
// A previously failed tab may have closed during a later return. Only
// confirmed close events remove borrows; ordinary move failures stay retryable.
const pendingReturnFailures = returnFailures.filter((failure) =>
ctx.borrowedTabs.has(failure.tab_id),
);
if (deps.signal?.aborted) {
const nonCancelFailures = returnFailures?.filter((failure) => failure.code !== "cancelled");
const nonCancelFailures = pendingReturnFailures.filter(
(failure) => failure.code !== "cancelled",
);
if (nonCancelFailures && nonCancelFailures.length > 0) {
return {
...(returnedTabIds.length > 0 ? { returned_tab_ids: returnedTabIds } : {}),
@@ -232,8 +260,8 @@ export async function handleSessionStop(
const result: SessionStopResult = {};
if (returnedTabIds.length > 0) result.returned_tab_ids = returnedTabIds;
if (returnFailures && returnFailures.length > 0) {
result.return_failures = returnFailures;
if (pendingReturnFailures.length > 0) {
result.return_failures = pendingReturnFailures;
// A failed return means at least one borrowed user tab may still be
// inside the Agent Window. Keep the session/window alive so the user
// can retry `bsk session stop` or explicitly `bsk tab return` after the
@@ -253,8 +281,66 @@ export async function handleSessionStop(
return { code: "cancelled", message: "session_stop aborted before window close" };
}
// Step 4: close the Agent Window and drop the context.
await manager.stop(params.session_id);
// Step 4: close every tab explicitly created by the agent, including the
// home tab. `tabsApi` is a
// TabMutationApi (remove only); `queryApi` is a separate read-only
// ChromeTabsApi. If neither is injected, we conservatively fall back to
// closing the window (see Step 5).
const tabsApi = deps.tabManagement?.tabs;
const queryApi = deps.tabsQuery;
if (tabsApi) {
// Close each agent-created tab that still exists.
const agentCreatedTabIds = Array.from(ctx.agentCreatedTabs);
for (const tabId of agentCreatedTabIds) {
try {
await tabsApi.remove(tabId);
ctx.agentCreatedTabs.delete(tabId);
} catch (err) {
// Tab may already be gone (closed by the user). Non-fatal.
console.warn(`[bsk session_stop] failed to close agent tab ${tabId}`, err);
}
}
}
// Step 5: decide whether to release (keep) the window or close it.
let shouldRelease = false;
if (queryApi) {
try {
const liveWindowTabs = await queryApi.query({ windowId: ctx.agentWindowId });
// Only genuine *user* tabs count toward keeping the window open.
// An agent tab that failed to close in Step 4 may still be present
// here; if we counted it as a reason to release (dropOnly), the
// window would be kept and that agent tab would leak (issue #57
// regression). Exclude any id still tracked in agentCreatedTabs.
const userTabs = liveWindowTabs.filter((t) => {
if (t.id === undefined) return false;
return !ctx.agentCreatedTabs.has(t.id);
});
const leakedAgentTabs = liveWindowTabs.filter(
(t) => t.id !== undefined && ctx.agentCreatedTabs.has(t.id),
);
if (leakedAgentTabs.length > 0) {
console.warn(
`[bsk session_stop] ${leakedAgentTabs.length} agent tab(s) failed to close; forcing window close instead of release`,
leakedAgentTabs.map((t) => t.id),
);
}
shouldRelease = userTabs.length > 0;
} catch {
// Query failed (e.g. window already gone) — conservatively close it.
shouldRelease = false;
}
}
if (shouldRelease) {
// Keep the window + its user tabs; only drop the session binding.
await manager.stop(params.session_id, { dropOnly: true });
result.window_released = true;
} else {
// Window is empty (or we couldn't verify state) — close it.
await manager.stop(params.session_id);
}
return result;
}
+1
View File
@@ -57,6 +57,7 @@ export interface CdpRunner {
getFrameGraph?(tabId: number): Promise<CdpFrameGraph>;
ensureAttachedToUrl?(tabId: number, expectedUrl: string | undefined): Promise<void>;
trackSessionTab?(sessionId: string, tabId: number): void;
releaseSessionTab?(sessionId: string, tabId: number): Promise<void>;
onEvent?(handler: (source: CdpDebuggee, method: string, params: unknown) => void): {
dispose(): void;
};
+46 -29
View File
@@ -10,7 +10,7 @@ import {
import type { SessionContext, SessionManager } from "@/session-manager/manager";
import type { RpcError } from "@/transport/types";
import { rpcError } from "./errors";
import { isRpcError, lookupSession } from "./shared";
import { type CdpRunner, isRpcError, lookupSession } from "./shared";
export type TabScope = "user" | "agent" | "all";
@@ -287,6 +287,10 @@ export interface TabManagementDeps {
approveBorrow?: BorrowConfirmationApprover;
/** Clears Agent-scoped overlays after a borrowed tab is returned. */
agentOverlayReset?: AgentOverlayResetApi;
/** Releases this session's CDP claim after a borrowed tab is returned. */
cdp?: Pick<CdpRunner, "releaseSessionTab">;
/** Runs after tab_return validation, before moving the borrowed tab. */
beforeReturn?: (sessionId: string, tabId: number) => Promise<void>;
/**
* Reports whether `windowId` is any live session's Agent Window.
* `tab_return`'s fallback window picker uses this to avoid moving a
@@ -369,6 +373,7 @@ function buildCreateProps(
* Returns the created tab on success, or an `RpcError` on failure.
*/
async function createTabAndCleanup(
ctx: SessionContext,
deps: TabManagementDeps,
createProps: chrome.tabs.CreateProperties,
): Promise<CreatedChromeTab | RpcError> {
@@ -381,25 +386,28 @@ async function createTabAndCleanup(
message: err instanceof Error ? err.message : String(err),
};
}
if (aborted(deps.signal, "tab_create")) {
// We already opened the tab; close it on abort so we don't leak.
if (typeof tab.id === "number") {
try {
await getTabsApi(deps).remove(tab.id);
} catch (cleanupErr) {
return rpcError(
"protocol_error",
"cleanup_failed",
`tab_create aborted but cleanup of tab ${tab.id} failed: ${describeError(cleanupErr)}`,
{ resource_type: "tab", resource_id: tab.id },
);
}
}
return { code: "cancelled", message: "tab_create aborted" };
}
if (typeof tab.id !== "number") {
return { code: "protocol_error", message: "chrome.tabs.create returned no tab id" };
}
// Claim the concrete id returned by Chrome. Ownership never depends on
// matching this request to an asynchronous onCreated event.
ctx.agentCreatedTabs.add(tab.id);
if (aborted(deps.signal, "tab_create")) {
try {
await getTabsApi(deps).remove(tab.id);
ctx.agentCreatedTabs.delete(tab.id);
} catch (cleanupErr) {
// Keep the claim when cleanup fails so session_stop can retry instead
// of releasing an agent-owned tab to the user.
return rpcError(
"protocol_error",
"cleanup_failed",
`tab_create aborted but cleanup of tab ${tab.id} failed: ${describeError(cleanupErr)}`,
{ resource_type: "tab", resource_id: tab.id },
);
}
return { code: "cancelled", message: "tab_create aborted" };
}
return { ...tab, id: tab.id };
}
@@ -426,7 +434,7 @@ export async function handleTabCreate(
const paramErr = validateTabCreateParams(params);
if (paramErr) return paramErr;
const tab = await createTabAndCleanup(deps, buildCreateProps(ctx, params));
const tab = await createTabAndCleanup(ctx, deps, buildCreateProps(ctx, params));
if (isRpcError(tab)) return tab;
return {
@@ -526,6 +534,9 @@ export async function handleTabClose(
}
try {
await getTabsApi(deps).remove(params.tab_id);
// Keep the tracking set accurate so session_stop won't try to close a
// tab that's already gone (design §3.1).
ctx.agentCreatedTabs.delete(params.tab_id);
} catch (err) {
return {
code: "protocol_error",
@@ -753,13 +764,6 @@ async function executeBorrowCore(
);
if (moveErr) return moveErr;
// Best-effort activation — failure is non-fatal.
try {
await p.tabsApi.update(p.tabId, { active: true });
} catch (err) {
console.debug("[bsk tab_borrow] activate after move failed", err);
}
return { originalWindowId, originalIndex };
}
@@ -823,6 +827,12 @@ export async function handleTabBorrow(
message: err instanceof Error ? err.message : String(err),
};
}
// Activate after commit so overlay/event observers see the tab as claimed.
try {
await tabsApi.update(params.tab_id, { active: true });
} catch (err) {
console.debug("[bsk tab_borrow] activate after move failed", err);
}
return {
tab_id: params.tab_id,
original_window_id: coreResult.originalWindowId,
@@ -939,16 +949,22 @@ async function cleanupUnusedFallbackWindow(
}
}
function resetAgentOverlaysInReturnedTab(
async function releaseReturnedTabState(
ctx: SessionContext,
tabId: number,
deps: TabManagementDeps,
): void {
): Promise<void> {
void getAgentOverlayResetApi(deps)
.resetAgentOverlays(tabId, ctx.sessionId)
.catch((err) => {
console.debug("[bsk tab_return] agent overlay reset failed", err);
});
try {
await deps.cdp?.releaseSessionTab?.(ctx.sessionId, tabId);
} catch (err) {
// A successful move must not be retried because debugger cleanup failed.
console.debug("[bsk tab_return] CDP release failed", err);
}
}
/**
@@ -1022,7 +1038,7 @@ export async function returnBorrowedTab(
});
const movedTab = Array.isArray(moved) ? moved[0] : moved;
const finalIndex = typeof movedTab?.index === "number" ? movedTab.index : targetIndex;
resetAgentOverlaysInReturnedTab(ctx, tabId, deps);
await releaseReturnedTabState(ctx, tabId, deps);
return {
tabId,
toWindowId: targetWindowId,
@@ -1064,7 +1080,7 @@ export async function returnBorrowedTab(
});
const movedTab = Array.isArray(moved) ? moved[0] : moved;
const finalIndex = typeof movedTab?.index === "number" ? movedTab.index : target.index;
resetAgentOverlaysInReturnedTab(ctx, tabId, deps);
await releaseReturnedTabState(ctx, tabId, deps);
return {
tabId,
toWindowId: target.windowId,
@@ -1110,6 +1126,7 @@ export async function handleTabReturn(
message: `tab_return: tab ${params.tab_id} is not borrowed by this session`,
};
}
await deps.beforeReturn?.(ctx.sessionId, params.tab_id);
const outcome = await returnBorrowedTab(ctx, params.tab_id, {
...deps,
isAgentWindowId:
@@ -1,5 +1,6 @@
import { describe, expect, it, vi } from "vitest";
import { OVERLAY_HOST_MARKER_ATTR, OVERLAY_HOST_NAME } from "../../../lib/overlay-bridge";
import type { CdpRunner } from "../../shared";
import { captureViewModel, collectOverlayExcludedBackendIds, probeHoverSurfaces } from "../capture";
// Minimal but format-accurate captureSnapshot reply: a body with one
@@ -210,26 +211,38 @@ function makeCdp(snapshot: unknown) {
if (method === "DOMSnapshot.enable") return {};
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 } };
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
};
}
if (method === "Runtime.evaluate") {
return {
result: {
value: {
controls: [
deepSerializedValue: {
type: "array",
value: [
{
state: "filled",
sensitive: true,
defaultValue: "hunter2",
value: "hunter2",
placeholder: "secret",
type: "array",
value: [
{ type: "node", value: { backendNodeId: 13 } },
{
type: "string",
value: JSON.stringify({
state: "filled",
sensitive: true,
placeholder: "secret",
}),
},
],
},
],
childFrames: [],
},
},
};
}
if (method === "Runtime.releaseObjectGroup") return {};
throw new Error(`unexpected ${method}`);
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
};
@@ -273,8 +286,11 @@ describe("captureViewModel", () => {
expect(cdp.send).toHaveBeenCalledWith(
4,
"Runtime.evaluate",
expect.objectContaining({ returnByValue: true }),
expect.objectContaining({
serializationOptions: expect.objectContaining({ serialization: "deep" }),
}),
);
expect(cdp.send).toHaveBeenCalledWith(4, "Runtime.releaseObjectGroup", expect.anything());
expect(cdp.send).not.toHaveBeenCalledWith(4, "DOM.resolveNode", expect.anything());
expect(cdp.send).not.toHaveBeenCalledWith(4, "Runtime.callFunctionOn", expect.anything());
});
@@ -287,6 +303,8 @@ describe("captureViewModel", () => {
if (method === "DOMSnapshot.captureSnapshot") return hoverTriggerSnapshotReply();
if (method === "Page.getLayoutMetrics") {
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
};
}
@@ -350,6 +368,8 @@ describe("captureViewModel", () => {
if (method === "DOMSnapshot.captureSnapshot") return twoHoverTriggerSnapshotReply();
if (method === "Page.getLayoutMetrics") {
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
};
}
@@ -398,6 +418,8 @@ describe("captureViewModel", () => {
if (method === "DOMSnapshot.captureSnapshot") return nestedHoverTriggerSnapshotReply();
if (method === "Page.getLayoutMetrics") {
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
};
}
@@ -516,6 +538,9 @@ describe("captureViewModel", () => {
strings: S,
documents: [
{
scrollOffsetX: 0,
scrollOffsetY: 0,
frameId: "main",
nodes: {
parentIndex: [-1, 0, 1, 0, 3, 4],
nodeName: [i("html"), i("body"), i("div"), i(OVERLAY_HOST_NAME), i("div"), i("button")],
@@ -546,8 +571,12 @@ describe("captureViewModel", () => {
],
};
const { excludedBackendNodeIds } = await captureViewModel(makeCdp(snapshot), 4);
const { excludedBackendNodeIds, frameExcludedBackendNodeIds } = await captureViewModel(
makeCdp(snapshot),
4,
);
expect(excludedBackendNodeIds).toEqual(new Set([13, 14, 15]));
expect(frameExcludedBackendNodeIds?.get("main")).toEqual(excludedBackendNodeIds);
});
it("excludes overlay host matched by tag name only", async () => {
@@ -731,6 +760,8 @@ describe("captureViewModel", () => {
if (method === "DOMSnapshot.captureSnapshot") return fakeSnapshotReply();
if (method === "Page.getLayoutMetrics") {
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 200 },
};
}
@@ -743,7 +774,46 @@ describe("captureViewModel", () => {
expect(div?.rect).toMatchObject({ y: 0, h: 600 });
});
it("normalizes device-pixel bounds by devicePixelRatio", async () => {
it("normalizes snapshot-owned scroll before using a stale CSS metrics fallback", async () => {
const snapshot = {
strings: ["html", "button"],
documents: [
{
frameId: "root",
scrollOffsetX: 40,
scrollOffsetY: 200,
nodes: {
parentIndex: [-1, 0],
nodeName: [0, 1],
backendNodeId: [10, 11],
attributes: [[], []],
},
layout: { nodeIndex: [1], bounds: [[200, 800, 240, 80]] },
},
],
};
const cdp: CdpRunner = {
send: vi.fn(async (_tab: number, method: string) => {
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
if (method === "Page.getLayoutMetrics")
return {
visualViewport: { clientWidth: 2000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 999, pageY: 999 },
};
return {};
}) as CdpRunner["send"],
};
const captured = await captureViewModel(cdp, 4);
expect(captured.nodes.find((node) => node.backendNodeId === 11)).toMatchObject({
localRect: { x: 80, y: 300, w: 120, h: 40 },
rect: { x: 80, y: 300, w: 120, h: 40 },
rendered: true,
});
expect(captured.frameGeometryIssues).toEqual([]);
});
it("normalizes raw snapshot bounds before viewport clipping", async () => {
const S = ["html", "body", "div", "position", "fixed", "static", "pointer-events", "auto"];
const i = (s: string) => S.indexOf(s);
const snapshot = {
@@ -777,6 +847,8 @@ describe("captureViewModel", () => {
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
if (method === "Page.getLayoutMetrics") {
return {
visualViewport: { clientWidth: 2000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
layoutViewport: { clientWidth: 2000, clientHeight: 1600 },
};
@@ -791,6 +863,12 @@ describe("captureViewModel", () => {
w: 1000,
h: 800,
});
expect(nodes.find((n) => n.backendNodeId === 12)?.localRect).toEqual({
x: 0,
y: 0,
w: 1000,
h: 800,
});
});
it("extracts textContent from #text child nodes via nodeValue", async () => {
@@ -846,6 +924,8 @@ describe("captureViewModel", () => {
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
if (method === "Page.getLayoutMetrics") {
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
};
}
@@ -936,9 +1016,17 @@ describe("captureViewModel", () => {
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
if (method === "Page.getLayoutMetrics") {
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
};
}
if (method === "DOM.getBoxModel")
return { model: { content: [100, 300, 900, 300, 900, 700, 100, 700] } };
if (method === "DOM.resolveNode") return { object: { objectId: "owner" } };
if (method === "Runtime.callFunctionOn")
return { result: { value: { width: 800, height: 400 } } };
if (method === "Runtime.releaseObject") return {};
throw new Error(method);
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
};
@@ -968,7 +1056,7 @@ describe("captureViewModel", () => {
expect(frameParentIds?.get("child-frame")).toBe("main-frame");
});
it("captures iframe documents without requiring owner-frame geometry", async () => {
it("projects iframe documents independently of owner snapshot geometry", async () => {
const S = [
"html",
"body",
@@ -1026,9 +1114,17 @@ describe("captureViewModel", () => {
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
if (method === "Page.getLayoutMetrics") {
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
};
}
if (method === "DOM.getBoxModel")
return { model: { content: [100, 300, 900, 300, 900, 700, 100, 700] } };
if (method === "DOM.resolveNode") return { object: { objectId: "owner" } };
if (method === "Runtime.callFunctionOn")
return { result: { value: { width: 800, height: 400 } } };
if (method === "Runtime.releaseObject") return {};
throw new Error(method);
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
};
@@ -1037,7 +1133,7 @@ describe("captureViewModel", () => {
const input = iframeNodes.get(13)?.find((n) => n.backendNodeId === 101);
expect(input?.localRect).toEqual({ x: 20, y: 20, w: 120, h: 60 });
expect(input?.rect).toBeNull();
expect(input?.rect).toEqual({ x: 120, y: 320, w: 120, h: 60 });
});
it("recursively normalizes nested iframe sub-documents", async () => {
@@ -1079,6 +1175,8 @@ describe("captureViewModel", () => {
},
},
{
scrollOffsetX: 0,
scrollOffsetY: 0,
nodes: {
parentIndex: [-1, 0, 1],
nodeName: [i("html"), i("body"), i("iframe")],
@@ -1100,6 +1198,8 @@ describe("captureViewModel", () => {
},
},
{
scrollOffsetX: 0,
scrollOffsetY: 0,
nodes: {
parentIndex: [-1, 0],
nodeName: [i("body"), i("input")],
@@ -1116,16 +1216,45 @@ describe("captureViewModel", () => {
],
};
const cdp = {
send: vi.fn(async (_t: number, method: string) => {
if (method === "DOMSnapshot.enable") return {};
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
if (method === "Page.getLayoutMetrics") {
return {
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
};
}
throw new Error(method);
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
send: vi.fn(
async (
_t: number,
method: string,
params?: { backendNodeId?: number; objectId?: string },
) => {
if (method === "DOMSnapshot.enable") return {};
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
if (method === "Page.getLayoutMetrics") {
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
};
}
if (method === "DOM.getBoxModel")
return {
model: {
content:
params?.backendNodeId === 13
? [10, 20, 310, 20, 310, 220, 10, 220]
: [15, 26, 115, 26, 115, 106, 15, 106],
},
};
if (method === "DOM.resolveNode")
return { object: { objectId: String(params?.backendNodeId) } };
if (method === "Runtime.callFunctionOn")
return {
result: {
value:
params?.objectId === "13"
? { width: 300, height: 200 }
: { width: 100, height: 80 },
},
};
if (method === "Runtime.releaseObject") return {};
throw new Error(method);
},
) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
};
const { iframeNodes } = await captureViewModel(cdp, 4);
@@ -1139,7 +1268,7 @@ describe("captureViewModel", () => {
expect(input?.rect).toEqual({ x: 16, y: 28, w: 40, h: 20 });
});
it("normalizes bounds then subtracts CSS scroll at dpr>1", async () => {
it("subtracts CSS scroll once without dividing snapshot bounds by a metrics ratio", async () => {
const S = ["html", "body", "div", "position", "fixed", "static", "pointer-events", "auto"];
const i = (s: string) => S.indexOf(s);
const snapshot = {
@@ -1173,6 +1302,8 @@ describe("captureViewModel", () => {
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
if (method === "Page.getLayoutMetrics") {
return {
visualViewport: { clientWidth: 2000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 100 },
layoutViewport: { clientWidth: 2000, clientHeight: 1600 },
};
@@ -1206,3 +1337,200 @@ describe("captureViewModel", () => {
expect(excluded).toEqual(new Set([200, 201, 202]));
});
});
// Six sibling frames and one nested frame exercise scheduling through the real capture entry.
function siblingCaptureFixture(
beforeReply: (method: string, params: Record<string, unknown>) => Promise<void> = async () => {},
) {
const document = (id: number, owners: number[], childIndexes: number[]) => ({
scrollOffsetX: 0,
scrollOffsetY: 0,
frameId: `frame-${id}`,
nodes: {
parentIndex: [-1, ...owners.map(() => 0)],
nodeName: [0, ...owners.map(() => 1)],
backendNodeId: [1000 + id, ...owners],
attributes: [[], ...owners.map(() => [])],
contentDocumentIndex: { index: owners.map((_, i) => i + 1), value: childIndexes },
},
layout: {
nodeIndex: [0, ...owners.map((_, i) => i + 1)],
bounds: [[0, 0, 200, 100], ...owners.map(() => [0, 0, 200, 100])],
},
});
const snapshot = {
strings: ["body", "iframe"],
documents: [
document(0, [100, 101, 102, 103, 104, 105], [1, 2, 3, 4, 5, 6]),
document(1, [200], [7]),
...Array.from({ length: 6 }, (_, i) => document(i + 2, [], [])),
],
};
let active = 0;
let peak = 0;
const send = vi.fn(async (_tabId: number, method: string, params: object = {}) => {
const args = params as Record<string, unknown>;
active++;
peak = Math.max(peak, active);
try {
await beforeReply(method, args);
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
if (method === "Page.getLayoutMetrics")
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 },
};
if (method === "DOM.getBoxModel")
return { model: { content: [0, 0, 200, 0, 200, 100, 0, 100] } };
if (method === "DOM.resolveNode") return { object: { objectId: String(args.backendNodeId) } };
if (method === "Runtime.callFunctionOn")
return { result: { value: { width: 200, height: 100 } } };
return {};
} finally {
active--;
}
});
return {
cdp: { send: send as CdpRunner["send"] },
send,
peak: () => peak,
active: () => active,
measured: () =>
send.mock.calls
.filter(([, method]) => method === "DOM.getBoxModel")
.map(([, , params]) => (params as { backendNodeId: number }).backendNodeId),
};
}
describe("sibling frame measurement scheduling", () => {
it("batches same-target owner sizes without changing projected results", async () => {
const fixture = siblingCaptureFixture();
const original = fixture.cdp.send;
const methods: string[] = [];
fixture.cdp.send = async (tabId, method, params) => {
methods.push(method);
if (method === "Runtime.evaluate")
return {
result: {
deepSerializedValue: {
type: "array",
value: [100, 101, 102, 103, 104, 105, 200].map((backendNodeId) => ({
type: "array",
value: [
{ type: "node", value: { backendNodeId } },
{ type: "string", value: JSON.stringify({ width: 200, height: 100 }) },
],
})),
},
},
} as never;
return original(tabId, method, params);
};
const captured = await captureViewModel(fixture.cdp, 4);
expect(captured.frameGeometryIssues).toEqual([]);
expect([...captured.iframeNodes.keys()]).toEqual([100, 200, 101, 102, 103, 104, 105]);
expect(captured.iframeNodes.get(200)?.[0].rect).toEqual({ x: 0, y: 0, w: 200, h: 100 });
expect(methods.filter((method) => method === "DOM.getBoxModel")).toHaveLength(7);
expect(methods.filter((method) => method === "Runtime.evaluate")).toHaveLength(1);
expect(methods.filter((method) => method === "Runtime.releaseObjectGroup")).toHaveLength(1);
expect(methods).not.toContain("DOM.resolveNode");
expect(methods).not.toContain("Runtime.callFunctionOn");
});
it("bounds concurrent reads, fills free slots and preserves depth-first output despite reordered replies", async () => {
const pending = new Map<number, () => void>();
const fixture = siblingCaptureFixture(async (method, params) => {
if (method === "DOM.getBoxModel" && Number(params.backendNodeId) < 200)
await new Promise<void>((resolve) => pending.set(Number(params.backendNodeId), resolve));
});
const capture = captureViewModel(fixture.cdp, 4);
await vi.waitFor(() => expect(pending.size).toBe(4));
expect(fixture.measured()).toEqual([100, 101, 102, 103]);
pending.get(103)!();
await vi.waitFor(() => expect(pending.has(104)).toBe(true));
pending.get(104)!();
await vi.waitFor(() => expect(pending.has(105)).toBe(true));
expect(fixture.measured()).not.toContain(200);
for (const id of [105, 102, 101, 100]) pending.get(id)!();
const captured = await capture;
expect(captured.frameGeometryIssues).toEqual([]);
expect(fixture.peak()).toBe(4);
expect(fixture.active()).toBe(0);
expect(fixture.measured()).toEqual([100, 101, 102, 103, 104, 105, 200]);
expect([...captured.iframeNodes.keys()]).toEqual([100, 200, 101, 102, 103, 104, 105]);
expect(captured.iframeNodes.get(200)?.[0].rect).toEqual({ x: 0, y: 0, w: 200, h: 100 });
for (const method of [
"DOM.getBoxModel",
"DOM.resolveNode",
"Runtime.callFunctionOn",
"Runtime.releaseObject",
])
expect(fixture.send.mock.calls.filter(([, name]) => name === method)).toHaveLength(7);
});
it("does not measure descendants of a failed owner and retains sibling geometry and local nodes", async () => {
const fixture = siblingCaptureFixture(async (method, params) => {
if (method === "DOM.resolveNode" && params.backendNodeId === 100)
throw new Error("owner replaced");
});
const captured = await captureViewModel(fixture.cdp, 4);
expect(captured.frameGeometryIssues).toEqual([
{
status: "unavailable",
source: { target: { tabId: 4 }, frameId: "frame-1" },
ownerBackendNodeId: 100,
},
{
status: "blocked",
source: { target: { tabId: 4 }, frameId: "frame-7" },
cause: captured.frameGeometryIssues?.[0],
},
]);
expect(fixture.measured()).not.toContain(200);
expect(captured.iframeNodes.get(200)?.[0]).toMatchObject({
tag: "body",
rect: null,
localRect: { x: 0, y: 0, w: 200, h: 100 },
});
expect(captured.iframeNodes.get(101)?.[0].rect).toEqual({ x: 0, y: 0, w: 200, h: 100 });
expect([...captured.iframeNodes.keys()]).toEqual([100, 200, 101, 102, 103, 104, 105]);
});
it("stops scheduling on cancellation and waits for resolved objects to be released", async () => {
const controller = new AbortController();
const resolutions = new Map<string, () => void>();
const releases = new Map<string, () => void>();
const fixture = siblingCaptureFixture(async (method, params) => {
if (method === "DOM.resolveNode")
await new Promise<void>((resolve) =>
resolutions.set(String(params.backendNodeId), resolve),
);
if (method === "Runtime.releaseObject")
await new Promise<void>((resolve) => releases.set(String(params.objectId), resolve));
});
let settled = false;
const capture = captureViewModel(fixture.cdp, 4, { signal: controller.signal });
const rejected = expect(capture).rejects.toMatchObject({ name: "AbortError" });
void capture.then(
() => {
settled = true;
},
() => {
settled = true;
},
);
await vi.waitFor(() => expect(resolutions.size).toBe(4));
controller.abort();
for (const resolve of resolutions.values()) resolve();
await vi.waitFor(() => expect(releases.size).toBe(4));
expect(settled).toBe(false);
expect(fixture.measured()).toEqual([100, 101, 102, 103]);
for (const release of releases.values()) release();
await rejected;
expect(fixture.active()).toBe(0);
expect(fixture.send.mock.calls.some(([, method]) => method === "Runtime.callFunctionOn")).toBe(
false,
);
});
});
@@ -0,0 +1,127 @@
import { describe, expect, it, vi } from "vitest";
import { OVERLAY_HOST_MARKER_ATTR } from "@/lib/overlay-bridge";
import { createCaptureCheckpoint } from "../capture-abort";
import { buildDocumentIndex, type DecodedNode } from "../facts";
import { decodeDocument, REQUESTED_STYLES } from "../snapshot";
function node(id: number, parent: number | null): DecodedNode {
return {
backendNodeId: id,
parentBackendNodeId: parent,
tag: "div",
attrs: {},
paintOrder: 0,
position: "static",
pointerEvents: "auto",
};
}
describe("document facts", () => {
it.each([
1_000, 10_000, 100_000,
])("indexes deep and wide inputs of %i without copying ancestors", async (count) => {
for (const wide of [false, true]) {
let reads = 0;
const input = Array.from({ length: count }, (_, i) => {
const item = node(i, i ? (wide ? 0 : i - 1) : null);
if (i === 0) item.attrs[OVERLAY_HOST_MARKER_ATTR] = "";
Object.defineProperty(item, "parentBackendNodeId", {
get: () => {
reads++;
return i ? (wide ? 0 : i - 1) : null;
},
});
return item;
}).reverse(); // parent need not precede child
const index = await buildDocumentIndex(input);
expect(index.nodes.size).toBe(count);
expect(index.excludedBackendNodeIds.size).toBe(count);
expect(reads).toBeLessThanOrEqual(count * 8);
expect(index.nodes.get(count - 1)).toBe(input[0]);
}
});
it("handles cycles and orphans and propagates overlay through shadow roots", async () => {
const host = { ...node(1, null), attrs: { [OVERLAY_HOST_MARKER_ATTR]: "" } };
const input = [
node(3, 2),
{ ...node(2, 1), tag: "#document-fragment" },
host,
node(4, 99),
node(5, 6),
node(6, 5),
node(7, 6),
];
const index = await buildDocumentIndex(input);
expect([...index.excludedBackendNodeIds].sort()).toEqual([1, 2, 3]);
expect(index.nodes.size).toBe(input.length);
});
it("does not propagate a descendant overlay backwards into a cycle", async () => {
const overlay = { ...node(3, 1), attrs: { [OVERLAY_HOST_MARKER_ATTR]: "" } };
const index = await buildDocumentIndex([overlay, node(1, 2), node(2, 1)]);
expect([...index.excludedBackendNodeIds]).toEqual([3]);
});
it("decodes current observation bounds, styles and attributes without projecting coordinates", async () => {
const strings = ["div", "aria-hidden", "true", "inert", "", "visible", "0", "static", "auto"];
const styles = REQUESTED_STYLES.map((style) =>
style === "visibility" ? 5 : style === "opacity" ? 6 : style === "position" ? 7 : 8,
);
const { nodes } = await decodeDocument(
{
nodes: { backendNodeId: [1], parentIndex: [-1], nodeName: [0], attributes: [[1, 2, 3, 4]] },
layout: { nodeIndex: [0], bounds: [[10, 20, 120, 40]], styles: [styles] },
},
strings,
);
expect(nodes[0].attrs).toEqual({ "aria-hidden": "true", inert: "" });
expect(nodes[0].layout).toEqual({
boundsSpace: "snapshot-document-layout",
bounds: [10, 20, 120, 40],
styles: {
position: "static",
"pointer-events": "auto",
cursor: "auto",
visibility: "visible",
opacity: "0",
},
});
expect(nodes[0]).not.toHaveProperty("rect");
expect(nodes[0]).not.toHaveProperty("excluded");
});
it("keeps short blocks synchronous and checks cancellation after yielding", async () => {
const clock = vi.spyOn(performance, "now").mockReturnValue(0);
try {
const controller = new AbortController();
const checkpoint = createCaptureCheckpoint(controller.signal);
for (let i = 0; i < 20; i++) expect(checkpoint()).toBeUndefined();
clock.mockReturnValue(9);
const pending = checkpoint();
expect(pending).toBeInstanceOf(Promise);
controller.abort();
await expect(pending).rejects.toMatchObject({ name: "AbortError" });
} finally {
clock.mockRestore();
}
});
it("checks cancellation at bounded indexing blocks", async () => {
const controller = new AbortController();
const input = Array.from({ length: 100_000 }, (_, i) => node(i, i ? i - 1 : null));
let reads = 0;
for (const item of input)
Object.defineProperty(item, "attrs", {
get: () => {
reads++;
if (reads === 100) controller.abort();
return {};
},
});
await expect(buildDocumentIndex(input, controller.signal)).rejects.toMatchObject({
name: "AbortError",
});
expect(reads).toBeLessThanOrEqual(356);
});
});
@@ -0,0 +1,298 @@
import { describe, expect, it, vi } from "vitest";
import { type CdpRunner, cdpRunnerForTarget } from "../../shared";
import { enrichFormControlStates } from "../form-capture";
import { decodeDocument } from "../snapshot";
interface Control {
id: number;
value: string;
type?: string;
}
function snapshot(groups: Control[][]) {
const strings: string[] = [];
const str = (value: string) => {
let index = strings.indexOf(value);
if (index < 0) index = strings.push(value) - 1;
return index;
};
const documents = groups.map((controls, groupIndex) => ({
frameId: `frame-${groupIndex}`,
nodes: {
parentIndex: [-1, 0, ...controls.map(() => 1)],
nodeName: [str("html"), str("body"), ...controls.map(() => str("input"))],
backendNodeId: [8000 + groupIndex * 2, 8001 + groupIndex * 2, ...controls.map((c) => c.id)],
attributes: [
[],
[],
...controls.map((c) => [
str("type"),
str(c.type ?? "text"),
str("placeholder"),
str("snapshot placeholder"),
]),
],
inputValue: {
index: controls.map((_, i) => i + 2),
value: controls.map((c) => str(c.value)),
},
contentDocumentIndex: { index: [] as number[], value: [] as number[] },
},
}));
const root = documents[0].nodes;
for (let i = 1; i < documents.length; i++) {
root.contentDocumentIndex.index.push(root.backendNodeId.length);
root.contentDocumentIndex.value.push(i);
root.parentIndex.push(1);
root.nodeName.push(str("iframe"));
root.backendNodeId.push(9000 + i);
root.attributes.push([]);
}
return { strings, documents };
}
function state(value: string) {
return {
value,
defaultValue: "",
placeholder: "live placeholder",
state: value ? "filled" : "empty",
sensitive: false,
};
}
function reply(entries: Array<[number, unknown]>) {
return {
result: {
deepSerializedValue: {
type: "array",
value: entries.map(([backendNodeId, fields]) => ({
type: "array",
value: [
{ type: "node", value: { backendNodeId } },
{ type: "string", value: JSON.stringify(fields) },
],
})),
},
},
};
}
async function fakeCdp(
groups: Control[][],
evaluate: (params: Record<string, unknown>) => unknown,
) {
const snap = snapshot(groups);
const send = vi.fn(async (_tabId: number, method: string, params?: object) => {
if (method === "Runtime.evaluate") return evaluate(params as Record<string, unknown>);
return {};
});
const nodes = await decodeControls(snap);
return { send, cdp: { send: send as CdpRunner["send"] }, nodes };
}
async function decodeControls(snap: ReturnType<typeof snapshot>) {
return Promise.all(
snap.documents.map(async (doc) =>
(await decodeDocument(doc, snap.strings)).nodes.map((node) => ({ ...node, rect: null })),
),
);
}
describe("form state identity", () => {
it("matches reordered controls by identity and preserves controls missing from the batch", async () => {
// The snapshot includes a shadow input. The runtime query skips it,
// encounters a new input, and sees the remaining inputs in a new order.
const { cdp, nodes } = await fakeCdp(
[
[
{ id: 10, value: "" },
{ id: 11, value: "old email" },
{ id: 12, value: "old city" },
],
],
() =>
reply([
[99, state("new control")],
[12, state("上海")],
[11, state("alice@example.com")],
]),
);
await enrichFormControlStates(cdp, 7, nodes);
expect(nodes[0].find((n) => n.backendNodeId === 10)).toMatchObject({
formValue: "",
formState: "empty",
formPlaceholder: "snapshot placeholder",
});
expect(nodes[0].find((n) => n.backendNodeId === 11)).toMatchObject({
formValue: "alice@example.com",
});
expect(nodes[0].find((n) => n.backendNodeId === 12)).toMatchObject({ formValue: "上海" });
expect(nodes[0].some((n) => n.backendNodeId === 99)).toBe(false);
});
it("does not shift state between frames when one frame is absent from the runtime batch", async () => {
const { cdp, nodes } = await fakeCdp(
[
[],
[{ id: 11, value: "inaccessible frame" }],
[{ id: 21, value: "old second" }],
[{ id: 31, value: "old third" }],
],
() =>
reply([
[31, state("third frame")],
[21, state("second frame")],
]),
);
await enrichFormControlStates(cdp, 7, nodes);
const controls = nodes.slice(1).flat();
expect(controls.find((n) => n.backendNodeId === 11)?.formValue).toBe("inaccessible frame");
expect(controls.find((n) => n.backendNodeId === 21)?.formValue).toBe("second frame");
expect(controls.find((n) => n.backendNodeId === 31)?.formValue).toBe("third frame");
});
it("keeps identical backend ids in different OOPIF targets separate", async () => {
const sendToTarget = vi.fn(async (target, method) => {
if (method === "Runtime.evaluate") return reply([[11, state(target.sessionId)]]);
return {};
});
const send = vi.fn(async () => {
throw new Error("unexpected top-level target");
});
const cdp = { send, sendToTarget } as unknown as CdpRunner;
for (const sessionId of ["left-frame", "right-frame"]) {
const nodes = await decodeControls(snapshot([[{ id: 11, value: "old" }]]));
await enrichFormControlStates(cdpRunnerForTarget(cdp, { tabId: 7, sessionId }), 7, nodes);
expect(nodes[0].find((n) => n.backendNodeId === 11)?.formValue).toBe(sessionId);
expect(sendToTarget).toHaveBeenCalledWith(
{ tabId: 7, sessionId },
"Runtime.releaseObjectGroup",
expect.anything(),
);
}
expect(send).not.toHaveBeenCalled();
});
it.each([
"unsupported",
"legacy",
"malformed",
"invalid-state",
])("preserves snapshot data when enrichment is %s", async (mode) => {
const { cdp, send, nodes } = await fakeCdp([[{ id: 11, value: "snapshot value" }]], () => {
if (mode === "unsupported") throw new Error("serialization unsupported");
if (mode === "legacy") return { result: { value: { controls: [state("wrong value")] } } };
const result = reply([
[11, mode === "invalid-state" ? { value: "wrong value" } : state("wrong value")],
]);
if (mode === "malformed")
result.result.deepSerializedValue.value[0].value[1].value = "not json";
return result;
});
expect(await enrichFormControlStates(cdp, 7, nodes)).toBe(mode !== "unsupported");
expect(nodes[0].find((n) => n.backendNodeId === 11)).toMatchObject({
formValue: "snapshot value",
formPlaceholder: "snapshot placeholder",
});
expect(send).toHaveBeenCalledWith(7, "Runtime.releaseObjectGroup", expect.anything());
});
it("releases the batch's remote objects when capture is cancelled", async () => {
const controller = new AbortController();
const { cdp, send, nodes } = await fakeCdp([[{ id: 11, value: "old" }]], () => {
controller.abort();
return reply([[11, state("new")]]);
});
await expect(enrichFormControlStates(cdp, 7, nodes, controller.signal)).rejects.toMatchObject({
name: "AbortError",
});
expect(send).toHaveBeenCalledWith(7, "Runtime.releaseObjectGroup", expect.anything());
});
it("does not report success when cancelled during object cleanup", async () => {
const controller = new AbortController();
const nodes = await decodeControls(snapshot([[{ id: 11, value: "old" }]]));
const cdp: CdpRunner = {
send: vi.fn(async (_tabId, method) => {
if (method === "Runtime.evaluate") return reply([[11, state("new")]]);
if (method === "Runtime.releaseObjectGroup") controller.abort();
return {};
}) as CdpRunner["send"],
};
await expect(enrichFormControlStates(cdp, 7, nodes, controller.signal)).rejects.toMatchObject({
name: "AbortError",
});
expect(cdp.send).toHaveBeenCalledWith(7, "Runtime.releaseObjectGroup", expect.anything());
});
it("skips entries without node identity while still applying valid entries", async () => {
const { cdp, nodes } = await fakeCdp(
[
[
{ id: 11, value: "snapshot value" },
{ id: 12, value: "old value" },
],
],
() => {
const result = reply([
[11, state("wrong value")],
[12, state("live value")],
]);
const element = result.result.deepSerializedValue.value[0].value[0];
delete (element.value as { backendNodeId?: number }).backendNodeId;
return result;
},
);
await enrichFormControlStates(cdp, 7, nodes);
expect(nodes[0].find((n) => n.backendNodeId === 11)?.formValue).toBe("snapshot value");
expect(nodes[0].find((n) => n.backendNodeId === 12)?.formValue).toBe("live value");
});
it.each([
"password",
"text",
])("redacts a sensitive %s control after identity matching", async (type) => {
const { cdp, nodes } = await fakeCdp([[{ id: 11, value: "secret", type }]], () =>
reply([[11, { ...state("secret"), sensitive: true }]]),
);
await enrichFormControlStates(cdp, 7, nodes);
const control = nodes[0].find((n) => n.backendNodeId === 11);
expect(control?.formState).toBe("filled");
expect(control?.formValue).toBeUndefined();
expect(control?.formDefaultValue).toBeUndefined();
expect(control?.attrs.value).toBeUndefined();
});
it("keeps one bounded batch without per-control CDP queries", async () => {
const controls = Array.from({ length: 251 }, (_, i) => ({ id: i + 1, value: "old" }));
const elements = controls.map((c) => ({
backendNodeId: c.id,
tagName: "INPUT",
type: "text",
value: `live ${c.id}`,
defaultValue: "",
placeholder: "",
}));
const { cdp, send, nodes } = await fakeCdp([controls], (params) => {
const doc = {
querySelectorAll: (selector: string) =>
selector === "input,textarea,select" ? elements : [],
};
const entries = new Function("document", `return ${params.expression}`)(doc) as Array<
[{ backendNodeId: number }, string]
>;
expect(entries).toHaveLength(250);
return reply(entries.map(([element, json]) => [element.backendNodeId, JSON.parse(json)]));
});
await enrichFormControlStates(cdp, 7, nodes);
expect(nodes[0].find((n) => n.backendNodeId === 250)?.formValue).toBe("live 250");
expect(nodes[0].find((n) => n.backendNodeId === 251)?.formValue).toBe("old");
expect(send.mock.calls.filter(([, method]) => method === "Runtime.evaluate")).toHaveLength(1);
expect(
send.mock.calls.filter(([, method]) => method === "Runtime.releaseObjectGroup"),
).toHaveLength(1);
expect(send).not.toHaveBeenCalledWith(7, "DOM.resolveNode", expect.anything());
expect(send).not.toHaveBeenCalledWith(7, "Runtime.callFunctionOn", expect.anything());
});
});
@@ -1,4 +1,5 @@
import { describe, expect, it, vi } from "vitest";
import { OVERLAY_HOST_MARKER_ATTR } from "@/lib/overlay-bridge";
import type { CdpRunner } from "../../shared";
import type { CapturedNode, CapturedViewModel } from "../capture";
import { captureFrameData } from "../frame-capture";
@@ -25,12 +26,14 @@ function childSnapshot(frameId: string, backendNodeId: number) {
strings,
documents: [
{
scrollOffsetX: 0,
scrollOffsetY: 0,
frameId,
nodes: {
parentIndex: [-1, 0],
nodeName: [1, 2],
backendNodeId: [backendNodeId - 1, backendNodeId],
attributes: [[], []],
attributes: [[], []] as number[][],
},
layout: {
nodeIndex: [0, 1],
@@ -50,6 +53,72 @@ function childSnapshot(frameId: string, backendNodeId: number) {
}
describe("captureFrameData", () => {
it("retains nested owner failures when merging a captured OOPIF", async () => {
const owner = ownerNode(10, 50);
const captured: CapturedViewModel = {
nodes: [owner],
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
frameNodes: new Map([["main", [owner]]]),
rootFrameId: "main",
excludedBackendNodeIds: new Set(),
};
const child = childSnapshot("child", 101);
const nested = childSnapshot("nested", 201);
const document = child.documents[0];
const snapshot = {
strings: child.strings,
documents: [
{
...document,
nodes: { ...document.nodes, contentDocumentIndex: { index: [1], value: [1] } },
},
nested.documents[0],
],
};
const reply = async (_target: unknown, method: string) => {
if (method === "Page.getLayoutMetrics")
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 300, clientHeight: 200 },
};
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
if (method === "DOM.getBoxModel")
return { model: { content: [50, 100, 350, 100, 350, 300, 50, 300] } };
if (method === "DOM.resolveNode") throw new Error("nested owner replaced");
return {};
};
const cdp: CdpRunner = {
send: vi.fn(reply) as CdpRunner["send"],
sendToTarget: vi.fn(reply) as NonNullable<CdpRunner["sendToTarget"]>,
getFrameGraph: vi.fn(async () => ({
rootFrameId: "main",
frames: [
{ frameId: "main", target: { tabId: 4 } },
{
frameId: "child",
parentFrameId: "main",
ownerBackendNodeId: 10,
target: { tabId: 4, sessionId: "child-session" },
},
],
})),
};
await captureFrameData(cdp, 4, captured);
expect(captured.frameGeometryIssues).toEqual([
{
status: "unavailable",
source: { target: { tabId: 4, sessionId: "child-session" }, frameId: "nested" },
ownerBackendNodeId: 101,
},
]);
expect(captured.frameNodes?.get("nested")?.[1]).toMatchObject({
rect: null,
localRect: { x: 10, y: 20, w: 100, h: 40 },
});
});
it("captures and positions multiple OOPIF documents missing from the root snapshot", async () => {
const leftOwner = ownerNode(10, 50);
const rightOwner = ownerNode(20, 500);
@@ -64,13 +133,23 @@ describe("captureFrameData", () => {
};
const sendToTarget = vi.fn(async (target, method) => {
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 300, clientHeight: 200, pageX: 0, pageY: 0 } };
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 300, clientHeight: 200, pageX: 0, pageY: 0 },
};
}
if (method === "DOMSnapshot.enable" || method === "Accessibility.enable") return {};
if (method === "DOMSnapshot.captureSnapshot") {
return target.sessionId === "left-session"
? childSnapshot("left", 101)
: childSnapshot("right", 201);
if (target.sessionId !== "left-session") return childSnapshot("right", 201);
const snapshot = childSnapshot("left", 101);
const marker = snapshot.strings.push(OVERLAY_HOST_MARKER_ATTR) - 1;
const nodes = snapshot.documents[0].nodes;
nodes.backendNodeId.push(201); // Same id as a real button in the other target.
nodes.parentIndex.push(0);
nodes.nodeName.push(2);
nodes.attributes.push([marker, -1]);
return snapshot;
}
if (method === "Accessibility.getFullAXTree") return { nodes: [] };
throw new Error(`unexpected ${method}`);
@@ -85,7 +164,11 @@ describe("captureFrameData", () => {
return { model: { content: [x, 100, x + 300, 100, x + 300, 300, x, 300] } };
}
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } };
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 },
};
}
throw new Error(`unexpected root ${method}`);
}) as CdpRunner["send"],
@@ -112,6 +195,10 @@ describe("captureFrameData", () => {
const trees = await captureFrameData(cdp, 4, captured);
expect(trees.find((tree) => tree.frameId === "left")?.excludedBackendNodeIds).toEqual(
new Set([201]),
);
expect(trees.find((tree) => tree.frameId === "right")?.excludedBackendNodeIds?.size).toBe(0);
expect(trees.map((tree) => tree.frameId)).toEqual(["main", "left", "right"]);
expect(
captured.frameNodes?.get("left")?.find((node) => node.backendNodeId === 101)?.rect,
@@ -149,7 +236,11 @@ describe("captureFrameData", () => {
if (method === "Accessibility.getFullAXTree") return { nodes: [] };
if (method === "DOM.getBoxModel") throw new Error("owner geometry unavailable");
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } };
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 },
};
}
throw new Error(`unexpected root ${method}`);
}) as CdpRunner["send"],
@@ -192,7 +283,7 @@ describe("captureFrameData", () => {
expect.objectContaining({ backendDOMNodeId: 101, frameId: "child" }),
]);
expect(childDocument?.domNodes.find((node) => node.backendNodeId === 101)).toEqual(
expect.objectContaining({ rect: null, localRect: { x: 10, y: 20, w: 100, h: 40 } }),
expect.objectContaining({ rect: null, localRect: null, rendered: true }),
);
});
});
@@ -29,7 +29,7 @@ function captured(frameNodes: Map<string, CapturedNode[]>): CapturedViewModel {
}
describe("buildFrameDocuments", () => {
it("partitions mixed AX results by node ownership and cuts cross-frame parent edges", () => {
it("partitions mixed AX results by node ownership and cuts cross-frame parent edges", async () => {
const graph: CdpFrameGraph = {
rootFrameId: "main",
frames: [
@@ -54,7 +54,7 @@ describe("buildFrameDocuments", () => {
},
];
const documents = buildFrameDocuments(
const documents = await buildFrameDocuments(
graph,
batches,
captured(
@@ -72,7 +72,7 @@ describe("buildFrameDocuments", () => {
expect(childNodes.find((node) => node.nodeId === "virtual")?.frameId).toBe("child");
});
it("deduplicates overlapping same-target AX results using the strongest ownership", () => {
it("deduplicates overlapping same-target AX results using the strongest ownership", async () => {
const graph: CdpFrameGraph = {
rootFrameId: "main",
frames: [
@@ -92,7 +92,7 @@ describe("buildFrameDocuments", () => {
},
];
const documents = buildFrameDocuments(
const documents = await buildFrameDocuments(
graph,
batches,
captured(
@@ -107,7 +107,7 @@ describe("buildFrameDocuments", () => {
expect(documents.find((document) => document.frameId === "child")?.axNodes).toHaveLength(1);
});
it("uses target-scoped backend ownership when OOPIF backend ids collide", () => {
it("uses target-scoped backend ownership when OOPIF backend ids collide", async () => {
const graph: CdpFrameGraph = {
rootFrameId: "main",
frames: [
@@ -129,7 +129,7 @@ describe("buildFrameDocuments", () => {
nodes: [{ nodeId: `${frame.frameId}-node`, backendDOMNodeId: 7 }],
}));
const documents = buildFrameDocuments(
const documents = await buildFrameDocuments(
graph,
batches,
captured(
@@ -149,7 +149,7 @@ describe("buildFrameDocuments", () => {
);
});
it("uses captured frame parent identity instead of guessing from backend ids", () => {
it("uses captured frame parent identity instead of guessing from backend ids", async () => {
const frameNodes = new Map([
["main", [domNode("main", 1)]],
["left", [domNode("left", 7)]],
@@ -164,11 +164,25 @@ describe("buildFrameDocuments", () => {
nodes: [],
}));
const documents = buildFrameDocuments(null, frames, capture);
const documents = await buildFrameDocuments(null, frames, capture);
expect(documents.find((document) => document.frameId === "child")).toMatchObject({
parentFrameId: "left",
ownerBackendNodeId: 7,
});
});
it("resolves a reverse-ordered deep AX parent chain without recursion", async () => {
const frame = { frameId: "main", target: { tabId: 4 } };
const nodes: FrameOwnedAxNode[] = Array.from({ length: 10000 }, (_, i) => ({
nodeId: String(i),
...(i ? { parentId: String(i - 1) } : { frameId: "main" }),
})).reverse();
const documents = await buildFrameDocuments(
{ rootFrameId: "main", frames: [frame] },
[{ frame, nodes }],
captured(new Map()),
);
expect(documents[0].axNodes).toHaveLength(10000);
expect(documents[0].axNodes[0].frameId).toBe("main");
});
});
@@ -0,0 +1,40 @@
function captureAbortError(): Error {
const error = new Error("observation aborted");
error.name = "AbortError";
return error;
}
export function isAbortError(error: unknown): boolean {
return (
typeof error === "object" &&
error !== null &&
(error as { name?: string }).name === "AbortError"
);
}
export function throwIfAborted(signal: AbortSignal | undefined): void {
if (signal?.aborted) throw captureAbortError();
}
/** Check at bounded work blocks; yield only after a processing slice is spent.
* Each caller owns its deadline, so unrelated captures do not share state. */
export function createCaptureCheckpoint(signal?: AbortSignal): () => Promise<void> | undefined {
throwIfAborted(signal);
let deadline = performance.now() + 8;
return () => {
throwIfAborted(signal);
if (performance.now() < deadline) return;
return new Promise<void>((resolve) => {
const channel = new MessageChannel();
channel.port1.onmessage = () => {
channel.port1.close();
channel.port2.close();
resolve();
};
channel.port2.postMessage(null);
}).then(() => {
throwIfAborted(signal);
deadline = performance.now() + 8;
});
};
}
@@ -0,0 +1,37 @@
import type { Rect } from "@browser-skill/vom";
export interface CapturedNode {
backendNodeId: number;
parentBackendNodeId: number | null;
frameId?: string;
/** Owning iframe backend node id; `null` for the top-level document. */
ownerFrameBackendNodeId?: number | null;
tag: string;
attrs: Record<string, string>;
/** Top-level viewport-relative CSS px, clipped to the owning frame viewport. */
rect: Rect | null;
/** Frame-local viewport-relative CSS px before top-level projection. */
localRect?: Rect | null;
paintOrder: number;
position: string;
pointerEvents: string;
/**
* computed `cursor`. `cursor: pointer` is the strongest CDP-free signal
* that a non-semantic element (a `<div>`/`<span>` with a click handler)
* is actually an interactive control — used by the adapter to surface
* custom buttons/checkboxes the AX tree drops as `generic`. Optional like
* `textContent`: the live parser always sets it, hand-built fixtures may not.
*/
cursor?: string;
/**
* Whether the live DOM snapshot provides a painted, non-hidden box for this
* node. Semantic resolution uses this only for DOM fallback nodes; AX-backed
* nodes remain authoritative even when they are outside the viewport.
*/
rendered?: boolean;
textContent?: string;
formState?: "empty" | "filled" | "default";
formValue?: string;
formDefaultValue?: string;
formPlaceholder?: string;
}
+192 -523
View File
@@ -1,56 +1,35 @@
// CDP capture adapter: DOMSnapshot.captureSnapshot + Page.getLayoutMetrics
// → CapturedNode[] + Viewport. This is the ONLY VOM module that touches
// raw CDP. The captureSnapshot reply is columnar (parallel arrays + a
// shared string table); requested computed styles are decoded through
// STYLE_COL so capture and visibility policy share one explicit contract.
// Coordinate DOMSnapshot acquisition, frame geometry and hover probes.
// Snapshot decoding and form enrichment live in their own modules.
import type { Rect, Viewport } from "@browser-skill/vom";
import type { Viewport } from "@browser-skill/vom";
import type { CdpTarget } from "@/browser-driver/frame-graph";
import { evaluateHoverTrigger } from "@/lib/hover-trigger-policy";
import { isOverlayHostNode, OVERLAY_HOST_SELECTOR } from "../../lib/overlay-bridge";
import { childFrameProjection, type GeometryProjection, projectRectToViewport } from "../geometry";
import { OVERLAY_HOST_SELECTOR } from "../../lib/overlay-bridge";
import {
type FrameProjectionIssue,
type FrameProjectionState,
snapshotCoordinates,
} from "../geometry/coordinate-types";
import {
GeometryContext,
type LayoutMetrics,
snapshotLayoutScale,
} from "../geometry/frame-context";
import type { CdpRunner } from "../shared";
import { isAbortError, throwIfAborted } from "./capture-abort";
import type { CapturedNode } from "./capture-types";
import { enrichFormControlStates } from "./form-capture";
import { clearHover, ProbeBudget, waitForHover } from "./hover-perception";
import { type FrameContext, normalizeDocument } from "./normalize";
import {
REQUESTED_STYLES,
type SnapshotDocument,
type SnapshotReply,
snapshotFrameId,
sparseIndexMap,
} from "./snapshot";
const REQUESTED_STYLES = ["position", "pointer-events", "cursor", "visibility", "opacity"] as const;
const STYLE_COL = Object.fromEntries(
REQUESTED_STYLES.map((name, index) => [name, index]),
) as Record<(typeof REQUESTED_STYLES)[number], number>;
export interface CapturedNode {
backendNodeId: number;
parentBackendNodeId: number | null;
frameId?: string;
/** Owning iframe backend node id; `null` for the top-level document. */
ownerFrameBackendNodeId?: number | null;
tag: string;
attrs: Record<string, string>;
/** Top-level viewport-relative CSS px, clipped to the owning frame viewport. */
rect: Rect | null;
/** Frame-local viewport-relative CSS px before top-level projection. */
localRect?: Rect | null;
paintOrder: number;
position: string;
pointerEvents: string;
/**
* computed `cursor`. `cursor: pointer` is the strongest CDP-free signal
* that a non-semantic element (a `<div>`/`<span>` with a click handler)
* is actually an interactive control — used by the adapter to surface
* custom buttons/checkboxes the AX tree drops as `generic`. Optional like
* `textContent`: the live parser always sets it, hand-built fixtures may not.
*/
cursor?: string;
/**
* Whether the live DOM snapshot provides a painted, non-hidden box for this
* node. Semantic resolution uses this only for DOM fallback nodes; AX-backed
* nodes remain authoritative even when they are outside the viewport.
*/
rendered?: boolean;
textContent?: string;
formState?: "empty" | "filled" | "default";
formValue?: string;
formDefaultValue?: string;
formPlaceholder?: string;
}
export type { CapturedNode } from "./capture-types";
export type CapturedIframeNodes = Map<number, CapturedNode[]>;
@@ -67,230 +46,21 @@ export interface CapturedViewModel {
viewport: Viewport;
iframeNodes: CapturedIframeNodes;
frameNodes?: Map<string, CapturedNode[]>;
frameExcludedBackendNodeIds?: Map<string, ReadonlySet<number>>;
frameOwnerBackendNodeIds?: Map<string, number>;
/** Explicit DOMSnapshot frame ancestry; never inferred from backend node ids. */
frameParentIds?: Map<string, string>;
rootFrameId?: string;
/** Failed boundaries and descendants blocked by them; clipping is not a failure. */
frameGeometryIssues?: FrameProjectionIssue[];
/** Backend node ids belonging to the agent overlay host + its shadow subtree. */
excludedBackendNodeIds: Set<number>;
}
export interface CaptureViewModelOptions {
signal?: AbortSignal;
}
function captureAbortError(): Error {
const error = new Error("observation aborted");
error.name = "AbortError";
return error;
}
function isAbortError(error: unknown): boolean {
return (
typeof error === "object" &&
error !== null &&
(error as { name?: string }).name === "AbortError"
);
}
function throwIfAborted(signal: AbortSignal | undefined): void {
if (signal?.aborted) throw captureAbortError();
}
/** Sparse array format Chrome uses for infrequently-set per-node fields. */
interface SparseArray {
index: number[];
value: number[];
}
interface RareBooleanData {
index: number[];
}
interface SnapshotDocument {
frameId?: string | number;
scrollOffsetX?: number;
scrollOffsetY?: number;
nodes?: {
parentIndex?: number[];
nodeName?: number[];
backendNodeId?: number[];
attributes?: number[][];
/**
* Per-node text value (index into strings), set for `#text` / CDATA nodes.
* Element nodes carry -1. Same length as `backendNodeId`.
*/
nodeValue?: number[];
/** Maps node array index → index into `documents[]` for frame content. */
contentDocumentIndex?: SparseArray;
inputValue?: SparseArray;
textValue?: SparseArray;
inputChecked?: RareBooleanData;
optionSelected?: RareBooleanData;
};
layout?: {
nodeIndex?: number[];
styles?: number[][];
bounds?: number[][];
paintOrders?: number[];
};
}
function snapshotFrameId(document: SnapshotDocument, strings: string[]): string | undefined {
if (typeof document.frameId === "string") return document.frameId || undefined;
if (typeof document.frameId === "number") return str(strings, document.frameId) || undefined;
return undefined;
}
interface SnapshotReply {
strings?: string[];
documents?: SnapshotDocument[];
}
interface LayoutMetricsReply {
cssLayoutViewport?: {
clientWidth?: number;
clientHeight?: number;
pageX?: number;
pageY?: number;
};
layoutViewport?: { clientWidth?: number; clientHeight?: number; pageX?: number; pageY?: number };
}
function isFormControlTag(tag: string): boolean {
return tag === "input" || tag === "textarea" || tag === "select";
}
function isSensitiveFormControl(node: Pick<CapturedNode, "tag" | "attrs">): boolean {
return node.tag === "input" && (node.attrs.type ?? "").toLowerCase() === "password";
}
function snapshotFormState(
value: string | undefined,
defaultValue: string,
hasDefaultValue: boolean,
sensitive: boolean,
): CapturedNode["formState"] {
if (sensitive) {
if (value === undefined && !hasDefaultValue) return undefined;
return (value ?? defaultValue) === "" ? "empty" : "filled";
}
if (value === undefined) return undefined;
if (value === "") return "empty";
return value === defaultValue ? "default" : "filled";
}
const MAX_FORM_ENRICH_CONTROLS = 250;
interface CapturedFormState {
value?: string;
defaultValue?: string;
placeholder?: string;
state?: "empty" | "filled" | "default";
sensitive?: boolean;
}
interface CapturedFrameFormState {
controls: CapturedFormState[];
childFrames: CapturedFrameFormState[];
}
function formStateBatchExpression(maxControls: number): string {
return `(() => {
const maxControls = ${JSON.stringify(maxControls)};
let remaining = maxControls;
const controlSelector = "input,textarea,select";
const controlState = (el) => {
const tag = el.tagName.toLowerCase();
const type = tag === "input" ? String(el.type || "text").toLowerCase() : tag;
const sensitive = type === "password" || type === "credit-card";
const rawValue = typeof el.value === "string" ? el.value : "";
const defaultValue = typeof el.defaultValue === "string" ? el.defaultValue : "";
const placeholder = typeof el.placeholder === "string" ? el.placeholder : "";
const state = rawValue === "" ? "empty" : rawValue === defaultValue ? "default" : "filled";
return {
state,
sensitive,
placeholder,
...(sensitive ? {} : { value: rawValue, defaultValue }),
};
};
const collect = (doc) => {
const controls = [];
for (const el of Array.from(doc.querySelectorAll(controlSelector))) {
if (remaining <= 0) break;
controls.push(controlState(el));
remaining -= 1;
}
const childFrames = [];
for (const frame of Array.from(doc.querySelectorAll("iframe"))) {
let childDoc = null;
try { childDoc = frame.contentDocument; } catch { childDoc = null; }
childFrames.push(childDoc && remaining > 0 ? collect(childDoc) : { controls: [], childFrames: [] });
}
return { controls, childFrames };
};
return collect(document);
})()`;
}
function flattenFrameFormStates(
frame: CapturedFrameFormState | undefined,
out: CapturedFormState[][] = [],
): CapturedFormState[][] {
if (!frame) return out;
out.push(Array.isArray(frame.controls) ? frame.controls : []);
for (const child of Array.isArray(frame.childFrames) ? frame.childFrames : []) {
flattenFrameFormStates(child, out);
}
return out;
}
function applyFormStates(nodes: CapturedNode[], states: CapturedFormState[]): void {
let index = 0;
for (const node of nodes) {
if (!isFormControlTag(node.tag)) continue;
const state = states[index];
index += 1;
if (!state) continue;
node.formState = state.state;
node.formPlaceholder = state.placeholder ?? "";
if (state.sensitive || isSensitiveFormControl(node)) {
delete node.formValue;
delete node.formDefaultValue;
delete node.attrs.value;
} else {
node.formDefaultValue = state.defaultValue ?? "";
if (state.value !== undefined) node.formValue = state.value;
}
}
}
async function enrichFormControlStates(
cdp: CdpRunner,
tabId: number,
frameNodeGroups: CapturedNode[][],
signal?: AbortSignal,
): Promise<void> {
const hasControls = frameNodeGroups.some((nodes) =>
nodes.some((node) => isFormControlTag(node.tag)),
);
if (!hasControls) return;
try {
throwIfAborted(signal);
const result = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
expression: formStateBatchExpression(MAX_FORM_ENRICH_CONTROLS),
returnByValue: true,
});
throwIfAborted(signal);
const frameStates = flattenFrameFormStates(runtimeValue<CapturedFrameFormState>(result));
for (let i = 0; i < frameNodeGroups.length; i += 1) {
applyFormStates(frameNodeGroups[i], frameStates[i] ?? []);
}
} catch (error) {
if (isAbortError(error)) throw error;
// Best-effort enrichment. DOMSnapshot/AX data still carries the nodes.
}
geometry?: GeometryContext;
target?: CdpTarget;
}
interface RuntimeEvaluateReply {
@@ -314,44 +84,6 @@ interface CdpDomNode {
shadowRoots?: CdpDomNode[];
}
interface ParseDocumentResult {
nodes: CapturedNode[];
excludedBackendNodeIds: Set<number>;
}
interface FrameContext {
frameId?: string;
ownerFrameBackendNodeId: number | null;
projection: GeometryProjection | null;
scrollX: number;
scrollY: number;
}
function str(strings: string[], idx: number | undefined): string {
if (idx === undefined || idx < 0) return "";
return strings[idx] ?? "";
}
function devicePixelRatio(metrics: LayoutMetricsReply): number {
const layoutW = metrics.layoutViewport?.clientWidth ?? 0;
const cssW = metrics.cssLayoutViewport?.clientWidth ?? 0;
if (!layoutW || !cssW) return 1;
const dpr = layoutW / cssW;
if (!Number.isFinite(dpr) || dpr <= 0) return 1;
return dpr >= 1 ? dpr : 1;
}
function sparseIndexMap(sparse: SparseArray | undefined): Map<number, number> {
const out = new Map<number, number>();
if (!sparse?.index || !sparse.value) return out;
for (let i = 0; i < sparse.index.length; i++) {
const nodeIndex = sparse.index[i];
const docIndex = sparse.value[i];
if (nodeIndex !== undefined && docIndex !== undefined) out.set(nodeIndex, docIndex);
}
return out;
}
function collectBackendIdsFromDomNode(node: CdpDomNode | undefined, out: Set<number>): void {
if (!node) return;
if (typeof node.backendNodeId === "number") {
@@ -761,254 +493,146 @@ export async function collectOverlayExcludedBackendIds(
return excluded;
}
/**
* Parse one DOMSnapshot document into CapturedNode[].
*
* @param doc - the raw DOMSnapshot document object
* @param strings - the shared string table for the whole snapshot
* @param dpr - device-pixel-ratio from Page.getLayoutMetrics
* @param context - frame coordinate context; output rects are top viewport-relative
*/
function parseDocumentNodes(
doc: SnapshotDocument,
strings: string[],
dpr: number,
context: FrameContext,
): ParseDocumentResult {
const dn = doc.nodes;
const dl = doc.layout;
if (!dn?.backendNodeId) {
return { nodes: [], excludedBackendNodeIds: new Set() };
}
const count = dn.backendNodeId.length;
const layoutByNode = new Map<number, number>();
(dl?.nodeIndex ?? []).forEach((nodeIdx, layoutIdx) => layoutByNode.set(nodeIdx, layoutIdx));
// The agent's own overlay (WXT shadow host, marked with OVERLAY_HOST_MARKER_ATTR)
// is injected into the page and its open shadow root
// is inlined by DOMSnapshot as descendants of the host. Its fixed
// full-viewport click-blocker would otherwise dominate occlusion detection
// and hide the real page, so mark the host + its whole subtree excluded.
// parentIndex always references an earlier node, so one forward pass suffices.
const excluded = new Array<boolean>(count).fill(false);
const excludedBackendNodeIds = new Set<number>();
for (let n = 0; n < count; n++) {
const parentIdx = dn.parentIndex?.[n] ?? -1;
const inherited = parentIdx >= 0 && excluded[parentIdx];
let isOverlayHost = inherited;
if (!isOverlayHost) {
const pairs = dn.attributes?.[n] ?? [];
const attrNames: string[] = [];
for (let a = 0; a + 1 < pairs.length; a += 2) {
attrNames.push(str(strings, pairs[a]));
}
isOverlayHost = isOverlayHostNode(str(strings, dn.nodeName?.[n]), attrNames);
}
if (!isOverlayHost) continue;
excluded[n] = true;
const backendNodeId = dn.backendNodeId[n];
if (backendNodeId !== undefined) {
excludedBackendNodeIds.add(backendNodeId);
}
}
const posCol = STYLE_COL.position;
const peCol = STYLE_COL["pointer-events"];
const cursorCol = STYLE_COL.cursor;
const visibilityCol = STYLE_COL.visibility;
const opacityCol = STYLE_COL.opacity;
const inputValues = sparseIndexMap(dn.inputValue);
const textValues = sparseIndexMap(dn.textValue);
const checkedInputs = new Set(dn.inputChecked?.index ?? []);
const selectedOptions = new Set(dn.optionSelected?.index ?? []);
// Collect visible text from #text child nodes so element CapturedNodes
// carry a textContent value usable as a button/link label fallback.
// nodeValue is a parallel array: string index for text nodes, -1 otherwise.
const nodeTextContent = new Map<number, string>();
if (dn.nodeValue) {
for (let n = 0; n < count; n++) {
const nvIdx = dn.nodeValue[n] ?? -1;
if (nvIdx < 0) continue;
const text = str(strings, nvIdx).trim();
if (!text) continue;
const parentIdx = dn.parentIndex?.[n] ?? -1;
if (parentIdx >= 0) {
const existing = nodeTextContent.get(parentIdx);
nodeTextContent.set(parentIdx, existing ? `${existing} ${text}` : text);
}
}
}
// Build CapturedNodes.
const nodes: CapturedNode[] = [];
for (let n = 0; n < count; n++) {
if (excluded[n]) continue;
const backendNodeId = dn.backendNodeId[n];
const parentIdx = dn.parentIndex?.[n] ?? -1;
const parentBackendNodeId = parentIdx >= 0 ? (dn.backendNodeId[parentIdx] ?? null) : null;
const tag = str(strings, dn.nodeName?.[n]).toLowerCase();
const attrs: Record<string, string> = {};
const pairs = dn.attributes?.[n] ?? [];
for (let a = 0; a + 1 < pairs.length; a += 2) {
attrs[str(strings, pairs[a]).toLowerCase()] = str(strings, pairs[a + 1]);
}
let rect: Rect | null = null;
let localRect: Rect | null = null;
let paintOrder = 0;
let position = "static";
let pointerEvents = "auto";
let cursor = "auto";
let rendered = false;
const li = layoutByNode.get(n);
if (li !== undefined) {
const b = dl?.bounds?.[li];
if (b && b.length >= 4 && b[2] > 0 && b[3] > 0) {
localRect = {
x: b[0] / dpr - context.scrollX,
y: b[1] / dpr - context.scrollY,
w: b[2] / dpr,
h: b[3] / dpr,
};
const bounds = context.projection
? projectRectToViewport(localRect, context.projection)
: null;
rect = bounds ? { x: bounds.x, y: bounds.y, w: bounds.width, h: bounds.height } : null;
}
paintOrder = dl?.paintOrders?.[li] ?? 0;
const styleRow = dl?.styles?.[li] ?? [];
position = str(strings, styleRow[posCol]) || "static";
pointerEvents = str(strings, styleRow[peCol]) || "auto";
cursor = str(strings, styleRow[cursorCol]) || "auto";
const visibility = str(strings, styleRow[visibilityCol]) || "visible";
const opacity = str(strings, styleRow[opacityCol]) || "1";
rendered =
localRect !== null &&
visibility !== "hidden" &&
visibility !== "collapse" &&
(Number.parseFloat(opacity) || 0) > 0;
}
// Skip non-element nodes (#text, #cdata-section, etc.) — they carry no
// geometry and are never queried by callers.
if (tag.startsWith("#")) continue;
const textContent = nodeTextContent.get(n);
const rawFormValueIndex = tag === "textarea" ? textValues.get(n) : inputValues.get(n);
const rawFormValue =
rawFormValueIndex !== undefined ? str(strings, rawFormValueIndex) : undefined;
const sensitive = isSensitiveFormControl({ tag, attrs });
const formDefaultValue = attrs.value ?? "";
const formValue = sensitive ? undefined : rawFormValue;
const formState = snapshotFormState(
rawFormValue,
formDefaultValue,
Object.prototype.hasOwnProperty.call(attrs, "value"),
sensitive,
);
if (sensitive) delete attrs.value;
nodes.push({
backendNodeId,
parentBackendNodeId,
...(context.frameId ? { frameId: context.frameId } : {}),
ownerFrameBackendNodeId: context.ownerFrameBackendNodeId,
tag,
attrs,
rect,
localRect,
paintOrder,
position,
pointerEvents,
cursor,
rendered,
textContent,
...(formValue !== undefined ? { formValue } : {}),
...(tag === "input" || tag === "textarea"
? {
formPlaceholder: attrs.placeholder ?? "",
...(!sensitive ? { formDefaultValue } : {}),
...(formState ? { formState } : {}),
}
: {}),
...(checkedInputs.has(n) ? { formValue: "true", formState: "filled" } : {}),
...(selectedOptions.has(n) ? { formValue: attrs.value ?? textContent ?? "" } : {}),
});
}
return { nodes, excludedBackendNodeIds };
}
interface ParseFrameDocumentsResult {
frameGeometryIssues: FrameProjectionIssue[];
iframeNodes: CapturedIframeNodes;
frameOwnerBackendNodeIds: Map<string, number>;
frameParentIds: Map<string, string>;
frameExcludedBackendNodeIds: Map<string, ReadonlySet<number>>;
excludedBackendNodeIds: Set<number>;
}
function parseChildFrameDocuments(
async function parseChildFrameDocuments(
documents: SnapshotDocument[],
strings: string[],
dpr: number,
parentDocIndex: number,
parentNodes: CapturedNode[],
parentContext: FrameContext,
geometry: GeometryContext,
signal?: AbortSignal,
visited = new Set<number>(),
): ParseFrameDocumentsResult {
): Promise<ParseFrameDocumentsResult> {
const frameGeometryIssues: FrameProjectionIssue[] = [];
const iframeNodes: CapturedIframeNodes = new Map();
const frameOwnerBackendNodeIds = new Map<string, number>();
const frameParentIds = new Map<string, string>();
const frameExcludedBackendNodeIds = new Map<string, ReadonlySet<number>>();
const excludedBackendNodeIds = new Set<number>();
const parentDoc = documents[parentDocIndex];
const cdi = sparseIndexMap(parentDoc?.nodes?.contentDocumentIndex);
if (cdi.size === 0) {
return { iframeNodes, frameOwnerBackendNodeIds, frameParentIds, excludedBackendNodeIds };
return {
iframeNodes,
frameOwnerBackendNodeIds,
frameParentIds,
frameExcludedBackendNodeIds,
excludedBackendNodeIds,
frameGeometryIssues,
};
}
const parentBackendIds = parentDoc?.nodes?.backendNodeId ?? [];
const parentNodeByBackendId = new Map(parentNodes.map((node) => [node.backendNodeId, node]));
for (const [nodeArrayIdx, childDocIndex] of cdi) {
if (visited.has(childDocIndex)) continue;
const children = [...cdi].flatMap(([nodeArrayIdx, childDocIndex]) => {
const childDoc = documents[childDocIndex];
if (!childDoc) continue;
const iframeBackendId = parentBackendIds[nodeArrayIdx];
if (iframeBackendId === undefined) continue;
const iframeNode = parentNodeByBackendId.get(iframeBackendId);
const projection =
parentContext.projection && iframeNode?.localRect
? childFrameProjection(parentContext.projection, iframeNode.localRect)
: null;
if (visited.has(childDocIndex) || !childDoc || iframeBackendId === undefined) return [];
const source = { target: parentContext.target, frameId: snapshotFrameId(childDoc, strings) };
const coordinates = snapshotCoordinates(childDoc, parentContext.layoutUnitsPerCssPixel);
const parent = parentContext.projection;
const projection: FrameProjectionState =
parent.status === "available"
? coordinates
? { status: "unavailable", source, ownerBackendNodeId: iframeBackendId }
: { status: "unavailable", source, reason: "snapshot-coordinates-unavailable" }
: { status: "blocked", source, cause: parent.status === "blocked" ? parent.cause : parent };
return [
{
childDocIndex,
childDoc,
iframeBackendId,
source,
coordinates,
projection: projection as FrameProjectionState,
},
];
});
const parentProjection =
parentContext.projection.status === "available"
? parentContext.projection.projection.geometry
: null;
if (parentProjection) {
// Measure only direct siblings. Recursion starts after these workers finish,
// so descendants never hold or recursively acquire a measurement slot.
const edge = parentProjection.edges[0];
const clips = edge
? [edge.destinationQuad, ...(edge.destinationClips ?? [])]
: parentProjection.sourceClips;
let cursor = 0;
let failure: unknown;
await Promise.all(
Array.from({ length: Math.min(4, children.length) }, async () => {
while (cursor < children.length && !signal?.aborted && !failure) {
const child = children[cursor++];
if (!child.coordinates) continue;
try {
// Owner quads are target-relative; do not apply the parent transform twice.
child.projection = await geometry.snapshotProjection(
child.source,
child.iframeBackendId,
clips,
parentProjection.topViewport,
);
} catch (error) {
failure ??= error;
}
}
}),
);
// Join all active reads, including object cleanup, before propagating abort.
if (failure) throw failure;
}
throwIfAborted(signal);
// Completion order must not change document traversal or result insertion order.
for (const {
childDocIndex,
childDoc,
iframeBackendId,
source,
projection,
coordinates,
} of children) {
throwIfAborted(signal);
if (projection.status !== "available") frameGeometryIssues.push(projection);
const childContext: FrameContext = {
frameId: snapshotFrameId(childDoc, strings),
frameId: source.frameId,
target: source.target,
ownerFrameBackendNodeId: iframeBackendId,
projection,
scrollX: childDoc.scrollOffsetX ?? 0,
scrollY: childDoc.scrollOffsetY ?? 0,
coordinates,
layoutUnitsPerCssPixel: parentContext.layoutUnitsPerCssPixel,
};
const nextVisited = new Set(visited);
nextVisited.add(childDocIndex);
const parsed = parseDocumentNodes(childDoc, strings, dpr, childContext);
const parsed = await normalizeDocument(childDoc, strings, childContext, signal);
iframeNodes.set(iframeBackendId, parsed.nodes);
if (childContext.frameId) {
frameOwnerBackendNodeIds.set(childContext.frameId, iframeBackendId);
frameExcludedBackendNodeIds.set(childContext.frameId, parsed.index.excludedBackendNodeIds);
if (parentContext.frameId) frameParentIds.set(childContext.frameId, parentContext.frameId);
}
for (const id of parsed.excludedBackendNodeIds) excludedBackendNodeIds.add(id);
for (const id of parsed.index.excludedBackendNodeIds) excludedBackendNodeIds.add(id);
const nested = parseChildFrameDocuments(
const nested = await parseChildFrameDocuments(
documents,
strings,
dpr,
childDocIndex,
parsed.nodes,
childContext,
geometry,
signal,
nextVisited,
);
frameGeometryIssues.push(...nested.frameGeometryIssues);
for (const [nestedFrameId, nestedNodes] of nested.iframeNodes) {
iframeNodes.set(nestedFrameId, nestedNodes);
}
@@ -1018,10 +642,19 @@ function parseChildFrameDocuments(
for (const [frameId, parentFrameId] of nested.frameParentIds) {
frameParentIds.set(frameId, parentFrameId);
}
for (const [frameId, excluded] of nested.frameExcludedBackendNodeIds)
frameExcludedBackendNodeIds.set(frameId, excluded);
for (const id of nested.excludedBackendNodeIds) excludedBackendNodeIds.add(id);
}
return { iframeNodes, frameOwnerBackendNodeIds, frameParentIds, excludedBackendNodeIds };
return {
iframeNodes,
frameOwnerBackendNodeIds,
frameParentIds,
frameExcludedBackendNodeIds,
excludedBackendNodeIds,
frameGeometryIssues,
};
}
export async function captureViewModel(
@@ -1030,22 +663,21 @@ export async function captureViewModel(
options: CaptureViewModelOptions = {},
): Promise<CapturedViewModel> {
throwIfAborted(options.signal);
let metrics: LayoutMetricsReply = {};
const target = options.target ?? { tabId };
const geometry = options.geometry ?? new GeometryContext(cdp, tabId, undefined, options.signal);
let metrics: LayoutMetrics = {};
try {
metrics = await cdp.send<LayoutMetricsReply>(tabId, "Page.getLayoutMetrics", {});
metrics = await geometry.layoutMetrics(target);
} catch (error) {
if (isAbortError(error)) throw error;
console.debug("[bsk capture] layout metrics unavailable", error);
}
throwIfAborted(options.signal);
const dpr = devicePixelRatio(metrics);
const vpSrc = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
const viewport: Viewport = {
width: vpSrc.clientWidth ?? 0,
height: vpSrc.clientHeight ?? 0,
width: metrics.cssLayoutViewport?.clientWidth ?? 0,
height: metrics.cssLayoutViewport?.clientHeight ?? 0,
};
const scrollX = vpSrc.pageX ?? 0;
const scrollY = vpSrc.pageY ?? 0;
const layoutUnitsPerCssPixel = snapshotLayoutScale(metrics);
await cdp.send(tabId, "DOMSnapshot.enable", {});
throwIfAborted(options.signal);
@@ -1068,23 +700,55 @@ export async function captureViewModel(
};
}
const source = { target, frameId: snapshotFrameId(doc0, strings) };
const coordinates = snapshotCoordinates(doc0, layoutUnitsPerCssPixel, {
x: metrics.cssLayoutViewport?.pageX,
y: metrics.cssLayoutViewport?.pageY,
});
const topContext: FrameContext = {
frameId: snapshotFrameId(doc0, strings),
ownerFrameBackendNodeId: null,
projection: {
sourceClips: [],
edges: [],
topViewport: viewport,
},
scrollX,
scrollY,
target,
projection:
coordinates &&
[viewport.width, viewport.height].every((size) => Number.isFinite(size) && size > 0)
? {
status: "available",
projection: {
source,
geometry: { sourceClips: [], edges: [], topViewport: viewport },
},
}
: { status: "unavailable", source, reason: "snapshot-coordinates-unavailable" },
coordinates,
layoutUnitsPerCssPixel,
};
const mainParsed = parseDocumentNodes(doc0, strings, dpr, topContext);
const mainParsed = await normalizeDocument(doc0, strings, topContext, options.signal);
const nodes = mainParsed.nodes;
const excludedBackendNodeIds = new Set(mainParsed.excludedBackendNodeIds);
const excludedBackendNodeIds = new Set(mainParsed.index.excludedBackendNodeIds);
const frameParsed = parseChildFrameDocuments(documents, strings, dpr, 0, nodes, topContext);
const ownerIds = new Set<number>();
for (const document of documents) {
for (const index of document.nodes?.contentDocumentIndex?.index ?? []) {
const id = document.nodes?.backendNodeId?.[index];
if (id !== undefined) ownerIds.add(id);
}
}
geometry.registerSnapshotOwners(target, ownerIds);
const frameParsed = await parseChildFrameDocuments(
documents,
strings,
0,
topContext,
geometry,
options.signal,
);
const iframeNodes = frameParsed.iframeNodes;
if (topContext.frameId)
frameParsed.frameExcludedBackendNodeIds.set(
topContext.frameId,
mainParsed.index.excludedBackendNodeIds,
);
for (const id of frameParsed.excludedBackendNodeIds) {
excludedBackendNodeIds.add(id);
}
@@ -1109,7 +773,12 @@ export async function captureViewModel(
viewport,
iframeNodes,
frameNodes,
frameGeometryIssues: [
...(topContext.projection.status === "available" ? [] : [topContext.projection]),
...frameParsed.frameGeometryIssues,
],
frameOwnerBackendNodeIds: frameParsed.frameOwnerBackendNodeIds,
frameExcludedBackendNodeIds: frameParsed.frameExcludedBackendNodeIds,
frameParentIds: frameParsed.frameParentIds,
...(topContext.frameId ? { rootFrameId: topContext.frameId } : {}),
excludedBackendNodeIds,
+94
View File
@@ -0,0 +1,94 @@
import { isOverlayHostNode } from "@/lib/overlay-bridge";
import { createCaptureCheckpoint } from "./capture-abort";
import type { CapturedNode } from "./capture-types";
/** Bounds retain their raw snapshot document layout units until normalization. */
export interface SnapshotLayout {
readonly boundsSpace: "snapshot-document-layout";
bounds?: number[];
styles: Readonly<Record<string, string>>;
}
export interface DecodedNode extends Omit<CapturedNode, "rect" | "localRect" | "rendered"> {
nodeType?: number;
layout?: SnapshotLayout;
}
export interface NodeFacts extends CapturedNode {
nodeType?: number;
layout?: SnapshotLayout;
}
export interface DocumentIndex<T extends DecodedNode = NodeFacts> {
readonly nodes: ReadonlyMap<number, T>;
readonly excludedBackendNodeIds: ReadonlySet<number>;
}
export interface DecodedDocument {
nodes: DecodedNode[];
}
/** All snapshot nodes participate, including document and shadow roots. This
* preserves overlay propagation when the semantic adapter omits those nodes. */
export async function buildDocumentIndex<T extends DecodedNode>(
input: readonly T[],
signal?: AbortSignal,
): Promise<DocumentIndex<T>> {
const checkpoint = createCaptureCheckpoint(signal);
const nodes = new Map<number, T>();
const overlayByNode = new Map<number, boolean>();
const excludedBackendNodeIds = new Set<number>();
for (let i = 0; i < input.length; i++) {
if (i % 256 === 0) {
const pending = checkpoint();
if (pending) await pending;
}
const node = input[i];
nodes.set(node.backendNodeId, node);
}
let work = 0;
for (const node of input) {
if (work++ % 256 === 0) {
const pending = checkpoint();
if (pending) await pending;
}
if (overlayByNode.has(node.backendNodeId)) continue;
const path: DecodedNode[] = [];
const visiting = new Set<number>();
let current: DecodedNode | undefined = node;
let overlay = false;
while (current && !overlayByNode.has(current.backendNodeId)) {
if (work++ % 256 === 0) {
const pending = checkpoint();
if (pending) await pending;
}
if (visiting.has(current.backendNodeId)) {
// Only an overlay inside the cycle may mark the cycle and its descendants.
overlay = path
.slice(path.findIndex((item) => item.backendNodeId === current!.backendNodeId))
.some((item) => isOverlayHostNode(item.tag, Object.keys(item.attrs)));
break;
}
visiting.add(current.backendNodeId);
path.push(current);
if (current.parentBackendNodeId === null) {
current = undefined;
break;
}
current = nodes.get(current.parentBackendNodeId);
}
if (current && overlayByNode.has(current.backendNodeId))
overlay = overlayByNode.get(current.backendNodeId)!;
for (let i = path.length - 1; i >= 0; i--) {
if (work++ % 256 === 0) {
const pending = checkpoint();
if (pending) await pending;
}
const item = path[i];
overlay = overlay || isOverlayHostNode(item.tag, Object.keys(item.attrs));
overlayByNode.set(item.backendNodeId, overlay);
if (overlay) excludedBackendNodeIds.add(item.backendNodeId);
}
}
return { nodes, excludedBackendNodeIds };
}
@@ -0,0 +1,155 @@
import type { CdpRunner } from "../shared";
import { isAbortError, throwIfAborted } from "./capture-abort";
import type { CapturedNode } from "./capture-types";
interface RuntimeEvaluateReply {
result?: { deepSerializedValue?: DeepSerializedValue };
}
const MAX_FORM_ENRICH_CONTROLS = 250;
interface CapturedFormState {
value?: string;
defaultValue?: string;
placeholder?: string;
state?: "empty" | "filled" | "default";
sensitive?: boolean;
}
interface DeepSerializedValue {
type: string;
value?: unknown;
}
function formStateBatchExpression(maxControls: number): string {
return `(() => {
const maxControls = ${JSON.stringify(maxControls)};
let remaining = maxControls;
const controlSelector = "input,textarea,select";
const controlState = (el) => {
const tag = el.tagName.toLowerCase();
const type = tag === "input" ? String(el.type || "text").toLowerCase() : tag;
const sensitive = type === "password" || type === "credit-card";
const rawValue = typeof el.value === "string" ? el.value : "";
const defaultValue = typeof el.defaultValue === "string" ? el.defaultValue : "";
const placeholder = typeof el.placeholder === "string" ? el.placeholder : "";
const state = rawValue === "" ? "empty" : rawValue === defaultValue ? "default" : "filled";
return {
state,
sensitive,
placeholder,
...(sensitive ? {} : { value: rawValue, defaultValue }),
};
};
const controls = [];
const collect = (doc) => {
for (const el of Array.from(doc.querySelectorAll(controlSelector))) {
if (remaining <= 0) break;
// Deep serialization supplies the node's backend id. Keep the
// state as JSON so decoding needs no general-purpose V8 deserializer.
controls.push([el, JSON.stringify(controlState(el))]);
remaining -= 1;
}
for (const frame of Array.from(doc.querySelectorAll("iframe"))) {
if (remaining <= 0) break;
let childDoc = null;
try { childDoc = frame.contentDocument; } catch { childDoc = null; }
if (childDoc) collect(childDoc);
}
};
collect(document);
return controls;
})()`;
}
function formStatesByBackendId(result: RuntimeEvaluateReply): Map<number, CapturedFormState> {
const states = new Map<number, CapturedFormState>();
const serialized = result.result?.deepSerializedValue;
if (serialized?.type !== "array" || !Array.isArray(serialized.value)) return states;
for (const entry of serialized.value as DeepSerializedValue[]) {
if (entry?.type !== "array" || !Array.isArray(entry.value)) continue;
const [element, json] = entry.value as DeepSerializedValue[];
if (element?.type !== "node" || json?.type !== "string" || typeof json.value !== "string") {
continue;
}
const backendNodeId = (element.value as { backendNodeId?: number } | undefined)?.backendNodeId;
if (typeof backendNodeId !== "number" || !Number.isSafeInteger(backendNodeId)) continue;
try {
const state = JSON.parse(json.value) as CapturedFormState | null;
if (
!state ||
!["empty", "filled", "default"].includes(state.state ?? "") ||
typeof state.sensitive !== "boolean" ||
typeof state.placeholder !== "string" ||
(!state.sensitive &&
(typeof state.value !== "string" || typeof state.defaultValue !== "string"))
) {
continue;
}
states.set(backendNodeId, state);
} catch {
// A malformed entry must not overwrite the snapshot's own state.
}
}
return states;
}
function applyFormStates(nodes: CapturedNode[], states: Map<number, CapturedFormState>): void {
for (const node of nodes) {
if (!["input", "textarea", "select"].includes(node.tag)) continue;
const state = states.get(node.backendNodeId);
if (!state) continue;
node.formState = state.state;
node.formPlaceholder = state.placeholder ?? "";
if (
state.sensitive ||
(node.tag === "input" && (node.attrs.type ?? "").toLowerCase() === "password")
) {
delete node.formValue;
delete node.formDefaultValue;
delete node.attrs.value;
} else {
node.formDefaultValue = state.defaultValue ?? "";
if (state.value !== undefined) node.formValue = state.value;
}
}
}
export async function enrichFormControlStates(
cdp: CdpRunner,
tabId: number,
frameNodeGroups: CapturedNode[][],
signal?: AbortSignal,
): Promise<boolean> {
const hasControls = frameNodeGroups.some((nodes) =>
nodes.some((node) => ["input", "textarea", "select"].includes(node.tag)),
);
if (!hasControls) return true;
const objectGroup = `bsk-vom-forms-${crypto.randomUUID()}`;
try {
throwIfAborted(signal);
const result = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
expression: formStateBatchExpression(MAX_FORM_ENRICH_CONTROLS),
objectGroup,
serializationOptions: {
serialization: "deep",
maxDepth: 3,
additionalParameters: { maxNodeDepth: 0, includeShadowTree: "none" },
},
});
throwIfAborted(signal);
// Backend ids are scoped to this capture's CDP target. OOPIF targets
// have separate enrichment calls and never share this lookup.
const states = formStatesByBackendId(result);
for (const nodes of frameNodeGroups) {
applyFormStates(nodes, states);
}
return true;
} catch (error) {
if (isAbortError(error)) throw error;
// Best-effort enrichment. DOMSnapshot/AX data still carries the nodes.
return false;
} finally {
await cdp.send(tabId, "Runtime.releaseObjectGroup", { objectGroup }).catch(() => undefined);
throwIfAborted(signal);
}
}
+17 -15
View File
@@ -1,6 +1,6 @@
import type { CdpFrame, CdpFrameGraph } from "@/browser-driver/frame-graph";
import { resolveFrameProjection } from "../frame-geometry";
import { type GeometryProjection, projectRectToViewport } from "../geometry";
import { GeometryContext } from "../geometry/frame-context";
import { type CdpRunner, cdpRunnerForTarget, sendToCdpTarget } from "../shared";
import { type CapturedNode, type CapturedViewModel, captureViewModel } from "./capture";
import {
@@ -21,16 +21,6 @@ function isAbortError(err: unknown): boolean {
return err instanceof Error && err.name === "AbortError";
}
async function discoverFrameGraph(cdp: CdpRunner, tabId: number): Promise<CdpFrameGraph | null> {
if (!cdp.getFrameGraph) return null;
try {
return await cdp.getFrameGraph(tabId);
} catch (err) {
console.debug("[bsk observation] frame graph capture failed", err);
return null;
}
}
function transformFrameNodes(
nodes: CapturedNode[],
projection: GeometryProjection | null,
@@ -51,6 +41,7 @@ async function captureMissingFrameDocuments(
tabId: number,
graph: CdpFrameGraph,
captured: CapturedViewModel,
geometry: GeometryContext,
signal?: AbortSignal,
): Promise<void> {
if (!captured.frameNodes) captured.frameNodes = new Map();
@@ -62,10 +53,20 @@ async function captureMissingFrameDocuments(
try {
const child = await captureViewModel(cdpRunnerForTarget(cdp, frame.target), tabId, {
signal,
geometry,
target: frame.target,
});
if (child.frameExcludedBackendNodeIds?.size) {
captured.frameExcludedBackendNodeIds ??= new Map();
for (const [id, excluded] of child.frameExcludedBackendNodeIds)
captured.frameExcludedBackendNodeIds.set(id, excluded);
}
if (child.frameGeometryIssues?.length) {
(captured.frameGeometryIssues ??= []).push(...child.frameGeometryIssues);
}
let projection: GeometryProjection | null = null;
try {
projection = await resolveFrameProjection(cdp, graph, frame.frameId);
projection = await geometry.targetProjection(frame.frameId);
} catch (err) {
if (isAbortError(err)) throw err;
console.debug("[bsk observation] child frame geometry projection failed", {
@@ -171,11 +172,12 @@ export async function captureFrameData<T extends FrameAxNode>(
tabId: number,
captured: CapturedViewModel,
signal?: AbortSignal,
geometry = new GeometryContext(cdp, tabId, undefined, signal),
): Promise<CapturedFrameDocument<T>[]> {
const graph = await discoverFrameGraph(cdp, tabId);
const graph = await geometry.graph();
const frames = graph?.frames ?? [];
if (graph) await captureMissingFrameDocuments(cdp, tabId, graph, captured, signal);
if (graph) await captureMissingFrameDocuments(cdp, tabId, graph, captured, geometry, signal);
throwIfAborted(signal);
const batches = await captureAxTrees<T>(cdp, tabId, frames, captured, signal);
return buildFrameDocuments(graph, batches, captured);
return buildFrameDocuments(graph, batches, captured, signal);
}
+80 -46
View File
@@ -4,7 +4,8 @@ import {
type CdpTarget,
cdpTargetKey,
} from "@/browser-driver/frame-graph";
import type { CapturedNode, CapturedViewModel } from "./capture";
import { createCaptureCheckpoint } from "./capture-abort";
import type { CapturedNode } from "./capture-types";
export interface FrameOwnedAxNode {
nodeId: string;
@@ -23,6 +24,17 @@ export interface FrameDocument<T extends FrameOwnedAxNode> extends CdpFrame {
contextScopeId: string;
axNodes: T[];
domNodes: CapturedNode[];
excludedBackendNodeIds?: ReadonlySet<number>;
}
/** Explicit snapshot document data used to join DOM and AX ownership. */
export interface FrameDomInput {
nodes: CapturedNode[];
rootFrameId?: string;
frameNodes?: ReadonlyMap<string, CapturedNode[]>;
frameOwnerBackendNodeIds?: ReadonlyMap<string, number>;
frameParentIds?: ReadonlyMap<string, string>;
frameExcludedBackendNodeIds?: ReadonlyMap<string, ReadonlySet<number>>;
}
interface Ownership {
@@ -47,7 +59,7 @@ function targetBackendKey(target: CdpTarget, backendNodeId: number): string {
function frameList<T extends FrameOwnedAxNode>(
graph: CdpFrameGraph | null,
batches: FrameAxBatch<T>[],
captured: CapturedViewModel,
captured: FrameDomInput,
): CdpFrame[] {
const frames = new Map<string, CdpFrame>();
for (const frame of graph?.frames ?? []) frames.set(frame.frameId, frame);
@@ -70,11 +82,14 @@ function frameList<T extends FrameOwnedAxNode>(
});
}
export function buildFrameDocuments<T extends FrameOwnedAxNode>(
export async function buildFrameDocuments<T extends FrameOwnedAxNode>(
graph: CdpFrameGraph | null,
batches: FrameAxBatch<T>[],
captured: CapturedViewModel,
): FrameDocument<T>[] {
captured: FrameDomInput,
signal?: AbortSignal,
): Promise<FrameDocument<T>[]> {
const checkpoint = createCaptureCheckpoint(signal);
let work = 0;
const frames = frameList(graph, batches, captured);
const frameById = new Map(frames.map((frame) => [frame.frameId, frame]));
const rootFrameId = graph?.rootFrameId ?? captured.rootFrameId ?? frames[0]?.frameId;
@@ -83,54 +98,68 @@ export function buildFrameDocuments<T extends FrameOwnedAxNode>(
const backendOwner = new Map<string, string>();
for (const frame of frames) {
for (const node of domNodesForFrame(frame.frameId)) {
if (work++ % 256 === 0) {
const pending = checkpoint();
if (pending) await pending;
}
backendOwner.set(targetBackendKey(frame.target, node.backendNodeId), frame.frameId);
}
}
const candidates = new Map<string, OwnedCandidate<T>>();
for (const batch of batches) {
const nodeById = new Map(batch.nodes.map((node) => [node.nodeId, node]));
const ownershipByNodeId = new Map<string, Ownership>();
const resolving = new Set<string>();
const resolveOwnership = (node: T): Ownership => {
const cached = ownershipByNodeId.get(node.nodeId);
if (cached) return cached;
if (node.frameId && frameById.has(node.frameId)) {
const ownership = { frameId: node.frameId, strength: 4 };
ownershipByNodeId.set(node.nodeId, ownership);
return ownership;
}
if (typeof node.backendDOMNodeId === "number") {
const frameId = backendOwner.get(
targetBackendKey(batch.frame.target, node.backendDOMNodeId),
);
if (frameId) {
const ownership = { frameId, strength: 3 };
ownershipByNodeId.set(node.nodeId, ownership);
return ownership;
}
}
if (node.parentId && !resolving.has(node.nodeId)) {
const parent = nodeById.get(node.parentId);
if (parent) {
resolving.add(node.nodeId);
const inherited = resolveOwnership(parent);
resolving.delete(node.nodeId);
const ownership = {
frameId: inherited.frameId,
strength: Math.min(2, inherited.strength),
};
ownershipByNodeId.set(node.nodeId, ownership);
return ownership;
}
}
const ownership = { frameId: batch.frame.frameId, strength: 1 };
ownershipByNodeId.set(node.nodeId, ownership);
return ownership;
};
const nodeById = new Map<string, T>();
for (const node of batch.nodes) {
const ownership = resolveOwnership(node);
if (work++ % 256 === 0) {
const pending = checkpoint();
if (pending) await pending;
}
nodeById.set(node.nodeId, node);
}
const ownershipByNodeId = new Map<string, Ownership>();
for (const node of batch.nodes) {
const path: T[] = [];
const visiting = new Set<string>();
let current: T | undefined = node;
let ownership: Ownership = { frameId: batch.frame.frameId, strength: 1 };
while (current) {
if (work++ % 256 === 0) {
const pending = checkpoint();
if (pending) await pending;
}
const cached = ownershipByNodeId.get(current.nodeId);
if (cached) {
ownership = cached;
break;
}
if (visiting.has(current.nodeId)) break;
visiting.add(current.nodeId);
if (current.frameId && frameById.has(current.frameId)) {
ownership = { frameId: current.frameId, strength: 4 };
ownershipByNodeId.set(current.nodeId, ownership);
break;
}
const frameId =
typeof current.backendDOMNodeId === "number"
? backendOwner.get(targetBackendKey(batch.frame.target, current.backendDOMNodeId))
: undefined;
if (frameId) {
ownership = { frameId, strength: 3 };
ownershipByNodeId.set(current.nodeId, ownership);
break;
}
path.push(current);
current = current.parentId ? nodeById.get(current.parentId) : undefined;
}
for (let i = path.length - 1; i >= 0; i--) {
if (work++ % 256 === 0) {
const pending = checkpoint();
if (pending) await pending;
}
ownership = { frameId: ownership.frameId, strength: Math.min(2, ownership.strength) };
ownershipByNodeId.set(path[i].nodeId, ownership);
}
if (!ownershipByNodeId.has(node.nodeId)) ownershipByNodeId.set(node.nodeId, ownership);
const key = targetNodeKey(batch.frame.target, node.nodeId);
const existing = candidates.get(key);
if (!existing || ownership.strength > existing.ownership.strength) {
@@ -144,6 +173,10 @@ export function buildFrameDocuments<T extends FrameOwnedAxNode>(
);
const axNodesByFrame = new Map<string, T[]>();
for (const candidate of candidates.values()) {
if (work++ % 256 === 0) {
const pending = checkpoint();
if (pending) await pending;
}
const { node, target, ownership } = candidate;
const parentFrameId = node.parentId
? ownershipByTargetNode.get(targetNodeKey(target, node.parentId))
@@ -169,5 +202,6 @@ export function buildFrameDocuments<T extends FrameOwnedAxNode>(
contextScopeId: frame.frameId,
axNodes: axNodesByFrame.get(frame.frameId) ?? [],
domNodes: domNodesForFrame(frame.frameId),
excludedBackendNodeIds: captured.frameExcludedBackendNodeIds?.get(frame.frameId),
}));
}
+78
View File
@@ -0,0 +1,78 @@
import type { CdpTarget } from "@/browser-driver/frame-graph";
import {
type FrameProjectionState,
projectSnapshotRect,
type SnapshotCoordinates,
snapshotViewportRect,
} from "../geometry/coordinate-types";
import { createCaptureCheckpoint } from "./capture-abort";
import { buildDocumentIndex, type DocumentIndex, type NodeFacts } from "./facts";
import { decodeDocument, type SnapshotDocument } from "./snapshot";
export interface FrameContext {
frameId?: string;
ownerFrameBackendNodeId: number | null;
projection: FrameProjectionState;
target: CdpTarget;
coordinates: SnapshotCoordinates | null;
layoutUnitsPerCssPixel: number | null;
}
export interface NormalizedDocument {
nodes: NodeFacts[];
index: DocumentIndex;
}
/** Interpret one document using a supplied projection. No live reads or frame scheduling. */
export async function normalizeDocument(
doc: SnapshotDocument,
strings: string[],
context: FrameContext,
signal?: AbortSignal,
): Promise<NormalizedDocument> {
const decoded = await decodeDocument(doc, strings, signal);
const checkpoint = createCaptureCheckpoint(signal);
const nodes: NodeFacts[] = [];
for (let i = 0; i < decoded.nodes.length; i++) {
if (i % 256 === 0) {
const pending = checkpoint();
if (pending) await pending;
}
const node = decoded.nodes[i];
const bounds = node.layout?.bounds ?? [];
const input = snapshotViewportRect(
bounds,
{ target: context.target, frameId: context.frameId },
context.coordinates,
);
const local = input?.rect;
const rect =
input && context.projection.status === "available"
? projectSnapshotRect(input, context.projection.projection)
: null;
const visibility = node.layout?.styles.visibility || "visible";
const opacity = node.layout?.styles.opacity || "1";
nodes.push({
...node,
...(context.frameId ? { frameId: context.frameId } : {}),
ownerFrameBackendNodeId: context.ownerFrameBackendNodeId,
localRect: local ? { x: local.x, y: local.y, w: local.width, h: local.height } : null,
rect: rect ? { x: rect.x, y: rect.y, w: rect.width, h: rect.height } : null,
rendered:
bounds.length >= 4 &&
bounds.slice(0, 4).every(Number.isFinite) &&
bounds[2] > 0 &&
bounds[3] > 0 &&
visibility !== "hidden" &&
visibility !== "collapse" &&
(Number.parseFloat(opacity) || 0) > 0,
});
}
const index = await buildDocumentIndex(nodes, signal);
return {
nodes: nodes.filter(
(node) => !node.tag.startsWith("#") && !index.excludedBackendNodeIds.has(node.backendNodeId),
),
index,
};
}
@@ -2,7 +2,10 @@ import type { Rect, RenderedRef, VomResult } from "@browser-skill/vom";
import type { CdpTarget } from "@/browser-driver/frame-graph";
import { readRecordingDocumentIdentity } from "@/shared/recording-document-identity";
import type { CapturedSurfaceProbe } from "./capture";
import type { CapturedFrameDocument, FrameAxNode } from "./frame-capture";
import type {
FrameDocument as CapturedFrameDocument,
FrameOwnedAxNode as FrameAxNode,
} from "./frame-document";
/** Frame identity needed to resolve an `@eN` that lives in an iframe. */
export interface CaptureVomFrame {
@@ -61,8 +61,9 @@ export function buildSemanticGraph(input: BuildSemanticGraphInput): SemanticGrap
dom,
sourceOrder: index,
excluded:
document.frameId === rootFrameId &&
input.excludedBackendNodeIds?.has(dom.backendNodeId) === true,
document.excludedBackendNodeIds?.has(dom.backendNodeId) === true ||
(document.frameId === rootFrameId &&
input.excludedBackendNodeIds?.has(dom.backendNodeId) === true),
};
nodes.set(id, node);
frameNodeIds.push(id);
@@ -90,9 +91,10 @@ export function buildSemanticGraph(input: BuildSemanticGraphInput): SemanticGrap
ax,
sourceOrder: document.domNodes.length + index,
excluded:
document.frameId === rootFrameId &&
typeof ax.backendDOMNodeId === "number" &&
input.excludedBackendNodeIds?.has(ax.backendDOMNodeId) === true,
(document.excludedBackendNodeIds?.has(ax.backendDOMNodeId) === true ||
(document.frameId === rootFrameId &&
input.excludedBackendNodeIds?.has(ax.backendDOMNodeId) === true)),
});
frameNodeIds.push(id);
}
+224
View File
@@ -0,0 +1,224 @@
import { createCaptureCheckpoint } from "./capture-abort";
import type { CapturedNode } from "./capture-types";
import type { DecodedDocument, DecodedNode } from "./facts";
export const REQUESTED_STYLES = [
"position",
"pointer-events",
"cursor",
"visibility",
"opacity",
] as const;
const STYLE_COL = Object.fromEntries(
REQUESTED_STYLES.map((name, index) => [name, index]),
) as Record<(typeof REQUESTED_STYLES)[number], number>;
/** Sparse array format Chrome uses for infrequently-set per-node fields. */
interface SparseArray {
index: number[];
value: number[];
}
interface RareBooleanData {
index: number[];
}
export interface SnapshotDocument {
frameId?: string | number;
scrollOffsetX?: number;
scrollOffsetY?: number;
nodes?: {
parentIndex?: number[];
nodeName?: number[];
nodeType?: number[];
backendNodeId?: number[];
attributes?: number[][];
/**
* Per-node text value (index into strings), set for `#text` / CDATA nodes.
* Element nodes carry -1. Same length as `backendNodeId`.
*/
nodeValue?: number[];
/** Maps node array index → index into `documents[]` for frame content. */
contentDocumentIndex?: SparseArray;
inputValue?: SparseArray;
textValue?: SparseArray;
inputChecked?: RareBooleanData;
optionSelected?: RareBooleanData;
};
layout?: {
nodeIndex?: number[];
styles?: number[][];
bounds?: number[][];
paintOrders?: number[];
};
}
export function snapshotFrameId(document: SnapshotDocument, strings: string[]): string | undefined {
if (typeof document.frameId === "string") return document.frameId || undefined;
if (typeof document.frameId === "number") return str(strings, document.frameId) || undefined;
return undefined;
}
export interface SnapshotReply {
strings?: string[];
documents?: SnapshotDocument[];
}
function isSensitiveFormControl(node: Pick<CapturedNode, "tag" | "attrs">): boolean {
return node.tag === "input" && (node.attrs.type ?? "").toLowerCase() === "password";
}
function snapshotFormState(
value: string | undefined,
defaultValue: string,
hasDefaultValue: boolean,
sensitive: boolean,
): CapturedNode["formState"] {
if (sensitive) {
if (value === undefined && !hasDefaultValue) return undefined;
return (value ?? defaultValue) === "" ? "empty" : "filled";
}
if (value === undefined) return undefined;
if (value === "") return "empty";
return value === defaultValue ? "default" : "filled";
}
function str(strings: string[], idx: number | undefined): string {
if (idx === undefined || idx < 0) return "";
return strings[idx] ?? "";
}
export function sparseIndexMap(sparse: SparseArray | undefined): Map<number, number> {
const out = new Map<number, number>();
if (!sparse?.index || !sparse.value) return out;
for (let i = 0; i < sparse.index.length; i++) {
const nodeIndex = sparse.index[i];
const docIndex = sparse.value[i];
if (nodeIndex !== undefined && docIndex !== undefined) out.set(nodeIndex, docIndex);
}
return out;
}
export async function decodeDocument(
doc: SnapshotDocument,
strings: string[],
signal?: AbortSignal,
): Promise<DecodedDocument> {
const checkpoint = createCaptureCheckpoint(signal);
const dn = doc.nodes;
const dl = doc.layout;
if (!dn?.backendNodeId) {
return { nodes: [] };
}
const count = dn.backendNodeId.length;
const layoutByNode = new Map<number, number>();
for (let i = 0; i < (dl?.nodeIndex?.length ?? 0); i++) {
if (i % 256 === 0) {
const pending = checkpoint();
if (pending) await pending;
}
layoutByNode.set(dl!.nodeIndex![i], i);
}
const inputValues = sparseIndexMap(dn.inputValue);
const textValues = sparseIndexMap(dn.textValue);
const checkedInputs = new Set(dn.inputChecked?.index ?? []);
const selectedOptions = new Set(dn.optionSelected?.index ?? []);
// Collect visible text from #text child nodes so element CapturedNodes
// carry a textContent value usable as a button/link label fallback.
// nodeValue is a parallel array: string index for text nodes, -1 otherwise.
const nodeTextContent = new Map<number, string[]>();
if (dn.nodeValue) {
for (let n = 0; n < count; n++) {
if (n % 256 === 0) {
const pending = checkpoint();
if (pending) await pending;
}
const nvIdx = dn.nodeValue[n] ?? -1;
if (nvIdx < 0) continue;
const text = str(strings, nvIdx).trim();
if (!text) continue;
const parentIdx = dn.parentIndex?.[n] ?? -1;
if (parentIdx >= 0) {
const existing = nodeTextContent.get(parentIdx);
if (existing) existing.push(text);
else nodeTextContent.set(parentIdx, [text]);
}
}
}
// Decode fields without assigning a coordinate projection or semantic policy.
const nodes: DecodedNode[] = [];
for (let n = 0; n < count; n++) {
if (n % 256 === 0) {
const pending = checkpoint();
if (pending) await pending;
}
const backendNodeId = dn.backendNodeId[n];
const parentIdx = dn.parentIndex?.[n] ?? -1;
const parentBackendNodeId = parentIdx >= 0 ? (dn.backendNodeId[parentIdx] ?? null) : null;
const tag = str(strings, dn.nodeName?.[n]).toLowerCase();
const attrs: Record<string, string> = {};
const pairs = dn.attributes?.[n] ?? [];
for (let a = 0; a + 1 < pairs.length; a += 2) {
attrs[str(strings, pairs[a]).toLowerCase()] = str(strings, pairs[a + 1]);
}
const li = layoutByNode.get(n);
const styleRow = li === undefined ? [] : (dl?.styles?.[li] ?? []);
const styles: Record<string, string> = {};
for (const name of REQUESTED_STYLES) styles[name] = str(strings, styleRow[STYLE_COL[name]]);
const layout =
li === undefined
? undefined
: {
boundsSpace: "snapshot-document-layout" as const,
bounds: dl?.bounds?.[li],
styles,
};
const textContent = nodeTextContent.get(n)?.join(" ");
const rawFormValueIndex = tag === "textarea" ? textValues.get(n) : inputValues.get(n);
const rawFormValue =
rawFormValueIndex !== undefined ? str(strings, rawFormValueIndex) : undefined;
const sensitive = isSensitiveFormControl({ tag, attrs });
const formDefaultValue = attrs.value ?? "";
const formValue = sensitive ? undefined : rawFormValue;
const formState = snapshotFormState(
rawFormValue,
formDefaultValue,
Object.prototype.hasOwnProperty.call(attrs, "value"),
sensitive,
);
if (sensitive) delete attrs.value;
nodes.push({
backendNodeId,
nodeType: dn.nodeType?.[n],
parentBackendNodeId,
tag,
attrs,
layout,
paintOrder: li === undefined ? 0 : (dl?.paintOrders?.[li] ?? 0),
position: styles.position || "static",
pointerEvents: styles["pointer-events"] || "auto",
cursor: styles.cursor || "auto",
textContent,
...(formValue !== undefined ? { formValue } : {}),
...(tag === "input" || tag === "textarea"
? {
formPlaceholder: attrs.placeholder ?? "",
...(!sensitive ? { formDefaultValue } : {}),
...(formState ? { formState } : {}),
}
: {}),
...(checkedInputs.has(n) ? { formValue: "true", formState: "filled" } : {}),
...(selectedOptions.has(n) ? { formValue: attrs.value ?? textContent ?? "" } : {}),
});
}
return { nodes };
}
+6
View File
@@ -26,11 +26,17 @@ export type RpcErrorReason =
| "ref_not_found"
| "selector_not_found"
| "target_not_fillable"
| "fill_value_invalid"
| "fill_target_changed"
| "fill_focus_lost"
| "fill_value_mismatch"
| "fill_failed"
| "target_not_select"
| "option_not_found"
| "single_select_value_count"
| "tab_not_active"
| "restricted_tab_url"
| "cdp_extension_access_denied"
| "borrow_conflict"
| "screenshot_capture_failed"
| "file_input_probe_failed"
+4
View File
@@ -172,6 +172,10 @@ intended. Read `bsk upload --help` and `bsk download --help` for all flags and e
- Stale ref: observe again and retry the intended action once.
- Unknown tab or session: list current tabs/sessions; never guess identifiers.
- Timeout: inspect current page state before deciding whether one longer purposeful wait is useful.
- Fill result unconfirmed (`fill_value_mismatch`): observe the field first; the page may have
formatted the value. Continue if the visible result satisfies the user's intent. Otherwise correct
the remaining difference; do not blindly repeat fill or immediately request human help. For other
fill errors, follow the returned hint and inspect current state before retrying.
- Unsupported command: continue with available capabilities; suggest updating only when the missing
command is necessary.
- Unrecoverable failure: report the blocker and stop the session in a finally-style path.
+103 -6
View File
@@ -73,6 +73,14 @@ where
P: Serialize + Send + 'static,
R: DeserializeOwned + Send + 'static,
{
let parent_cancel = stdin_cancellation();
if parent_cancel.as_ref().is_some_and(|rx| *rx.borrow()) {
return Err(CliError::from_rpc(RpcError {
code: ErrorCode::Cancelled,
message: "parent closed the cancellation pipe".into(),
data: Some(serde_json::json!({"reason": "cancelled_before_dispatch"})),
}));
}
let rpc_id: RpcId = format!("{}-{}", rpc_id_prefix, random_short_id());
let mut client = IpcClient::connect(sock.clone()).await?;
let rpc_id_for_call = rpc_id.clone();
@@ -88,16 +96,16 @@ where
let outcome = tokio::select! {
biased;
res = &mut main_fut => res?,
sig = wait_for_sigint() => {
sig.context("install SIGINT handler").map_err(CliError::Local)?;
debug!(rpc_id = %rpc_id_for_cancel, "SIGINT: forwarding cancel to daemon");
sig = wait_for_cancel(parent_cancel) => {
sig.context("wait for cancellation").map_err(CliError::Local)?;
debug!(rpc_id = %rpc_id_for_cancel, "forwarding cancel to daemon");
let _ = send_cancel(&sock, &rpc_id_for_cancel).await;
match tokio::time::timeout(CANCEL_RESPONSE_GRACE, &mut main_fut).await {
Ok(res) => res?,
Err(_) => {
debug!(
rpc_id = %rpc_id_for_cancel,
"SIGINT: cancel grace elapsed; synthesising cancelled error"
"cancel grace elapsed; synthesising cancelled error"
);
return Err(CliError::from_rpc(RpcError {
code: ErrorCode::Cancelled,
@@ -112,6 +120,46 @@ where
outcome.map_err(CliError::from_rpc)
}
/// Release caller-owned staging even after cancellation. This deliberately
/// exposes only transfer.release, not a general bypass for business RPCs.
/// The entire batch, including connection setup, shares one bounded budget.
pub fn release_transfers<'a>(
sock: &Path,
transfer_ids: impl IntoIterator<Item = &'a str>,
) -> Result<(), CliError> {
use bsk_protocol::tools::{TransferIdParams, TransferReleaseResult};
const CLEANUP_TIMEOUT: Duration = Duration::from_secs(5);
let ids: Vec<_> = transfer_ids.into_iter().collect();
if ids.is_empty() {
return Ok(());
}
let rt = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.context("build runtime for transfer cleanup")?;
rt.block_on(async {
tokio::time::timeout(CLEANUP_TIMEOUT, async {
let mut client = IpcClient::connect(sock).await?;
for id in ids {
client
.call::<_, TransferReleaseResult>(
"transfer-release",
Method::TransferRelease,
Some(TransferIdParams {
transfer_id: id.to_string(),
}),
CLEANUP_TIMEOUT,
)
.await?
.map_err(CliError::from_rpc)?;
}
Ok::<_, CliError>(())
})
.await
.context("transfer cleanup exceeded its total time budget")?
})
}
/// Send a `cancel { rpc_id }` frame over a fresh connection so it
/// lands on the daemon while the original call is still parked.
///
@@ -146,8 +194,57 @@ fn random_short_id() -> String {
bytes.iter().map(|b| format!("{b:02x}")).collect()
}
/// Wait for either Ctrl-C or process SIGTERM. Returning here means
/// the user has asked us to stop.
/// Opt-in control pipe for parent processes that cannot send Ctrl-C (Node on
/// Windows). One reader survives the short-lived runtimes used by multi-RPC
/// commands such as upload. Ordinary CLI stdin and terminal Ctrl-C are unchanged.
fn stdin_cancellation() -> Option<tokio::sync::watch::Receiver<bool>> {
if std::env::var("BSK_CANCEL_ON_STDIN_CLOSE").as_deref() != Ok("1") {
return None;
}
static CANCEL: std::sync::OnceLock<tokio::sync::watch::Receiver<bool>> =
std::sync::OnceLock::new();
Some(
CANCEL
.get_or_init(|| {
let (tx, rx) = tokio::sync::watch::channel(false);
std::thread::spawn(move || {
use std::io::Read;
let mut stdin = std::io::stdin().lock();
let mut buf = [0; 64];
loop {
match stdin.read(&mut buf) {
Ok(0) => break,
Ok(_) => {}
Err(err) if err.kind() == std::io::ErrorKind::Interrupted => continue,
Err(_) => break,
}
}
let _ = tx.send(true);
});
rx
})
.clone(),
)
}
async fn wait_for_cancel(
parent_cancel: Option<tokio::sync::watch::Receiver<bool>>,
) -> anyhow::Result<()> {
let parent_closed = async move {
match parent_cancel {
Some(mut rx) => {
let _ = rx.wait_for(|closed| *closed).await;
}
None => std::future::pending::<()>().await,
}
};
tokio::select! {
result = wait_for_sigint() => result,
_ = parent_closed => Ok(()),
}
}
/// Wait for terminal Ctrl-C.
async fn wait_for_sigint() -> anyhow::Result<()> {
tokio::signal::ctrl_c().await.context("listen for SIGINT")?;
Ok(())
+2 -9
View File
@@ -9,8 +9,7 @@ use anyhow::Context;
use base64::Engine;
use bsk_protocol::Method;
use bsk_protocol::tools::{
DownloadParams, DownloadResult, TransferChunkParams, TransferChunkResult, TransferIdParams,
TransferReleaseResult,
DownloadParams, DownloadResult, TransferChunkParams, TransferChunkResult,
};
use clap::Args;
use uuid::Uuid;
@@ -70,13 +69,7 @@ pub fn dispatch(args: DownloadArgs, format: Format) -> Result<(), CliError> {
CliError::Local(anyhow::anyhow!("daemon returned no download transfer id"))
})?;
let write_result = write_transfer(&info.sock_path, &transfer_id, &args.out, args.overwrite);
let _: Result<TransferReleaseResult, CliError> = crate::cli::business_rpc::call(
info.sock_path,
"transfer-release",
Method::TransferRelease,
Some(TransferIdParams { transfer_id }),
Duration::from_secs(5),
);
let _ = crate::cli::business_rpc::release_transfers(&info.sock_path, [transfer_id.as_str()]);
write_result?;
match format {
Format::Json => {
+111 -1
View File
@@ -43,10 +43,16 @@ pub mod reason {
pub const REF_NOT_FOUND: &str = "ref_not_found";
pub const SELECTOR_NOT_FOUND: &str = "selector_not_found";
pub const TARGET_NOT_FILLABLE: &str = "target_not_fillable";
pub const FILL_VALUE_INVALID: &str = "fill_value_invalid";
pub const FILL_TARGET_CHANGED: &str = "fill_target_changed";
pub const FILL_FOCUS_LOST: &str = "fill_focus_lost";
pub const FILL_VALUE_MISMATCH: &str = "fill_value_mismatch";
pub const FILL_FAILED: &str = "fill_failed";
pub const TARGET_NOT_SELECT: &str = "target_not_select";
pub const OPTION_NOT_FOUND: &str = "option_not_found";
pub const SINGLE_SELECT_VALUE_COUNT: &str = "single_select_value_count";
pub const TAB_NOT_ACTIVE: &str = "tab_not_active";
pub const CDP_EXTENSION_ACCESS_DENIED: &str = "cdp_extension_access_denied";
pub const BORROW_CONFLICT: &str = "borrow_conflict";
pub const SCREENSHOT_CAPTURE_FAILED: &str = "screenshot_capture_failed";
pub const FILE_INPUT_PROBE_FAILED: &str = "file_input_probe_failed";
@@ -206,6 +212,13 @@ pub fn info_for_error(code: ErrorCode, data: Option<&serde_json::Value>) -> Rend
return base;
};
match (code, reason) {
(ErrorCode::CdpFailed, reason::CDP_EXTENSION_ACCESS_DENIED) => RenderInfo {
summary: "Chrome blocked CDP access to another extension's content in this tab",
hint: Some(
"a web page can contain a restricted extension frame; disable the conflicting extension and reload, or use `bsk navigate <url>` to leave this page; reconnecting BrowserSkill alone does not remove the restriction",
),
exit_code: base.exit_code,
},
(_, reason::TRANSFER_OUTCOME_UNKNOWN) => RenderInfo {
summary: "the file transfer outcome could not be confirmed",
hint: Some(
@@ -247,7 +260,42 @@ pub fn info_for_error(code: ErrorCode, data: Option<&serde_json::Value>) -> Rend
(ErrorCode::InvalidParams, reason::TARGET_NOT_FILLABLE) => RenderInfo {
summary: "target element is not fillable",
hint: Some(
"choose an input, textarea, or contenteditable element from the latest snapshot",
"observe the page and choose an enabled, editable text input, textarea, or contenteditable; use the appropriate interaction for other control types",
),
exit_code: base.exit_code,
},
(ErrorCode::InvalidParams, reason::FILL_VALUE_INVALID) => RenderInfo {
summary: "the requested text does not fit the field constraints",
hint: Some(
"check the field's input type and maxlength; use a compatible value without silently truncating or changing the user's intended data",
),
exit_code: base.exit_code,
},
(ErrorCode::CdpFailed, reason::FILL_TARGET_CHANGED) => RenderInfo {
summary: "the fill target changed during the action",
hint: Some(
"observe the current page and field value; if the intended result is missing, use a fresh editable target before retrying",
),
exit_code: base.exit_code,
},
(ErrorCode::CdpFailed, reason::FILL_FOCUS_LOST) => RenderInfo {
summary: "the page moved focus away from the fill target",
hint: Some(
"observe the page for a dialog or rerender and resolve it before retrying; fill manages field focus and does not require bringing the browser to the foreground",
),
exit_code: base.exit_code,
},
(ErrorCode::CdpFailed, reason::FILL_VALUE_MISMATCH) => RenderInfo {
summary: "the fill result could not be confirmed",
hint: Some(
"observe the field before retrying; the page may have formatted the value. Continue if the visible result satisfies the user's intent; otherwise correct the remaining difference. Do not blindly repeat fill or immediately request human help",
),
exit_code: base.exit_code,
},
(ErrorCode::CdpFailed, reason::FILL_FAILED) => RenderInfo {
summary: "fill encountered a browser or page-script error",
hint: Some(
"observe the page before retrying because the field may already have changed; retry only if the intended result is missing and the target is still available",
),
exit_code: base.exit_code,
},
@@ -442,6 +490,56 @@ mod tests {
ErrorCode::NoBrowserConnected,
];
#[test]
fn fill_reasons_provide_recovery_without_changing_exit_codes() {
for (code, reason, hint_fragment) in [
(
ErrorCode::InvalidParams,
reason::TARGET_NOT_FILLABLE,
"enabled, editable",
),
(
ErrorCode::InvalidParams,
reason::FILL_VALUE_INVALID,
"maxlength",
),
(
ErrorCode::CdpFailed,
reason::FILL_TARGET_CHANGED,
"fresh editable target",
),
(
ErrorCode::CdpFailed,
reason::FILL_FOCUS_LOST,
"does not require bringing the browser",
),
(
ErrorCode::CdpFailed,
reason::FILL_VALUE_MISMATCH,
"Continue if the visible result satisfies",
),
(
ErrorCode::CdpFailed,
reason::FILL_FAILED,
"may already have changed",
),
] {
let data = serde_json::json!({ "reason": reason });
let info = info_for_error(code, Some(&data));
assert!(info.hint.unwrap().contains(hint_fragment), "{reason}");
assert_eq!(info.exit_code, info_for(code).exit_code);
}
let mismatch = serde_json::json!({ "reason": reason::FILL_VALUE_MISMATCH });
let info = info_for_error(ErrorCode::CdpFailed, Some(&mismatch));
assert_eq!(info.summary, "the fill result could not be confirmed");
assert!(info.hint.unwrap().contains("Do not blindly repeat fill"));
let unknown = serde_json::json!({ "reason": "future_fill_reason" });
assert_eq!(
info_for_error(ErrorCode::CdpFailed, Some(&unknown)),
info_for(ErrorCode::CdpFailed)
);
}
#[test]
fn element_not_visible_overrides_permission_denied_copy() {
let data = serde_json::json!({ "reason": reason::ELEMENT_NOT_VISIBLE });
@@ -454,6 +552,18 @@ mod tests {
assert!(!info.summary.contains("sandbox"));
}
#[test]
fn extension_access_denied_explains_the_frame_restriction_and_navigation_recovery() {
let data = serde_json::json!({ "reason": reason::CDP_EXTENSION_ACCESS_DENIED });
let info = info_for_error(ErrorCode::CdpFailed, Some(&data));
assert!(info.summary.contains("another extension"));
let hint = info.hint.unwrap();
assert!(hint.contains("restricted extension frame"));
assert!(hint.contains("bsk navigate <url>"));
assert!(hint.contains("disable the conflicting extension"));
assert_eq!(info.exit_code, 3);
}
#[test]
fn session_busy_overrides_timeout_copy() {
let data = serde_json::json!({ "reason": reason::SESSION_BUSY });
+238 -73
View File
@@ -6,9 +6,7 @@ use std::path::{Path, PathBuf};
use std::time::{Duration, SystemTime};
#[cfg(windows)]
use std::os::windows::process::CommandExt;
#[cfg(windows)]
use std::process::{Command, Stdio};
mod windows;
use anyhow::{Context, Result, bail};
use clap::Args;
@@ -363,7 +361,8 @@ fn install_candidate_with_client(
crate::daemon::start::run_stop().context("stop bsk daemon before update")?;
}
let action = replace_binary_for_update(&target, &binary, daemon_was_running)?;
let restart_args = daemon_was_running.then(StartArgs::default);
let action = replace_binary_for_update(&target, &binary, restart_args.as_ref())?;
if daemon_was_running && matches!(action, InstallAction::Replaced) {
crate::daemon::start::run_start(StartArgs::default())
@@ -394,8 +393,8 @@ pub(crate) fn download_candidate_binary(
/// Daemon-side install: download, verify, and replace the executable at
/// `target` (on Windows: stage the replacement next to it). Unlike the
/// CLI path this never stops or starts the daemon — the daemon drives
/// its own restart once the binary is replaced.
/// CLI path this never stops the daemon. On Windows the helper starts
/// its replacement after exit; elsewhere the daemon drives its own restart.
///
/// `target` must be captured *before* any replacement happens: on Linux
/// `std::env::current_exe` starts returning a ` (deleted)`-suffixed
@@ -403,10 +402,11 @@ pub(crate) fn download_candidate_binary(
pub(crate) fn self_install_candidate(
candidate: &UpdateCandidate,
target: &Path,
restart_args: &StartArgs,
) -> Result<InstallAction> {
let client = update_http_client(ARCHIVE_FETCH_TIMEOUT)?;
let binary = download_candidate_binary(candidate, &client)?;
replace_binary_at_path(target, &binary)
replace_binary_for_update(target, &binary, Some(restart_args))
}
/// Outcome of one daemon auto-update step (see [`auto_update_step`]).
@@ -423,8 +423,8 @@ pub(crate) enum AutoUpdateOutcome {
/// The new binary replaced the old one; the daemon should now
/// restart into it.
Replaced { latest: String },
/// Windows staged the replacement next to the running binary; a
/// daemon/terminal restart finishes it.
/// Windows started a helper that will replace the executable and
/// restart the daemon once the current process exits.
Staged { latest: String },
}
@@ -668,6 +668,7 @@ fn render_report(format: Format, report: &UpdateReport) -> Result<()> {
}
if matches!(report.install_action, Some("staged")) {
println!("the detached update helper will apply the replacement after exit");
println!("if it fails, see the *.update-*.log file next to the executable");
}
}
Format::Json => {
@@ -688,22 +689,22 @@ pub fn extract_bsk_binary(archive_bytes: &[u8], kind: ArchiveKind) -> Result<Vec
}
pub fn replace_binary_at_path(target: &Path, binary: &[u8]) -> Result<InstallAction> {
replace_binary_for_update(target, binary, false)
replace_binary_for_update(target, binary, None)
}
fn replace_binary_for_update(
target: &Path,
binary: &[u8],
restart_daemon: bool,
restart_args: Option<&StartArgs>,
) -> Result<InstallAction> {
#[cfg(windows)]
{
stage_windows_replacement(target, binary, restart_daemon)
stage_windows_replacement(target, binary, restart_args)
}
#[cfg(not(windows))]
{
let _ = restart_daemon;
let _ = restart_args;
replace_binary_atomically(target, binary)
}
}
@@ -749,33 +750,69 @@ fn replace_binary_atomically(target: &Path, binary: &[u8]) -> Result<InstallActi
fn stage_windows_replacement(
target: &Path,
binary: &[u8],
restart_daemon: bool,
restart_args: Option<&StartArgs>,
) -> Result<InstallAction> {
let paths = staged_replacement_paths(target, std::process::id())?;
std::fs::write(&paths.binary_path, binary)
.with_context(|| format!("write {}", paths.binary_path.display()))?;
std::fs::write(
&paths.script_path,
windows_replacement_script(target, &paths.binary_path, restart_daemon),
)
.with_context(|| format!("write {}", paths.script_path.display()))?;
// The helper owns replacement/restart after this process exits. Do not
// report Staged until it has actually begun executing the script.
let _child = spawn_windows_replacement(target, binary, restart_args, 120)?;
Ok(InstallAction::Staged)
}
const CREATE_NO_WINDOW: u32 = 0x0800_0000;
let command = format!("\"{}\"", paths.script_path.display());
if let Err(err) = Command::new("cmd.exe")
.args(["/D", "/S", "/C"])
.arg(command)
.stdin(Stdio::null())
.stdout(Stdio::null())
.stderr(Stdio::null())
.creation_flags(CREATE_NO_WINDOW)
.spawn()
{
#[cfg(windows)]
fn spawn_windows_replacement(
target: &Path,
binary: &[u8],
restart_args: Option<&StartArgs>,
attempts: u32,
) -> Result<windows::Helper> {
let paths = staged_replacement_paths(target, std::process::id())?;
let ready_path = paths.script_path.with_extension("ready");
let log_path = paths.script_path.with_extension("log");
let result = (|| {
// A previous failed attempt by this process must not acknowledge a
// new helper. All paths are private to this target and process id.
match std::fs::remove_file(&ready_path) {
Ok(()) => {}
Err(err) if err.kind() == std::io::ErrorKind::NotFound => {}
Err(err) => return Err(err.into()),
}
std::fs::write(&paths.binary_path, binary)
.with_context(|| format!("write {}", paths.binary_path.display()))?;
std::fs::write(
&paths.script_path,
windows_replacement_script(restart_args, attempts),
)
.with_context(|| format!("write {}", paths.script_path.display()))?;
let mut child = windows::spawn(
&paths.script_path,
&paths.binary_path,
target,
&ready_path,
&log_path,
)
.context("launch detached Windows update helper")?;
let deadline = std::time::Instant::now() + Duration::from_secs(5);
loop {
if std::fs::remove_file(&ready_path).is_ok() {
return Ok(child);
}
if let Some(status) = child.try_wait()? {
bail!("Windows update helper exited before becoming ready: {status}");
}
if std::time::Instant::now() >= deadline {
let _ = child.kill();
let _ = child.wait();
bail!("Windows update helper did not become ready within 5 seconds");
}
std::thread::sleep(Duration::from_millis(20));
}
})();
if result.is_err() {
let _ = std::fs::remove_file(&paths.binary_path);
let _ = std::fs::remove_file(&paths.script_path);
return Err(err).context("launch detached Windows update helper");
let _ = std::fs::remove_file(&ready_path);
}
Ok(InstallAction::Staged)
result.with_context(|| format!("Windows update helper failed; see {}", log_path.display()))
}
pub fn staged_replacement_paths(target: &Path, pid: u32) -> Result<StagedReplacementPaths> {
@@ -792,25 +829,43 @@ pub fn staged_replacement_paths(target: &Path, pid: u32) -> Result<StagedReplace
}
#[cfg(any(windows, test))]
fn windows_replacement_script(target: &Path, staged_binary: &Path, restart_daemon: bool) -> String {
let target = target.display().to_string().replace('%', "%%");
let staged_binary = staged_binary.display().to_string().replace('%', "%%");
let restart = if restart_daemon {
format!("start \"\" /B \"{target}\" daemon start >nul 2>nul\r\n")
} else {
String::new()
};
fn windows_replacement_script(restart_args: Option<&StartArgs>, attempts: u32) -> String {
// Only fixed switches and numeric values go into the ASCII script. Paths
// stay in Unicode environment variables, including %, ! and shell symbols.
let restart = restart_args.map_or_else(String::new, |args| {
format!(
"\"%BSK_UPDATE_TARGET%\" daemon start --port {} --session-idle {}ms --daemon-idle {}ms\r\nif errorlevel 1 goto failed_restart\r\n",
args.resolved_port(),
args.resolved_session_idle().as_millis(),
args.resolved_daemon_idle().as_millis(),
)
});
format!(
"@echo off\r\n\
setlocal\r\n\
setlocal DisableDelayedExpansion\r\n\
> \"%BSK_UPDATE_READY%\" echo ready\r\n\
if errorlevel 1 exit /b 1\r\n\
set /a attempts=0\r\n\
:retry\r\n\
move /Y \"{staged_binary}\" \"{target}\" >nul 2>nul\r\n\
if errorlevel 1 (\r\n\
timeout /t 1 /nobreak >nul\r\n\
goto retry\r\n\
)\r\n\
move /Y \"%BSK_UPDATE_SOURCE%\" \"%BSK_UPDATE_TARGET%\" >nul\r\n\
if not errorlevel 1 goto replaced\r\n\
set /a attempts+=1\r\n\
if %attempts% geq {attempts} goto failed_replace\r\n\
if not exist \"%BSK_UPDATE_SOURCE%\" goto failed_replace\r\n\
ping -n 2 127.0.0.1 >nul\r\n\
goto retry\r\n\
:failed_replace\r\n\
echo Failed to replace the executable after %attempts% attempts.\r\n\
goto failed\r\n\
:replaced\r\n\
{restart}\
del /F /Q \"%~f0\" >nul 2>nul\r\n"
del /F /Q \"%BSK_UPDATE_LOG%\" >nul 2>nul\r\n\
(del /F /Q \"%BSK_UPDATE_SCRIPT%\" >nul 2>nul & exit 0)\r\n\
:failed_restart\r\n\
echo Replaced the executable but failed to restart the daemon.\r\n\
:failed\r\n\
del /F /Q \"%BSK_UPDATE_SOURCE%\" >nul 2>nul\r\n\
(del /F /Q \"%BSK_UPDATE_SCRIPT%\" >nul 2>nul & exit 1)\r\n"
)
}
@@ -964,6 +1019,7 @@ mod tests {
assert_eq!(extracted, b"windows binary");
}
#[cfg(not(windows))]
#[test]
fn replaces_binary_at_path() {
let tmp = tempfile::TempDir::new().unwrap();
@@ -998,29 +1054,139 @@ mod tests {
}
#[test]
fn windows_replacement_script_retries_restarts_and_cleans_itself() {
let script = windows_replacement_script(
Path::new(r"C:\Program Files\bsk.exe"),
Path::new(r"C:\Program Files\bsk.exe.update-42"),
true,
fn windows_replacement_script_preserves_restart_config() {
let args = StartArgs {
port: Some(54321),
session_idle: Some(Duration::from_millis(1234)),
daemon_idle: Some(Duration::from_secs(75)),
..Default::default()
};
let script = windows_replacement_script(Some(&args), 120);
assert!(script.is_ascii());
assert!(
script
.contains("daemon start --port 54321 --session-idle 1234ms --daemon-idle 75000ms")
);
assert!(script.contains(":retry"));
assert!(script.contains("move /Y"));
assert!(script.contains(r#"start "" /B "C:\Program Files\bsk.exe" daemon start"#));
assert!(script.contains(r#"del /F /Q "%~f0""#));
assert!(!windows_replacement_script(None, 120).contains("daemon start"));
}
#[test]
fn windows_replacement_script_omits_restart_when_daemon_was_not_running() {
let script = windows_replacement_script(
Path::new(r"C:\bsk.exe"),
Path::new(r"C:\bsk.exe.update-42"),
false,
);
#[cfg(windows)]
fn wait_for_helper(mut child: windows::Helper) -> std::process::ExitStatus {
let deadline = std::time::Instant::now() + Duration::from_secs(10);
loop {
if let Some(status) = child.try_wait().unwrap() {
return status;
}
if std::time::Instant::now() >= deadline {
let _ = child.kill();
let _ = child.wait();
panic!("Windows update helper did not exit");
}
std::thread::sleep(Duration::from_millis(20));
}
}
assert!(!script.contains("daemon start"));
assert!(script.contains(r#"del /F /Q "%~f0""#));
#[cfg(windows)]
#[test]
fn windows_helper_replaces_in_unicode_spaces_and_shell_symbol_paths() {
let tmp = tempfile::TempDir::new().unwrap();
for name in [
"ascii",
"with space",
"中文目录",
"literal %PATH% ! & (folder)",
] {
let dir = tmp.path().join(name);
std::fs::create_dir(&dir).unwrap();
let target = dir.join("bsk.exe");
std::fs::write(&target, b"old binary").unwrap();
let child = spawn_windows_replacement(&target, b"new binary", None, 5).unwrap();
let paths = staged_replacement_paths(&target, std::process::id()).unwrap();
let status = wait_for_helper(child);
assert!(
status.success(),
"{name}: {:?}",
std::fs::read_to_string(paths.script_path.with_extension("log"))
);
assert_eq!(std::fs::read(&target).unwrap(), b"new binary", "{name}");
// Successful updates leave no helper, staged binary, ready file or log.
assert_eq!(std::fs::read_dir(&dir).unwrap().count(), 1, "{name}");
}
}
#[cfg(windows)]
#[test]
fn windows_helper_waits_for_unlock_then_replaces() {
use std::os::windows::fs::OpenOptionsExt;
let tmp = tempfile::TempDir::new().unwrap();
let target = tmp.path().join("bsk.exe");
std::fs::write(&target, b"old binary").unwrap();
// Deny delete sharing, like a running Windows executable.
let lock = std::fs::OpenOptions::new()
.read(true)
.share_mode(1)
.open(&target)
.unwrap();
let mut child = spawn_windows_replacement(&target, b"new binary", None, 5).unwrap();
std::thread::sleep(Duration::from_millis(150));
assert!(child.try_wait().unwrap().is_none());
assert_eq!(std::fs::read(&target).unwrap(), b"old binary");
drop(lock);
assert!(wait_for_helper(child).success());
assert_eq!(std::fs::read(&target).unwrap(), b"new binary");
}
#[cfg(windows)]
#[test]
fn windows_helper_bounds_retries_and_preserves_old_binary_on_failure() {
use std::os::windows::fs::OpenOptionsExt;
let tmp = tempfile::TempDir::new().unwrap();
let target = tmp.path().join("bsk.exe");
std::fs::write(&target, b"old binary").unwrap();
let _lock = std::fs::OpenOptions::new()
.read(true)
.share_mode(1)
.open(&target)
.unwrap();
let child = spawn_windows_replacement(&target, b"new binary", None, 2).unwrap();
assert!(!wait_for_helper(child).success());
let paths = staged_replacement_paths(&target, std::process::id()).unwrap();
assert_eq!(std::fs::read(&target).unwrap(), b"old binary");
assert!(!paths.binary_path.exists());
assert!(!paths.script_path.exists());
let log_bytes = std::fs::read(paths.script_path.with_extension("log")).unwrap();
let log = String::from_utf8_lossy(&log_bytes);
assert!(
log.contains("Failed to replace the executable after 2 attempts."),
"{log}"
);
}
#[cfg(windows)]
#[test]
fn windows_helper_reports_restart_failure_without_losing_replacement() {
let tmp = tempfile::TempDir::new().unwrap();
let target = tmp.path().join("bsk.exe");
std::fs::write(&target, b"old binary").unwrap();
// Not a valid PE executable: replacement succeeds but restart fails.
let child = spawn_windows_replacement(
&target,
b"invalid executable",
Some(&StartArgs::default()),
2,
)
.unwrap();
assert!(!wait_for_helper(child).success());
let paths = staged_replacement_paths(&target, std::process::id()).unwrap();
let bytes = std::fs::read(paths.script_path.with_extension("log")).unwrap();
let log = String::from_utf8_lossy(&bytes);
assert!(
log.contains("Replaced the executable but failed to restart the daemon."),
"{log}"
);
assert_eq!(std::fs::read(&target).unwrap(), b"invalid executable");
assert!(!paths.script_path.exists());
assert!(!paths.binary_path.exists());
}
#[test]
@@ -1279,10 +1445,9 @@ mod tests {
}
#[test]
fn auto_update_step_staged_outcome_requests_no_restart() {
// The Windows shape: the replacement is staged next to the
// running binary, so the outcome must not ask for the immediate
// self-restart that `Replaced` triggers.
fn auto_update_step_reports_staged_handoff() {
// The helper, rather than this process, starts the new executable.
// The daemon must exit to let that staged handoff finish.
let candidate = test_candidate();
let outcome =
auto_update_step(Some(&candidate), true, 0, |_| Ok(InstallAction::Staged)).unwrap();
+203
View File
@@ -0,0 +1,203 @@
//! Launch the update helper with only its stdio handles inherited.
//!
//! std::process::Command on Windows also inherits other inheritable handles.
//! A daemon's listening socket must not survive in the helper: it would keep
//! the port occupied after the daemon exits and prevent its replacement starting.
use std::ffi::OsStr;
use std::fs::File;
use std::io;
use std::os::windows::ffi::OsStrExt;
use std::os::windows::io::{AsRawHandle, FromRawHandle, OwnedHandle};
use std::os::windows::process::ExitStatusExt;
use std::path::Path;
use std::process::ExitStatus;
use windows_sys::Win32::Foundation::{
HANDLE, HANDLE_FLAG_INHERIT, SetHandleInformation, WAIT_OBJECT_0, WAIT_TIMEOUT,
};
use windows_sys::Win32::System::Threading::{
CREATE_NEW_PROCESS_GROUP, CREATE_NO_WINDOW, CREATE_UNICODE_ENVIRONMENT, CreateProcessW,
DeleteProcThreadAttributeList, EXTENDED_STARTUPINFO_PRESENT, GetExitCodeProcess,
InitializeProcThreadAttributeList, PROC_THREAD_ATTRIBUTE_HANDLE_LIST, PROCESS_INFORMATION,
STARTF_USESTDHANDLES, STARTUPINFOEXW, TerminateProcess, UpdateProcThreadAttribute,
WaitForSingleObject,
};
pub(super) struct Helper(OwnedHandle);
impl Helper {
pub(super) fn try_wait(&mut self) -> io::Result<Option<ExitStatus>> {
// SAFETY: the owned process handle remains valid for both calls.
match unsafe { WaitForSingleObject(self.0.as_raw_handle(), 0) } {
WAIT_OBJECT_0 => {
let mut code = 0;
if unsafe { GetExitCodeProcess(self.0.as_raw_handle(), &mut code) } == 0 {
return Err(io::Error::last_os_error());
}
Ok(Some(ExitStatus::from_raw(code)))
}
WAIT_TIMEOUT => Ok(None),
_ => Err(io::Error::last_os_error()),
}
}
pub(super) fn kill(&mut self) -> io::Result<()> {
// SAFETY: this handle belongs to the helper we created.
if unsafe { TerminateProcess(self.0.as_raw_handle(), 1) } == 0 {
return Err(io::Error::last_os_error());
}
Ok(())
}
pub(super) fn wait(&mut self) -> io::Result<()> {
// Only used after kill(); do not leave a live helper on startup failure.
if unsafe { WaitForSingleObject(self.0.as_raw_handle(), 5000) } != WAIT_OBJECT_0 {
return Err(io::Error::new(
io::ErrorKind::TimedOut,
"update helper did not exit",
));
}
Ok(())
}
}
// The opaque attribute list needs pointer-aligned storage and explicit teardown.
struct AttributeList(Vec<usize>);
impl Drop for AttributeList {
fn drop(&mut self) {
// SAFETY: this wrapper is constructed only after successful initialization.
unsafe { DeleteProcThreadAttributeList(self.0.as_mut_ptr().cast()) };
}
}
pub(super) fn spawn(
script: &Path,
source: &Path,
target: &Path,
ready: &Path,
log_path: &Path,
) -> io::Result<Helper> {
let root = std::env::var_os("SystemRoot")
.ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, "SystemRoot is not set"))?;
let application = wide(Path::new(&root).join("System32/cmd.exe").as_os_str());
// /S /C strips one outer pair of quotes. Environment expansion keeps paths
// Unicode and avoids CRT escaping, batch-file encoding, and CALL expansion.
let mut command = wide(OsStr::new(
"cmd.exe /D /V:OFF /S /C \"\"%BSK_UPDATE_SCRIPT%\"\"",
));
let overrides = [
("BSK_UPDATE_SCRIPT", script.as_os_str()),
("BSK_UPDATE_SOURCE", source.as_os_str()),
("BSK_UPDATE_TARGET", target.as_os_str()),
("BSK_UPDATE_READY", ready.as_os_str()),
("BSK_UPDATE_LOG", log_path.as_os_str()),
];
let mut env: Vec<_> = std::env::vars_os()
.filter(|(key, _)| {
let key = key.to_string_lossy();
!overrides
.iter()
.any(|(name, _)| key.eq_ignore_ascii_case(name))
&& !key.eq_ignore_ascii_case(crate::daemon::start::DAEMONIZED_ENV)
&& !key.eq_ignore_ascii_case(crate::daemon::start::DAEMON_REPLACEMENT_WAIT_ENV)
})
.collect();
env.extend(
overrides
.into_iter()
.map(|(key, value)| (key.into(), value.to_owned())),
);
env.sort_by_key(|(key, _)| key.to_string_lossy().to_uppercase());
let mut environment = Vec::new();
for (key, value) in env {
environment.extend(key.encode_wide());
environment.push(b'=' as u16);
environment.extend(value.encode_wide());
environment.push(0);
}
environment.push(0);
let input = File::open("NUL")?;
let log = File::create(log_path)?;
let handles: [HANDLE; 2] = [input.as_raw_handle(), log.as_raw_handle()];
for &handle in &handles {
// SAFETY: these are dedicated helper stdio handles, held until spawn
// finishes. The explicit handle list excludes every other parent handle.
if unsafe { SetHandleInformation(handle, HANDLE_FLAG_INHERIT, HANDLE_FLAG_INHERIT) } == 0 {
return Err(io::Error::last_os_error());
}
}
let mut size = 0;
// SAFETY: the first call queries the allocation size, as required by Win32.
unsafe { InitializeProcThreadAttributeList(std::ptr::null_mut(), 1, 0, &mut size) };
if size == 0 {
return Err(io::Error::last_os_error());
}
let mut storage = vec![0usize; size.div_ceil(std::mem::size_of::<usize>())];
// SAFETY: storage has the requested size and pointer alignment.
if unsafe { InitializeProcThreadAttributeList(storage.as_mut_ptr().cast(), 1, 0, &mut size) }
== 0
{
return Err(io::Error::last_os_error());
}
let mut attributes = AttributeList(storage);
// SAFETY: the handle array and attribute storage outlive CreateProcessW.
if unsafe {
UpdateProcThreadAttribute(
attributes.0.as_mut_ptr().cast(),
0,
PROC_THREAD_ATTRIBUTE_HANDLE_LIST as usize,
handles.as_ptr().cast(),
std::mem::size_of_val(&handles),
std::ptr::null_mut(),
std::ptr::null(),
)
} == 0
{
return Err(io::Error::last_os_error());
}
// SAFETY: these Win32 structs permit zero initialization; required fields
// are populated below before the process is created.
let mut startup: STARTUPINFOEXW = unsafe { std::mem::zeroed() };
startup.StartupInfo.cb = std::mem::size_of::<STARTUPINFOEXW>() as u32;
startup.StartupInfo.dwFlags = STARTF_USESTDHANDLES;
startup.StartupInfo.hStdInput = handles[0];
startup.StartupInfo.hStdOutput = handles[1];
startup.StartupInfo.hStdError = handles[1];
startup.lpAttributeList = attributes.0.as_mut_ptr().cast();
let mut process: PROCESS_INFORMATION = unsafe { std::mem::zeroed() };
// SAFETY: strings are NUL-terminated UTF-16, the environment is double-NUL
// terminated, and all buffers/handles remain alive for the duration of spawn.
if unsafe {
CreateProcessW(
application.as_ptr(),
command.as_mut_ptr(),
std::ptr::null(),
std::ptr::null(),
1,
CREATE_NO_WINDOW
| CREATE_NEW_PROCESS_GROUP
| CREATE_UNICODE_ENVIRONMENT
| EXTENDED_STARTUPINFO_PRESENT,
environment.as_ptr().cast(),
std::ptr::null(),
&startup.StartupInfo,
&mut process,
)
} == 0
{
return Err(io::Error::last_os_error());
}
// SAFETY: successful CreateProcessW transfers these two handles to us.
let _thread = unsafe { OwnedHandle::from_raw_handle(process.hThread) };
Ok(Helper(unsafe {
OwnedHandle::from_raw_handle(process.hProcess)
}))
}
fn wide(value: &OsStr) -> Vec<u16> {
value.encode_wide().chain(Some(0)).collect()
}
+48 -51
View File
@@ -10,8 +10,7 @@ use base64::Engine;
use bsk_protocol::Method;
use bsk_protocol::tools::{
TransferBeginParams, TransferBeginResult, TransferChunkParams, TransferChunkResult,
TransferIdParams, TransferReadyResult, TransferReleaseResult, UploadFile, UploadMode,
UploadParams, UploadResult,
TransferIdParams, TransferReadyResult, UploadFile, UploadMode, UploadParams, UploadResult,
};
use clap::{Args, ValueEnum};
@@ -69,14 +68,12 @@ pub fn dispatch(args: UploadArgs, format: Format) -> Result<(), CliError> {
let (ref_, selector) = split_target(args.target, args.ref_, args.selector)?;
let mut staged = Vec::new();
for path in &args.files {
match stage_file(&info.sock_path, &args.session, path) {
Ok(file) => staged.push(file),
Err(err) => {
for (id, _) in &staged {
let _ = release(&info.sock_path, id);
}
return Err(err);
}
if let Err(err) = stage_file(&info.sock_path, &args.session, path, &mut staged) {
let _ = crate::cli::business_rpc::release_transfers(
&info.sock_path,
staged.iter().map(|(id, _)| id.as_str()),
);
return Err(err);
}
}
let params = UploadParams {
@@ -102,6 +99,17 @@ pub fn dispatch(args: UploadArgs, format: Format) -> Result<(), CliError> {
Some(params),
ipc_timeout(args.timeout),
);
// The helper can reject cancellation before handing any request to the
// daemon. In that case staging still belongs to this CLI.
if result.as_ref().is_err_and(|err| {
err.data()
.is_some_and(|data| data["reason"] == "cancelled_before_dispatch")
}) {
let _ = crate::cli::business_rpc::release_transfers(
&info.sock_path,
staged.iter().map(|(id, _)| id.as_str()),
);
}
// Once tool.upload is dispatched, staging ownership belongs to the
// session. A transport timeout cannot prove that Chrome did not attach
// the file, so releasing here could invalidate a late successful attach.
@@ -118,7 +126,12 @@ pub fn dispatch(args: UploadArgs, format: Format) -> Result<(), CliError> {
Ok(())
}
fn stage_file(sock: &Path, session: &str, path: &PathBuf) -> Result<(String, String), CliError> {
fn stage_file(
sock: &Path,
session: &str,
path: &PathBuf,
staged: &mut Vec<(String, String)>,
) -> Result<(), CliError> {
let mut file = File::open(path)
.with_context(|| format!("open upload file {}", path.display()))
.map_err(CliError::Local)?;
@@ -149,55 +162,39 @@ fn stage_file(sock: &Path, session: &str, path: &PathBuf) -> Result<(String, Str
}),
Duration::from_secs(10),
)?;
let staged = (|| {
let mut offset = 0u64;
let mut buf = vec![0u8; begin.chunk_size as usize];
loop {
let n = file.read(&mut buf).map_err(|e| CliError::Local(e.into()))?;
if n == 0 {
break;
}
let reply: TransferChunkResult = crate::cli::business_rpc::call(
sock.to_path_buf(),
"transfer-chunk",
Method::TransferChunk,
Some(TransferChunkParams {
transfer_id: begin.transfer_id.clone(),
offset,
data_base64: base64::engine::general_purpose::STANDARD.encode(&buf[..n]),
}),
Duration::from_secs(30),
)?;
offset = reply.next_offset;
// Record ownership immediately so one error path releases both completed
// files and the current partially staged file.
staged.push((begin.transfer_id.clone(), name));
let mut offset = 0u64;
let mut buf = vec![0u8; begin.chunk_size as usize];
loop {
let n = file.read(&mut buf).map_err(|e| CliError::Local(e.into()))?;
if n == 0 {
break;
}
let _: TransferReadyResult = crate::cli::business_rpc::call(
let reply: TransferChunkResult = crate::cli::business_rpc::call(
sock.to_path_buf(),
"transfer-finish",
Method::TransferFinish,
Some(TransferIdParams {
"transfer-chunk",
Method::TransferChunk,
Some(TransferChunkParams {
transfer_id: begin.transfer_id.clone(),
offset,
data_base64: base64::engine::general_purpose::STANDARD.encode(&buf[..n]),
}),
Duration::from_secs(10),
Duration::from_secs(30),
)?;
Ok::<_, CliError>(())
})();
if let Err(err) = staged {
let _ = release(sock, &begin.transfer_id);
return Err(err);
offset = reply.next_offset;
}
Ok((begin.transfer_id, name))
}
fn release(sock: &Path, id: &str) -> Result<TransferReleaseResult, CliError> {
crate::cli::business_rpc::call(
let _: TransferReadyResult = crate::cli::business_rpc::call(
sock.to_path_buf(),
"transfer-release",
Method::TransferRelease,
"transfer-finish",
Method::TransferFinish,
Some(TransferIdParams {
transfer_id: id.to_string(),
transfer_id: begin.transfer_id.clone(),
}),
Duration::from_secs(5),
)
Duration::from_secs(10),
)?;
Ok(())
}
fn ipc_timeout(timeout_ms: u32) -> Duration {
+134
View File
@@ -1399,6 +1399,28 @@ mod windows {
connected = pipe.connect() => {
match connected {
Ok(()) => {
// Keep the connected instance alive until its replacement
// exists. Otherwise a fast handler can close the last
// instance and make new clients fail with NotFound.
loop {
match ServerOptions::new()
.access_inbound(true)
.access_outbound(true)
.create(&listener.pipe_name)
{
Ok(next) => {
listener.first = Some(next);
break;
}
Err(err) => {
warn!(?err, "create next named-pipe instance failed");
tokio::select! {
_ = &mut shutdown => return,
_ = tokio::time::sleep(std::time::Duration::from_millis(50)) => {}
}
}
}
}
on_open();
let handler = handler.clone();
let on_act = on_activity.clone();
@@ -1467,6 +1489,118 @@ mod windows {
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::Mutex;
use std::time::Duration;
use tokio::net::windows::named_pipe::ClientOptions;
use tokio::sync::oneshot;
fn isolated_listener() -> NamedPipeListener {
let pipe_name = format!(r"\\.\pipe\bsk-test-{}", uuid::Uuid::new_v4());
let first = ServerOptions::new()
.first_pipe_instance(true)
.create(&pipe_name)
.unwrap();
NamedPipeListener {
pipe_name,
first: Some(first),
}
}
#[tokio::test]
async fn next_instance_exists_before_connection_is_handed_off() {
let listener = isolated_listener();
let name = listener.pipe_name.clone();
let probe_name = name.clone();
let (probe_tx, probe_rx) = oneshot::channel();
let probe_tx = Mutex::new(Some(probe_tx));
let (stop_tx, stop_rx) = oneshot::channel();
let server = tokio::spawn(serve(
listener,
crate::daemon::ipc::default_ping_handler(),
move || {
if let Some(tx) = probe_tx.lock().unwrap().take() {
// This callback runs before the connection task can finish.
// A second client must already have an instance to open.
let result = ClientOptions::new().open(&probe_name).map(drop);
let _ = tx.send(result);
}
},
|| {},
|| {},
async {
let _ = stop_rx.await;
},
));
let client = ClientOptions::new().open(&name).unwrap();
let result = tokio::time::timeout(Duration::from_secs(5), probe_rx).await;
drop(client);
let _ = stop_tx.send(());
server.await.unwrap();
result
.expect("accept callback ran")
.unwrap()
.expect("next pipe instance is ready");
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn rapid_disconnects_and_concurrent_rpc_connections() {
let listener = isolated_listener();
let name = listener.pipe_name.clone();
let (stop_tx, stop_rx) = oneshot::channel();
let server = tokio::spawn(serve(
listener,
crate::daemon::ipc::default_ping_handler(),
|| {},
|| {},
|| {},
async {
let _ = stop_rx.await;
},
));
let exercise = async {
// Include clients that close without sending any request.
for _ in 0..64 {
drop(
crate::ipc_client::Client::connect_path(name.clone().into())
.await
.unwrap(),
);
}
let mut clients = tokio::task::JoinSet::new();
for _ in 0..8 {
let name = name.clone();
clients.spawn(async move {
for _ in 0..32 {
let mut client =
crate::ipc_client::Client::connect_path(name.clone().into())
.await
.unwrap();
let reply: bsk_protocol::PingResult = client
.call(
bsk_protocol::Method::SystemPing,
&serde_json::json!({}),
Duration::from_secs(5),
)
.await
.unwrap()
.unwrap();
assert!(reply.pong);
}
});
}
while let Some(result) = clients.join_next().await {
result.unwrap();
}
};
let result = tokio::time::timeout(Duration::from_secs(30), exercise).await;
let _ = stop_tx.send(());
server.await.unwrap();
result.expect("connection exercise completed");
}
}
}
#[cfg(windows)]
+44 -4
View File
@@ -90,17 +90,21 @@ pub fn pid_alive(pid: u32) -> bool {
#[cfg(windows)]
{
use windows_sys::Win32::Foundation::CloseHandle;
use windows_sys::Win32::Foundation::{CloseHandle, WAIT_TIMEOUT};
use windows_sys::Win32::System::Threading::{
OpenProcess, PROCESS_QUERY_LIMITED_INFORMATION,
OpenProcess, PROCESS_SYNCHRONIZE, WaitForSingleObject,
};
unsafe {
let handle = OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, 0, pid);
let handle = OpenProcess(PROCESS_SYNCHRONIZE, 0, pid);
if handle.is_null() {
return false;
}
// An exited process can still be opened while another handle
// keeps its kernel object alive. Poll its termination signal;
// unlike checking STILL_ACTIVE, this also handles exit code 259.
let alive = WaitForSingleObject(handle, 0) == WAIT_TIMEOUT;
CloseHandle(handle);
true
alive
}
}
@@ -110,3 +114,39 @@ pub fn pid_alive(pid: u32) -> bool {
false
}
}
#[cfg(all(test, windows))]
mod tests {
use super::pid_alive;
use std::os::windows::process::CommandExt;
use std::process::Command;
#[test]
fn current_process_is_alive() {
assert!(pid_alive(std::process::id()));
}
#[test]
fn invalid_process_is_not_alive() {
assert!(!pid_alive(0));
assert!(!pid_alive(u32::MAX));
}
#[test]
fn exited_process_with_retained_handle_is_not_alive() {
for exit_code in [0, 259] {
let mut child = Command::new("cmd.exe")
.args(["/D", "/C", &format!("exit {exit_code}")])
.creation_flags(0x0800_0000) // CREATE_NO_WINDOW
.spawn()
.expect("spawn short-lived process");
assert_eq!(child.wait().unwrap().code(), Some(exit_code));
// Keep Child (and its process handle) alive during the probe.
assert!(
!pid_alive(child.id()),
"exited process with code {exit_code}"
);
drop(child);
}
}
}
+14 -9
View File
@@ -232,8 +232,8 @@ pub fn run_foreground(cfg: DaemonConfig) -> Result<()> {
let session_idle_task = spawn_session_idle_reaper(Arc::clone(&state));
let browser_liveness_task = spawn_browser_liveness_reaper(Arc::clone(&state));
// Fired by the update check task after a successful auto-update:
// the replacement daemon has already been spawned, so this
// process should shut down and let it take over.
// the replacement daemon (or Windows update helper) is ready, so
// this process should shut down and let it take over.
let restart_notify = Arc::new(tokio::sync::Notify::new());
let update_check_task =
spawn_update_check_task(Arc::clone(&state), Arc::clone(&restart_notify));
@@ -539,8 +539,8 @@ pub(crate) fn spawn_session_idle_reaper(state: Arc<DaemonState>) -> tokio::task:
/// task spawns the replacement daemon (see
/// [`DAEMON_REPLACEMENT_WAIT_ENV`]) and fires `restart` so this process
/// shuts down and the new version takes over; on Windows the
/// replacement can only be staged, so it just logs that a restart is
/// needed.
/// helper waits for this process to exit, replaces the binary, and starts
/// the new daemon with the same configuration.
pub(crate) fn spawn_update_check_task(
state: Arc<DaemonState>,
restart: Arc<tokio::sync::Notify>,
@@ -605,7 +605,11 @@ pub(crate) fn spawn_update_check_task(
|candidate| {
let target =
exe_path.as_deref().context("current executable unknown")?;
update::self_install_candidate(candidate, target)
update::self_install_candidate(
candidate,
target,
&restart_start_args(&state.config),
)
},
)
})
@@ -635,10 +639,11 @@ pub(crate) fn spawn_update_check_task(
sessions,
"auto-update postponed: agent session(s) active; will retry on the next tick"
),
update::AutoUpdateOutcome::Staged { latest } => warn!(
%latest,
"auto-update staged the new binary but cannot replace the running daemon in place; restart the daemon (or terminal) to finish the upgrade"
),
update::AutoUpdateOutcome::Staged { latest } => {
info!(%latest, "update helper ready; exiting so it can replace and restart the daemon");
restart.notify_one();
return;
}
update::AutoUpdateOutcome::Replaced { latest } => {
info!(
current = env!("CARGO_PKG_VERSION"),
@@ -0,0 +1,342 @@
//! Real Windows CLI cancellation, including caller-owned transfer cleanup.
#![cfg(windows)]
use std::path::Path;
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
use bsk::daemon::{file_transfer::TransferRegistry, info::DaemonInfo};
use bsk_protocol::tools::{TransferBeginParams, TransferIdParams};
use bsk_protocol::{ErrorCode, Frame, Method, ResponseBody, ResponseFrame, RpcError};
use serde_json::json;
use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader};
use tokio::net::windows::named_pipe::ServerOptions;
use tokio::process::Command;
use tokio::sync::{Notify, watch};
fn response<T: serde::Serialize>(result: Result<T, RpcError>) -> ResponseBody {
match result {
Ok(value) => ResponseBody::Ok(serde_json::to_value(value).unwrap()),
Err(err) => ResponseBody::Err(err),
}
}
#[test]
fn stdin_close_cancels_business_but_releases_resources_with_a_bounded_budget() {
// Resolve the production registry's root before starting any runtime threads.
let registry_home = tempfile::tempdir().unwrap();
let original_home = std::env::var_os("BSK_HOME");
unsafe {
std::env::set_var("BSK_HOME", registry_home.path());
}
let transfers = Arc::new(TransferRegistry::new().unwrap());
unsafe {
match original_home {
Some(value) => std::env::set_var("BSK_HOME", value),
None => std::env::remove_var("BSK_HOME"),
}
}
transfers.initialize().unwrap();
tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.unwrap()
.block_on(exercise(
transfers,
&registry_home.path().join("run/transfers"),
));
}
async fn exercise(transfers: Arc<TransferRegistry>, staging_root: &Path) {
for mode in [
"wait",
"upload",
"upload-multi",
"upload-before-dispatch",
"download",
"upload-multi-stall",
"ordinary",
] {
let home = tempfile::tempdir().unwrap();
let pipe_name = format!(r"\\.\pipe\bsk-parent-cancel-{}", uuid::Uuid::new_v4());
let info = DaemonInfo::now(std::process::id(), (&pipe_name).into(), 0, "0.2.0");
std::fs::write(
home.path().join("daemon.json"),
serde_json::to_vec(&info).unwrap(),
)
.unwrap();
let mut pipe = ServerOptions::new()
.first_pipe_instance(true)
.create(&pipe_name)
.unwrap();
let registered = Arc::new(Notify::new());
let pending = Arc::new(Mutex::new(None::<String>));
let ids = Arc::new(Mutex::new(Vec::<String>::new()));
let released = Arc::new(Mutex::new(Vec::<String>::new()));
let received = Arc::new(Mutex::new(Vec::<Method>::new()));
let download_id = if mode == "download" {
let staging = transfers.begin_download("fixture").unwrap();
let browser_dir = home.path().join("BrowserSkill").join(&staging.transfer_id);
std::fs::create_dir_all(&browser_dir).unwrap();
let file = browser_dir.join("download.txt");
std::fs::write(&file, b"download fixture").unwrap();
transfers
.import_download(&staging.transfer_id, &file)
.unwrap();
ids.lock().unwrap().push(staging.transfer_id.clone());
Some(staging.transfer_id)
} else {
None
};
let (cancel, cancelled) = watch::channel(false);
let ready = Arc::clone(&registered);
let name = pipe_name.clone();
let registry = Arc::clone(&transfers);
let server_ids = Arc::clone(&ids);
let server_released = Arc::clone(&released);
let server_received = Arc::clone(&received);
let server = tokio::spawn(async move {
let mut connections = tokio::task::JoinSet::new();
loop {
pipe.connect().await.unwrap();
let next = ServerOptions::new().create(&name).unwrap();
let connection = std::mem::replace(&mut pipe, next);
let ready = Arc::clone(&ready);
let pending = Arc::clone(&pending);
let cancel = cancel.clone();
let mut cancelled = cancelled.clone();
let name = name.clone();
let registry = Arc::clone(&registry);
let ids = Arc::clone(&server_ids);
let released = Arc::clone(&server_released);
let received = Arc::clone(&server_received);
let download_id = download_id.clone();
connections.spawn(async move {
let (read, mut write) = tokio::io::split(connection);
let mut reader = BufReader::new(read);
let mut line = String::new();
while reader.read_line(&mut line).await.unwrap_or(0) > 0 {
let Frame::Request(req) = serde_json::from_str::<Frame>(&line).unwrap()
else {
panic!("expected request");
};
line.clear();
received.lock().unwrap().push(req.method.clone());
let params = req.params.clone().unwrap_or_default();
let body = match req.method {
Method::SystemStatus => ResponseBody::Ok(json!({
"daemon_version": "0.2.0", "protocol_version": "1.1",
"pid": std::process::id(), "uptime_secs": 0,
"ws_port": 0, "sock_path": name,
"browsers": [], "sessions": [], "version_skew_browsers": []
})),
Method::TransferBegin => {
let begin = registry
.begin_upload(serde_json::from_value(params).unwrap())
.unwrap();
ids.lock().unwrap().push(begin.transfer_id.clone());
response(Ok(begin))
}
Method::ToolDownload => ResponseBody::Ok(json!({
"tab_id": 1, "suggested_filename": "download.txt",
"byte_size": 16, "transfer_id": download_id,
})),
Method::ToolWaitMs if mode == "ordinary" => {
ResponseBody::Ok(json!({"waited_ms": 1}))
}
Method::TransferChunk
if mode == "upload-before-dispatch"
|| (mode.contains("multi")
&& ids.lock().unwrap().len() == 1) =>
{
response(
registry.write_chunk(serde_json::from_value(params).unwrap()),
)
}
Method::TransferFinish if mode != "upload-before-dispatch" => response(
registry.finish_upload(serde_json::from_value(params).unwrap()),
),
Method::ToolWaitMs
| Method::TransferChunk
| Method::TransferRead
| Method::TransferFinish => {
*pending.lock().unwrap() = Some(req.id.clone());
ready.notify_one();
cancelled.wait_for(|value| *value).await.unwrap();
if mode == "upload-before-dispatch" {
// Finish won the race with cancellation, but tool.upload must
// not be dispatched and the CLI still owns the staging.
response(
registry
.finish_upload(serde_json::from_value(params).unwrap()),
)
} else {
ResponseBody::Err(RpcError {
code: ErrorCode::Cancelled,
message: "parent cancellation reached daemon".into(),
data: None,
})
}
}
Method::Cancel => {
let target = params["rpc_id"].as_str().unwrap();
assert_eq!(pending.lock().unwrap().as_deref(), Some(target));
cancel.send(true).unwrap();
ResponseBody::Ok(json!({"cancelled": true}))
}
Method::TransferRelease => {
let params: TransferIdParams =
serde_json::from_value(params).unwrap();
released.lock().unwrap().push(params.transfer_id.clone());
if mode.contains("stall") {
std::future::pending::<()>().await;
}
response(Ok(registry.release(params)))
}
other => panic!("unexpected request after cancellation: {other:?}"),
};
let response = Frame::Response(ResponseFrame { id: req.id, body });
let mut bytes = serde_json::to_vec(&response).unwrap();
bytes.push(b'\n');
if write.write_all(&bytes).await.is_err() {
break;
}
}
});
}
});
let mut cmd = Command::new(env!("CARGO_BIN_EXE_bsk"));
cmd.env("BSK_HOME", home.path())
.env("BSK_AUTO_UPDATE", "off")
.env_remove("BSK_CANCEL_ON_STDIN_CLOSE")
.stdout(std::process::Stdio::piped())
.stderr(std::process::Stdio::piped())
.creation_flags(0x0800_0000)
.kill_on_drop(true);
if mode.starts_with("upload") {
let source = home.path().join("upload.txt");
// Declare 300 MiB without allocating or transferring that much data.
std::fs::File::create(&source)
.unwrap()
.set_len(if mode == "upload-before-dispatch" {
16
} else {
300 * 1024 * 1024
})
.unwrap();
cmd.args(["upload", "--session", "fixture", "--selector", "input"]);
if mode.contains("multi") {
let first = home.path().join("first.txt");
std::fs::write(&first, b"first file").unwrap();
cmd.arg("--file").arg(first);
}
cmd.arg("--file").arg(source);
} else if mode == "download" {
let out = home.path().join("out.txt");
std::fs::write(&out, b"existing output").unwrap();
cmd.args([
"download",
"--session",
"fixture",
"--selector",
"a",
"--overwrite",
"--out",
])
.arg(out);
} else {
cmd.args(["wait-ms", if mode == "ordinary" { "1ms" } else { "60s" }]);
}
cmd.arg("--json");
if mode == "ordinary" {
cmd.stdin(std::process::Stdio::null());
} else {
cmd.env("BSK_CANCEL_ON_STDIN_CLOSE", "1")
.stdin(std::process::Stdio::piped());
}
let mut child = cmd.spawn().unwrap();
if mode != "ordinary" {
tokio::time::timeout(Duration::from_secs(10), registered.notified())
.await
.expect("CLI registered business RPC");
child
.stdin
.as_mut()
.unwrap()
.write_all(b"still connected")
.await
.unwrap();
drop(child.stdin.take());
}
let start = Instant::now();
let output = tokio::time::timeout(Duration::from_secs(8), child.wait_with_output())
.await
.expect("CLI must settle promptly, including cleanup")
.unwrap();
server.abort();
let _ = server.await;
assert_eq!(
output.status.code(),
Some(if mode == "ordinary" { 0 } else { 2 }),
"{mode}: stdout={} stderr={}",
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr)
);
if mode != "ordinary" {
let body: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap();
assert_eq!(body["code"], "cancelled");
if mode != "upload-before-dispatch" {
assert_eq!(body["message"], "parent cancellation reached daemon");
}
let received = received.lock().unwrap();
let cancel = received.iter().position(|m| *m == Method::Cancel).unwrap();
assert!(
received[cancel + 1..]
.iter()
.all(|m| *m == Method::TransferRelease),
"only cleanup may follow cancellation: {received:?}"
);
}
let ids = ids.lock().unwrap();
if mode.contains("stall") {
assert_eq!(released.lock().unwrap().len(), 1);
assert!(start.elapsed() >= Duration::from_secs(4));
// The CLI cannot force an unresponsive daemon to clean up.
transfers.release_session("fixture");
} else {
assert_eq!(
*released.lock().unwrap(),
*ids,
"{mode}: every allocation must be released"
);
for id in ids.iter() {
assert!(!staging_root.join(id).exists(), "staging leaked: {id}");
}
if mode.starts_with("upload") {
let retry = transfers
.begin_upload(TransferBeginParams {
session_id: "fixture".into(),
name: "retry.txt".into(),
byte_size: 300 * 1024 * 1024,
})
.expect("same session must have room for another 300 MiB upload");
transfers.release(TransferIdParams {
transfer_id: retry.transfer_id,
});
}
}
if mode == "download" {
assert_eq!(
std::fs::read(home.path().join("out.txt")).unwrap(),
b"existing output"
);
assert!(
!std::fs::read_dir(home.path())
.unwrap()
.flatten()
.any(|entry| entry.file_name().to_string_lossy().ends_with(".part"))
);
}
}
}
+343
View File
@@ -0,0 +1,343 @@
//! Exercise self-update with a real executable and a local release server.
#![cfg(windows)]
use std::fs;
use std::io::{Cursor, Read, Write};
use std::net::{TcpListener, TcpStream};
use std::os::windows::process::CommandExt;
use std::path::PathBuf;
use std::process::{Child, Command, Stdio};
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::thread;
use std::time::{Duration, Instant};
use bsk::daemon::info::DaemonInfo;
use sha2::{Digest, Sha256};
const MARKER: &[u8] = b"windows-update-regression-fixture";
struct ReleaseServer {
url: String,
requests: Arc<AtomicUsize>,
stop: Arc<AtomicBool>,
worker: Option<thread::JoinHandle<()>>,
}
impl ReleaseServer {
fn new(binary: &[u8]) -> Self {
let mut archive = zip::ZipWriter::new(Cursor::new(Vec::new()));
archive
.start_file(
"bsk.exe",
zip::write::SimpleFileOptions::default()
.compression_method(zip::CompressionMethod::Stored),
)
.unwrap();
archive.write_all(binary).unwrap();
let archive = archive.finish().unwrap().into_inner();
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
listener.set_nonblocking(true).unwrap();
let url = format!("http://{}", listener.local_addr().unwrap());
let manifest = serde_json::to_vec(&serde_json::json!({
"version": "999.0.0",
"assets": {"windows-x64": {
"url": format!("{url}/bsk.zip"),
"sha256": Sha256::digest(&archive).iter().map(|byte| format!("{byte:02x}")).collect::<String>(),
}},
}))
.unwrap();
let stop = Arc::new(AtomicBool::new(false));
let worker_stop = Arc::clone(&stop);
let requests = Arc::new(AtomicUsize::new(0));
let worker_requests = Arc::clone(&requests);
let worker = thread::spawn(move || {
while !worker_stop.load(Ordering::SeqCst) {
let (mut stream, _) = match listener.accept() {
Ok(connection) => connection,
Err(err) if err.kind() == std::io::ErrorKind::WouldBlock => {
thread::sleep(Duration::from_millis(10));
continue;
}
Err(err) => panic!("accept release request: {err}"),
};
// Windows accept() inherits the listener's nonblocking mode.
// Keep accept polling for shutdown, but read/write each request
// in blocking mode with the bounded timeouts below.
stream.set_nonblocking(false).unwrap();
stream
.set_read_timeout(Some(Duration::from_secs(5)))
.unwrap();
stream
.set_write_timeout(Some(Duration::from_secs(5)))
.unwrap();
let mut request = Vec::new();
let mut chunk = [0; 1024];
while !request.windows(4).any(|window| window == b"\r\n\r\n") {
let len = stream.read(&mut chunk).unwrap();
if len == 0 {
break;
}
request.extend_from_slice(&chunk[..len]);
}
let body = if request.starts_with(b"GET /bsk.zip ") {
worker_requests.fetch_add(1, Ordering::SeqCst);
&archive
} else {
&manifest
};
write!(
stream,
"HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
body.len()
)
.unwrap();
stream.write_all(body).unwrap();
}
});
Self {
url: format!("{url}/version.json"),
requests,
stop,
worker: Some(worker),
}
}
}
impl Drop for ReleaseServer {
fn drop(&mut self) {
self.stop.store(true, Ordering::SeqCst);
let _ = self.worker.take().unwrap().join();
}
}
#[test]
fn release_server_waits_for_delayed_and_fragmented_request_headers() {
let server = ReleaseServer::new(b"test binary");
let address = server
.url
.strip_prefix("http://")
.unwrap()
.strip_suffix("/version.json")
.unwrap();
let mut stream = TcpStream::connect(address).unwrap();
stream
.set_read_timeout(Some(Duration::from_secs(5)))
.unwrap();
stream
.set_write_timeout(Some(Duration::from_secs(5)))
.unwrap();
// Give accept() time to run before any bytes arrive, then split the
// headers across writes. Both reads must wait rather than return WouldBlock.
thread::sleep(Duration::from_millis(100));
stream.write_all(b"GET /version.json HTTP/1.1\r\n").unwrap();
thread::sleep(Duration::from_millis(100));
stream.write_all(b"Host: localhost\r\n\r\n").unwrap();
let mut response = String::new();
stream.read_to_string(&mut response).unwrap();
assert!(response.starts_with("HTTP/1.1 200 OK\r\n"), "{response}");
let (_, body) = response.split_once("\r\n\r\n").unwrap();
let manifest: serde_json::Value = serde_json::from_str(body).unwrap();
assert_eq!(manifest["version"], "999.0.0");
}
struct Fixture {
_tmp: tempfile::TempDir,
exe: PathBuf,
home: PathBuf,
binary: Vec<u8>,
server: ReleaseServer,
daemon: Option<Child>,
}
impl Fixture {
fn new() -> Self {
let tmp = tempfile::TempDir::new().unwrap();
let dir = tmp.path().join("中文 space %PATH% ! & (update)");
fs::create_dir(&dir).unwrap();
let exe = dir.join("bsk.exe");
let home = tmp.path().join("home");
fs::create_dir(&home).unwrap();
fs::copy(env!("CARGO_BIN_EXE_bsk"), &exe).unwrap();
// A PE overlay distinguishes the replacement without requiring a
// second build or changing the executable's behavior/version.
let mut binary = fs::read(&exe).unwrap();
binary.extend_from_slice(MARKER);
let server = ReleaseServer::new(&binary);
Self {
_tmp: tmp,
exe,
home,
binary,
server,
daemon: None,
}
}
fn command(&self) -> Command {
let mut cmd = Command::new(&self.exe);
cmd.env("BSK_HOME", &self.home)
.env("BSK_UPDATE_MANIFEST_URL", &self.server.url)
.env("BSK_AUTO_UPDATE", "off")
.env("RUST_LOG", "info")
.env("NO_PROXY", "127.0.0.1,localhost")
.env_remove("BSK_DAEMONIZED")
.env_remove("BSK_DAEMON_REPLACES_PID")
.stdin(Stdio::null())
.creation_flags(0x0800_0000);
cmd
}
fn wait_for(&self, description: &str, mut check: impl FnMut() -> bool) {
let deadline = Instant::now() + Duration::from_secs(30);
while Instant::now() < deadline {
if check() {
return;
}
thread::sleep(Duration::from_millis(50));
}
let diagnostics: Vec<_> = fs::read_dir(self.exe.parent().unwrap())
.unwrap()
.flatten()
.filter(|entry| entry.path().extension().is_some_and(|ext| ext == "log"))
.map(|entry| {
(
entry.path(),
String::from_utf8_lossy(&fs::read(entry.path()).unwrap_or_default())
.into_owned(),
)
})
.collect();
let home_logs: Vec<_> = fs::read_dir(&self.home)
.unwrap()
.flatten()
.filter(|entry| entry.file_name().to_string_lossy().contains("log"))
.map(|entry| {
(
entry.path(),
String::from_utf8_lossy(&fs::read(entry.path()).unwrap_or_default())
.into_owned(),
)
})
.collect();
panic!(
"timed out waiting for {description}; helper logs: {diagnostics:?}; daemon logs: {home_logs:?}"
);
}
fn info(&self) -> Option<DaemonInfo> {
serde_json::from_slice(&fs::read(self.home.join("daemon.json")).ok()?).ok()
}
fn updated(&self) -> bool {
// A locked executable may briefly reject reads during replacement.
fs::read(&self.exe).is_ok_and(|binary| binary == self.binary)
}
}
impl Drop for Fixture {
fn drop(&mut self) {
// Stop only the daemon belonging to this isolated BSK_HOME. Also reap
// the foreground process on assertion failures to avoid leaking locks.
let _ = self.command().args(["daemon", "stop"]).output();
if let Some(child) = self.daemon.as_mut() {
let _ = child.kill();
let _ = child.wait();
}
}
}
#[test]
fn manual_update_replaces_the_running_executable_after_cli_exit() {
let fixture = Fixture::new();
let out = fixture
.command()
.args(["--json", "update", "--yes"])
.output()
.unwrap();
assert!(
out.status.success(),
"{}",
String::from_utf8_lossy(&out.stderr)
);
let report: serde_json::Value = serde_json::from_slice(&out.stdout).unwrap();
assert_eq!(report["status"], "staged");
fixture.wait_for("CLI replacement and helper cleanup", || {
fixture.updated() && fs::read_dir(fixture.exe.parent().unwrap()).unwrap().count() == 1
});
assert!(
fixture.info().is_none(),
"an update must not start a previously absent daemon"
);
assert_eq!(fixture.server.requests.load(Ordering::SeqCst), 1);
}
#[test]
fn automatic_update_exits_old_daemon_and_restarts_on_the_same_port() {
let mut fixture = Fixture::new();
let port = unused_port();
let log = fs::File::create(fixture.home.join("foreground.log")).unwrap();
let child = fixture
.command()
.env("BSK_AUTO_UPDATE", "on")
// Exercise the environment inherited by a normally detached daemon.
.env("BSK_DAEMONIZED", "1")
.args([
"daemon",
"start",
"--port",
&port.to_string(),
"--session-idle",
"1234ms",
"--daemon-idle",
"30s",
])
.stdout(Stdio::from(log.try_clone().unwrap()))
.stderr(Stdio::from(log))
.spawn()
.unwrap();
let old_pid = child.id();
fixture.daemon = Some(child);
fixture.wait_for("replacement daemon", || {
fixture.updated()
&& fixture
.info()
.is_some_and(|info| info.pid != old_pid && info.ws_port == port)
});
assert!(
fixture
.daemon
.as_mut()
.unwrap()
.try_wait()
.unwrap()
.is_some(),
"old daemon must exit"
);
fixture.wait_for("helper cleanup", || {
fs::read_dir(fixture.exe.parent().unwrap()).unwrap().count() == 1
});
let status = fixture
.command()
.args(["--json", "status"])
.output()
.unwrap();
assert!(
status.status.success(),
"{}",
String::from_utf8_lossy(&status.stderr)
);
assert_eq!(
fixture.server.requests.load(Ordering::SeqCst),
1,
"the replacement must not stage another update immediately"
);
}
fn unused_port() -> u16 {
TcpListener::bind("127.0.0.1:0")
.unwrap()
.local_addr()
.unwrap()
.port()
}
@@ -0,0 +1,44 @@
# DOMSnapshot layout-unit regression (#195)
Source: https://github.com/Tencent/BrowserSkill/pull/195
DOMSnapshot bounds and document scroll offsets retain Blink layout units. Treating them as CSS
pixels changes both positions and sizes under device scaling/browser zoom, before iframe
projection or viewport clipping even starts.
The fixture contains a scrolled root, a scrolled same-process iframe, an OOPIF served from the
other loopback hostname, and a nested iframe in each. Owners have borders, padding and CSS
transforms. The geometry test compares the production capture's local and top-level rectangles
against independent DOM border boxes, with at most 2 CSS pixels of layout rounding. It also
checks that each target reads layout metrics only once. No credentials or personal tabs are used.
Run the geometric assertions with Node 22+ and a local Chrome executable:
```sh
BSK_GEOMETRY_CHROME=/path/to/chrome pnpm --filter @browser-skill/extension exec vitest run \
src/tools/__tests__/snapshot-coordinates.browser.test.ts
```
The test launches and cleans up its own headless Chrome profiles. It explicitly verifies the
requested browser zoom and OOPIF topology, so unsupported setups fail rather than silently
testing a different configuration. It is skipped in ordinary unit runs unless the executable
is specified. No browser download or new package dependency is required.
The corpus smoke case checks fixture readiness and observable semantics through the real CLI:
```sh
BSK_AUTO_UPDATE=off pnpm eval:browser smoke --case snapshot-coordinates --bsk ./target/debug/bsk
```
Smoke alone does **not** certify coordinate accuracy; CLI observations do not expose raw boxes.
Use the geometric test above for the regression's numeric assertions.
## Separate existing boundary
The OOPIF root uses non-occupying scrollbars in the unit-conversion regression. Root and
same-process frames retain normal scrollbars. Append `&classic-scrollbars` to the fixture URL
to reproduce the separate existing OOPIF projection issue: `targetProjection()` maps the CSS
layout viewport (which excludes occupying scrollbars) onto the entire owner content quad.
This inflates the projected coordinates when scrollbars occupy space. Main already has that
mapping; fixing it also changes shared live geometry and is outside this snapshot-unit fix.
The test does not relax its numeric tolerance to absorb that error.
@@ -0,0 +1,99 @@
import { spawn } from "node:child_process";
import { once } from "node:events";
import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
// This regression owns its browser/profile. It never attaches to a user's Chrome.
export async function withChrome({ executable, deviceScale, zoom }, run) {
const profile = await mkdtemp(join(tmpdir(), "bsk-snapshot-coordinates-"));
let chrome;
let socket;
const pending = new Map();
try {
await mkdir(join(profile, "Default"));
await writeFile(
join(profile, "Default", "Preferences"),
JSON.stringify({
partition: { default_zoom_level: { x: Math.log(zoom) / Math.log(1.2) } },
}),
);
chrome = spawn(
executable,
[
"--headless=new",
"--no-first-run",
"--no-default-browser-check",
"--disable-extensions",
"--site-per-process",
"--remote-debugging-port=0",
"--window-size=1600,1200",
`--force-device-scale-factor=${deviceScale}`,
`--user-data-dir=${profile}`,
"about:blank",
],
{ stdio: ["ignore", "ignore", "pipe"] },
);
const endpoint = await new Promise((resolve, reject) => {
let output = "";
const timeout = setTimeout(
() => reject(new Error(`Chrome startup timed out: ${output}`)),
15_000,
);
const fail = (error) => {
clearTimeout(timeout);
reject(error);
};
chrome.once("error", fail);
chrome.once("exit", (code) => fail(new Error(`Chrome exited (${code}): ${output}`)));
chrome.stderr.on("data", (chunk) => {
output += chunk;
const match = output.match(/DevTools listening on (ws:\/\/\S+)/);
if (match) {
clearTimeout(timeout);
resolve(match[1]);
}
});
});
socket = new WebSocket(endpoint);
await new Promise((resolve, reject) => {
socket.addEventListener("open", resolve, { once: true });
socket.addEventListener("error", reject, { once: true });
});
socket.addEventListener("message", ({ data }) => {
const reply = JSON.parse(data);
const request = pending.get(reply.id);
if (!request) return;
clearTimeout(request.timeout);
pending.delete(reply.id);
if (reply.error) request.reject(new Error(JSON.stringify(reply.error)));
else request.resolve(reply.result);
});
let sequence = 0;
const send = (method, params = {}, sessionId) =>
new Promise((resolve, reject) => {
const id = ++sequence;
const timeout = setTimeout(() => {
pending.delete(id);
reject(new Error(`CDP timed out: ${method}`));
}, 10_000);
pending.set(id, { resolve, reject, timeout });
socket.send(JSON.stringify({ id, method, params, sessionId }));
});
await run(send);
} finally {
for (const request of pending.values()) {
clearTimeout(request.timeout);
request.reject(new Error("regression browser closed"));
}
socket?.close();
if (chrome && chrome.exitCode === null && chrome.signalCode === null) {
const exited = once(chrome, "exit");
const timeout = setTimeout(() => chrome.kill("SIGKILL"), 5_000);
chrome.kill();
await exited;
clearTimeout(timeout);
}
await rm(profile, { recursive: true, force: true });
}
}
@@ -0,0 +1,33 @@
{
"$schema": "../../../schemas/case.schema.json",
"schemaVersion": 1,
"id": "snapshot-coordinates",
"title": "Scrolled and nested frame coordinate fixture",
"suite": "regression",
"tags": ["geometry", "iframe", "scroll"],
"fixture": { "startPath": "/snapshot-coordinates" },
"prompts": {
"en": "Open {url}, observe the page and its frames, report the GEOMETRY marker, then close the session.",
"zh-CN": "打开 {url},观察页面及其 iframe,报告 GEOMETRY 标记,然后关闭会话。"
},
"coverage": ["session.start", "session.stop", "page.navigate", "inspect.observe"],
"assertions": {
"site": [
{
"label": "root fixture scrolled and settled",
"type": "geometry.ready",
"where": { "data.root": true },
"minCount": 1
}
],
"response": [{ "label": "observed the fixture marker", "includes": "GEOMETRY-195" }],
"adapter": [{ "label": "browser session was closed", "key": "sessionStopped" }]
},
"smoke": {
"steps": [
{ "action": "navigate" },
{ "action": "wait-site-event", "type": "geometry.ready", "where": { "root": true } },
{ "action": "observe" }
]
}
}
@@ -0,0 +1,52 @@
import { page, withRun } from "../../../lib/fixtures.mjs";
export default {
id: "snapshot-coordinates",
routes: ["/snapshot-coordinates", "/snapshot-coordinates/frame", "/snapshot-coordinates/nested"],
render({ pathname, runId, query }) {
const root = pathname === "/snapshot-coordinates";
const nested = pathname.endsWith("/nested");
return page({
title: "Snapshot coordinate regression",
body: `
<style>
main { margin: 0; width: auto; }
body { width: 2000px; height: 2400px; }
html { overflow-anchor: none; }
${query.get("scrollbars") === "none" ? "html { scrollbar-width: none; }" : ""}
button { position: absolute; box-sizing: border-box; padding: 0; border: 2px solid black; border-radius: 0; }
#probe { left: ${root ? 180 : 100}px; top: ${root ? 360 : 160}px; width: 120px; height: 40px; }
iframe { position: absolute; box-sizing: content-box; width: 320px; height: 220px;
border: 6px solid black; padding: 8px; transform: scale(1.25); transform-origin: 0 0; }
#same { left: 340px; top: 400px; }
#cross { left: 820px; top: 400px; }
#nested { left: 150px; top: 210px; width: 130px; height: 70px; border-width: 2px; padding: 4px; transform: scale(0.8); }
${nested ? "#probe { left: 30px; top: 40px; width: 60px; height: 20px; }" : ""}
</style>
<button id="probe" data-geometry-probe>GEOMETRY-195</button>
${root ? `<iframe id="same" name="same" title="same-process frame" src="${withRun("/snapshot-coordinates/frame", runId)}"></iframe><iframe id="cross" name="cross" title="cross-site frame"></iframe>` : ""}
${!root && !nested ? `<iframe id="nested" name="nested" title="nested frame" src="${withRun("/snapshot-coordinates/nested", runId)}"></iframe>` : ""}
`,
script: `
history.scrollRestoration = "manual";
const cross = document.querySelector("#cross");
if (cross) {
const url = new URL(${JSON.stringify(withRun("/snapshot-coordinates/frame", runId))}, location.href);
url.hostname = location.hostname === "127.0.0.1" ? "localhost" : "127.0.0.1";
// Isolate unit conversion from the pre-existing OOPIF scrollbar-width
// projection issue. ?classic-scrollbars retains that separate reproduction.
if (!new URLSearchParams(location.search).has("classic-scrollbars")) url.searchParams.set("scrollbars", "none");
cross.src = url.href;
}
window.addEventListener("load", () => {
scrollTo(${root ? "80, 240" : nested ? "10, 20" : "40, 100"});
requestAnimationFrame(() => requestAnimationFrame(() => {
window.geometryReady = true;
document.documentElement.dataset.geometryReady = "true";
browserEval.send("geometry.ready", { root: ${root} });
}));
});
`,
});
},
};
+9 -4
View File
@@ -41,17 +41,22 @@ test("case manifests are discovered, ordered, and grouped into suites", () => {
"diagnostics",
"mobile-emulation",
"generated-form",
"snapshot-coordinates",
],
);
assert.deepEqual(repositorySummary(cases, fixtureRegistry).suites, { core: 6, matrix: 1 });
assert.deepEqual(repositorySummary(cases, fixtureRegistry).suites, {
core: 6,
matrix: 1,
regression: 1,
});
});
test("repository validation links every case to a fixture and valid workflow evidence", () => {
assert.deepEqual(validateRepositoryCases(cases, fixtureRegistry), []);
const summary = repositorySummary(cases, fixtureRegistry);
assert.equal(summary.cases, 7);
assert.equal(summary.fixtureModules, 8);
assert.equal(summary.fixtureRoutes, 12);
assert.equal(summary.cases, 8);
assert.equal(summary.fixtureModules, 9);
assert.equal(summary.fixtureRoutes, 15);
});
test("manifest validation rejects unknown operations and incomplete workflow steps", () => {
+41 -15
View File
@@ -21,6 +21,7 @@ $ErrorActionPreference = "Stop"
$Repo = if ($env:BSK_REPO) { $env:BSK_REPO } else { "Tencent/BrowserSkill" }
$InstallDir = if ($env:BSK_INSTALL_DIR) { $env:BSK_INSTALL_DIR } else { Join-Path $HOME ".local\bin" }
$InstallDir = $ExecutionContext.SessionState.Path.GetUnresolvedProviderPathFromPSPath($InstallDir)
$GitHub = "https://github.com/${Repo}"
function Write-Log {
@@ -64,7 +65,7 @@ function Get-PlatformTriple {
function Add-ToUserPath {
param([string]$Dir)
$currentUserPath = [Environment]::GetEnvironmentVariable("PATH", "User") -split ";" | Where-Object { $_ }
$currentUserPath = @([Environment]::GetEnvironmentVariable("PATH", "User") -split ";" | Where-Object { $_ })
if ($currentUserPath -contains $Dir) {
Write-Log "$Dir is already in your user PATH"
@@ -90,30 +91,59 @@ function Add-ToSessionPath {
# ── Git Bash (bash environment) PATH helper ──────────────────────────────────
function Add-ToBashProfile {
param([string]$Dir)
param([string]$Dir, [string]$BashRc = (Join-Path $HOME ".bashrc"))
# Convert Windows path (e.g. C:\Users\foo\.local\bin) to Git-Bash Unix-style (/c/Users/foo/.local/bin)
$unixPath = $Dir -replace '\\', '/'
if ($unixPath -match '^([A-Z]):(.*)$') {
$unixPath = '/' + $matches[1].ToLower() + $matches[2]
}
$bashRc = Join-Path $HOME ".bashrc"
$exportLine = "export PATH=""${unixPath}:`$PATH"" # bsk CLI"
# Single-quote the literal directory; only the existing PATH is expanded.
$shellQuote = "'" + [char]34 + "'" + [char]34 + "'"
$quotedPath = "'" + $unixPath.Replace("'", $shellQuote) + "'"
$exportLine = "export PATH=${quotedPath}:`"`$PATH`" # bsk CLI"
if (Test-Path $bashRc) {
$content = Get-Content $bashRc -Raw -ErrorAction SilentlyContinue
if ($content -match [regex]::Escape($unixPath)) {
if (Test-Path -LiteralPath $BashRc) {
$content = [System.IO.File]::ReadAllText($BashRc)
if ($content.Contains($exportLine)) {
Write-Log "$unixPath is already in ~/.bashrc"
return
}
}
Add-Content $bashRc "`n$exportLine" -Encoding ASCII
# Explicit BOM-less UTF-8 also works in Windows PowerShell 5.1.
[System.IO.File]::AppendAllText($BashRc, "`n$exportLine`n", (New-Object System.Text.UTF8Encoding($false)))
Write-Log "added ${unixPath} to ~/.bashrc"
}
# ── Main ──────────────────────────────────────────────────────────────────────
# Stage on the destination volume before stopping the daemon. Never truncate
# the installed executable: failed replacement must leave it usable.
function Install-Binary {
param([string]$Source, [string]$Target)
$staged = "$Target.install-$([Guid]::NewGuid().ToString('N'))"
try {
[System.IO.File]::Copy($Source, $staged)
if ([System.IO.File]::Exists($Target)) {
# Use the downloaded CLI: older versions have broken Windows
# liveness checks. Stop verifies daemon identity before terminating it.
& $Source daemon stop
if ($LASTEXITCODE -ne 0) { throw "could not stop bsk daemon; existing installation was not replaced" }
# PowerShell 5.1 converts $null to an empty path for string parameters.
[System.IO.File]::Replace($staged, $Target, [NullString]::Value)
Write-Log "daemon will restart automatically on the next browser command"
}
else {
[System.IO.File]::Move($staged, $Target)
}
}
finally {
if ([System.IO.File]::Exists($staged)) { [System.IO.File]::Delete($staged) }
}
}
function Main {
$platform = Get-PlatformTriple
@@ -184,7 +214,7 @@ function Main {
New-Item -ItemType Directory -Path $InstallDir -Force | Out-Null
}
Copy-Item -Path (Join-Path $tempDir "bsk.exe") -Destination (Join-Path $InstallDir "bsk.exe") -Force
Install-Binary -Source (Join-Path $tempDir "bsk.exe") -Target (Join-Path $InstallDir "bsk.exe")
Write-Log "installed bsk to $InstallDir\bsk.exe"
@@ -199,12 +229,8 @@ function Main {
# Verify
$bskPath = Join-Path $InstallDir "bsk.exe"
if (Get-Command bsk -ErrorAction SilentlyContinue) {
& bsk --version
}
else {
Write-Log "verify install: & ""$bskPath"" --version"
}
& $bskPath --version
if ($LASTEXITCODE -ne 0) { throw "installed bsk failed verification" }
Write-Log "done"
Write-Host ""
+138 -133
View File
@@ -1,13 +1,51 @@
# dsh-plugin-browserskill
# BrowserSkill for DeepSeek Harness
npm: [`@wxg-prc-cpg/browser-skill-dsh-plugin`](https://www.npmjs.com/package/@wxg-prc-cpg/browser-skill-dsh-plugin)
[![npm version](https://img.shields.io/npm/v/@wxg-prc-cpg/browser-skill-dsh-plugin)](https://www.npmjs.com/package/@wxg-prc-cpg/browser-skill-dsh-plugin)
A [DeepSeek Harness](https://github.com/deepseek-ai/deepseek-harness) (dsh) tool plugin that exposes
[BrowserSkill](https://github.com/Tencent/BrowserSkill) (`bsk`) browser automation to the model.
Use [BrowserSkill](https://github.com/Tencent/BrowserSkill) in
[DeepSeek Harness](https://github.com/deepseek-ai/deepseek-harness) (`dsh`) to browse
websites, fill forms, and capture screenshots through native `browser_*` tools.
Browser tasks run in Agent Windows, with a live view in the dsh Web UI.
Each action maps to one `bsk <cmd> --json` invocation: the plugin spawns the bsk CLI, parses its
structured JSON output, and returns a canonical typed value. The bsk daemon, browser, and browser
extension keep owning the actual browser control — this package is a thin, well-typed bridge.
## Installation
Before installing the plugin:
- Install [DeepSeek Harness](https://github.com/deepseek-ai/deepseek-harness) and
[pnpm](https://pnpm.io/installation), which dsh uses to manage plugins.
- Install the `bsk` CLI and connect the BrowserSkill extension in Chrome or Edge.
Follow the [BrowserSkill setup guide](https://github.com/Tencent/BrowserSkill#quick-start).
- Make sure `bsk` is on the `PATH` used to start dsh, or set `bskPath` in the plugin configuration.
Install the plugin into the `web` profile, then start it:
```sh
dsh plugin --profile web add @wxg-prc-cpg/browser-skill-dsh-plugin
dsh --profile web
```
Replace `web` with your profile name if you use a different profile. The plugin
includes the `browser-skill` skill; no separate `bsk install-skill` step is needed.
In a conversation, try:
```text
/browser-skill open example.com and summarize the page.
```
By default, the browser tools become available when the skill is invoked.
## Updating
Installed plugins do not update automatically. To upgrade this plugin to npm's
`latest` version, including versions outside the profile's saved dependency range:
```sh
dsh plugin --profile web update @wxg-prc-cpg/browser-skill-dsh-plugin --latest
```
Restart that dsh profile after upgrading. This command updates the plugin; update
the `bsk` CLI and browser extension separately when a release requires it.
## Tools
@@ -20,23 +58,7 @@ extension keep owning the actual browser control — this package is a thin, wel
| `browser_tabs` | `list`, `create`, `select`, `close`, `borrow`, `return` | Manage Agent Window tabs and temporarily borrow user tabs. |
| `browser_assist` | `resize`, `emulate`, `request-help` | Resize or emulate the browser and pause for human-only steps. |
`bsk evaluate` and `bsk record` are intentionally not exposed by this plugin: arbitrary page
evaluation is a higher-risk capability, while recording is long-running and needs a dedicated
DeepSeek Harness lifecycle before it can be added safely.
## Agent skill (progressive disclosure)
Beyond the tools, the plugin publishes the **`browser-skill` agent skill** through the harness's
official skill seam (`ctx.skills.register`): the catalog entry (name + routing description) is
resident in `<available_skills>`, and the body is loaded only when the model invokes the `skill`
tool. Its single source is the DSH-specific `skill/SKILL.md`, which documents only structured
`browser_*` calls and their plugin semantics. The repository-root CLI skill is intentionally not
concatenated: its command examples belong to a different execution interface and would bypass
the plugin's ownership, live observation UI, cancellation, and cleanup path if followed directly.
The build rejects internal CLI-name leakage, command-line code blocks, unknown browser tools, and
missing supported browser tools before embedding the Markdown. Registration and every pre-step
catalog snapshot are pure in-memory reads (no disk/process/daemon); compositions without the skill
seam degrade silently.
Arbitrary page-script evaluation and interaction recording are not supported.
## Multi-session model
@@ -55,115 +77,73 @@ an explicit `session` argument naming a foreign or unknown id is rejected, the `
`browser_session` shows plugin-created sessions only (no daemon-wide view), and stop/unload cleanup
can never touch a session owned by another program.
## Installation
The plugin follows the standard dsh bundle layout (`dsh.bundle` manifest + `cordis.patch.yml`):
```sh
dsh plugin --profile <name> add @wxg-prc-cpg/browser-skill-dsh-plugin
dsh --profile <name>
```
Prerequisite: the `bsk` CLI must be installed and on `PATH`, and the BrowserSkill browser extension
(Chrome or Edge) must be connected — see the
[BrowserSkill README](https://github.com/Tencent/BrowserSkill). When bsk is missing, tool calls fail
with install guidance instead of a bare spawn error.
## Configuration
All fields are optional and validated through the plugin's Schemastery `Config`:
After installing the plugin, edit your profile's `cordis.patch.yml`. For the `web`
profile, the default location is `~/.dsh/profiles/web/cordis.patch.yml`. If you set
`DSH_HOME`, use `$DSH_HOME/profiles/web/cordis.patch.yml` instead. Replace `web` with
your profile name as needed.
If the file contains only comments and `[]`, keep the comments and replace `[]`
with the YAML below. If it already contains patch entries, add this entry to the
existing list or edit its existing `id: browserskill` entry. Keep a single
top-level YAML list. This overrides the plugin registered by the installed bundle:
```yaml
# cordis.patch.yml override example
- insert:
- id: browserskill
name: "@wxg-prc-cpg/browser-skill-dsh-plugin"
config:
bskPath: bsk # path to the bsk binary (default: resolve from PATH)
defaultTimeoutMs: 120000
maxSessions: 5
# observationEnabled: true # live PiP/overlay observation (below)
# thumbnailIntervalMs: 1500 # frame cadence while a session is active
# idleIntervalMs: 8000 # idle cadence / recent-activity window
# lazyTools: true # reveal browser_* tools only after the skill is invoked
- id: browserskill
config:
bskPath: bsk
defaultTimeoutMs: 120000
maxSessions: 5
observationEnabled: true
thumbnailIntervalMs: 1500
idleIntervalMs: 8000
lazyTools: true
```
- **`lazyTools` (default `true`)** — the final progressive-disclosure stage: the six
`browser_*` tool schemas stay OUT of the system prompt (zero schema tokens) until the
`browser-skill` skill is actually invoked — the skill catalog entry is the only
advertisement. One successful invocation (model tool call, or a `/browser-skill` user
gesture) registers the whole suite for the rest of the process; repeated invocations are
no-ops, and sessions resumed with a past invocation in their durable log reveal the suite
on entry. Set `false` for the legacy always-on registration.
Change `bskPath` to the full path of your CLI binary if it is not on dsh's `PATH`.
A patch replaces the entry's entire `config` object, so keep all overrides you need
together in that object.
## Observation overlay (PiP mini-window)
Configuration changes follow `dsh.profile.patchReload` in the profile's
`package.json`: `live` (the default for `web`) applies changes when you save the
patch file; `startup` requires restarting the profile. Restart after upgrading
the plugin in either case.
When the plugin runs inside the dsh Web UI, an **observation overlay** floats over the app
(registered into the `shell.overlay` seat): a breathing thumbnail per owned session plus its
current action and elapsed time. The card docks at the top-right of the content area (clear of
the composer and the shell's header controls) and wears the **BrowserSkill product family's own
look**: the overlay reuses `@browser-skill/ui` components (`Button`, `cn`) and its oklch design
tokens (`--card`, `--primary`, `--destructive`, `--ring`, …), status dots spec'd after the
extension popup's connection indicator, and Remix icons. The BSK utility sheet is compiled
scoped under the `.bsk-obs` root class (`scripts/build-client-css.mjs`), so the overlay looks
like the BrowserSkill extension without leaking a single selector into the host shell — and the
shell's theme cannot bleed back in.
All fields are optional; omitted fields use the defaults below:
- **Lifecycle**: hidden while the plugin owns no sessions; appears on the first
`browser_session` start action; disappears when all sessions stop (or the plugin unloads).
- **Focus view**: status row (green/idle/red dot + session + action + mm:ss), the latest page
frame (refreshes every ~1.5s while active, ~8s when idle; the last good frame stays on
stage while the next one loads, and is kept on errors so the card does not flash),
and a compact icon toolbar (Interrupt + Pop out, hover for the label).
- **Interrupt**: one click kills the in-flight bsk command of the focus session (same semantics
as the chat Stop button — the current action fails, the agent run may continue). Strip items
carry their own hover interrupt button.
- **Multi-session strip**: every session gets a tile (mini frame + id + status dot); focus
auto-follows the most recently active session; clicking a tile pins focus (pin badge, click
again to release); errored sessions get a red edge without stealing focus; sessions the daemon
lost are greyed out; prolonged daemon/browser outage shows "browser unavailable" and greys
the interrupt button until captures recover.
- **Drag & resize**: drag the header to move the card; drag any of the four
corners to resize (min 240×180, max 80% of the viewport; no visible grip).
Both are remembered for the page lifetime.
- **Pop out (PiP)**: upgrades the card into a native Document PiP window (requires a user
gesture, per browser rules), sized from the current card; closing the PiP falls back to the
in-page card with state intact. Browsers without Document PiP simply hide the button.
- **Wire**: the host serves `GET /bsk-observation/state`, `GET /bsk-observation/events` (SSE),
`POST /bsk-observation/interrupt`, and `GET /bsk-observation/thumbnail/<attachmentId>` over
the dsh `webServer` route seam (dsh 0.1's Typert Remote pipeline is closed to out-of-tree
packages). All commands for one session — tool calls and frame captures alike — run through a
per-session FIFO, because the daemon accepts only one unfinished command per session.
- **Trust model**: these routes expose live screenshots (and an interrupt write), so they
replicate the browser-trust fence dsh applies to its own `/api` routes: the request Host
must be a loopback authority (`localhost`, `127.0.0.0/8`, `[::1]`), a present Origin must
match the Host, `sec-fetch-site: cross-site` is refused, and POST requires an
`application/json` body (cross-site simple requests can never satisfy that). The channel is
therefore built for **loopback-only serving** — binding the dsh web server to `0.0.0.0` and
reaching it through a LAN address will (deliberately) fail the fence; do not put these
routes behind a non-loopback reverse proxy without adding your own authentication.
- Configure with `observationEnabled` / `thumbnailIntervalMs` / `idleIntervalMs`.
| Option | Default | Purpose |
| --- | --- | --- |
| `bskPath` | `bsk` | Path to the CLI binary. |
| `defaultTimeoutMs` | `120000` | Default command timeout in milliseconds. |
| `maxSessions` | `5` | Maximum concurrent sessions started by this plugin. |
| `observationEnabled` | `true` | Enable live browser observation. |
| `thumbnailIntervalMs` | `1500` | Screenshot interval for active sessions, in milliseconds. |
| `idleIntervalMs` | `8000` | Screenshot interval for idle sessions and the recent-activity window, in milliseconds. |
| `lazyTools` | `true` | Reveal the browser tools when the skill is invoked. Set `false` to register them at startup. |
## Behavior notes
With `lazyTools: true`, only the skill's catalog entry is initially advertised to
the model. The six `browser_*` tool schemas are added to the system prompt after
the `browser-skill` skill is successfully invoked, either by the model or through
`/browser-skill`. Set `lazyTools: false` to make the tools available immediately.
- **Cancellation**: aborting a tool call (`exec.signal`) kills the underlying bsk child process,
matching BrowserSkill's cooperative tool-cancellation model.
- **UI cards**: calls render as terminal cards (command line as title, output as the completed
card). Screenshots additionally attach the image itself when the host mounts an attachment store
and the active model route declares image input; otherwise the PNG path is returned.
- **Web UI toolview (browser half)**: the package is dual-face. `dsh.client` (platform `web`) ships
`lib/client.cjs`, which registers a keyed `tool.call.toolview` view for `browser_inspect`. The
custom view keeps a terminal block for every inspect action and, when a screenshot result carries
an image block, resolves the durable attachment through the client session's authorized
`readAttachment` RPC and renders it with the shared `MessageImage` thumbnail/lightbox atoms.
The other five tools keep the stock terminal card. The bundle follows the dsh client
contract: a CJS closure factory handed to `window.__ModuleLoader__.load`, platform modules
(`react`, `dsh-client-ui-*`) external, everything else inlined, CSS Modules compiled by
lightningcss.
- **Errors**: non-zero bsk exits surface the CLI's JSON error envelope (`code`, `message`, `hint`)
so the model gets the daemon's actionable guidance.
- **Long-running work** (e.g. `bsk record`) is not backgrounded via `ctx.jobs` yet — tracked as a
follow-up.
## Live browser view
The dsh Web UI shows the plugin's browser sessions in a floating panel. If your
profile provides the `dsh-better-sidebar` integration, the view appears in a
**Browser Skill** sidebar tab instead.
- See the current action, elapsed time, and recent screenshot for each session.
- Select a session to focus on it. The sidebar view follows the current conversation.
- Use **Interrupt** to cancel the current browser command. The agent may continue
with another action afterward.
- Drag or resize the floating panel, or use **Pop out** to open a Picture-in-Picture
window in browsers that support it.
- Periodic screenshots are requested while a browser observation view is visible.
Configure the active and idle intervals with the options above.
The observation endpoints require a loopback address such as `localhost` or
`127.0.0.1`. Access through a LAN hostname or non-loopback reverse proxy is not supported.
## Development
@@ -174,22 +154,47 @@ pnpm --filter @wxg-prc-cpg/browser-skill-dsh-plugin test # unit tests mock b
pnpm --filter @wxg-prc-cpg/browser-skill-dsh-plugin build # tsdown -> lib/
```
See the [development notes](https://github.com/Tencent/BrowserSkill/blob/main/packages/dsh-plugin-browserskill/docs/development.md)
for skill registration, tool results, and observation APIs in the current source.
## Publishing
The GitHub Actions workflow **Release dsh plugin** publishes this package to npm
as `@wxg-prc-cpg/browser-skill-dsh-plugin`. The Cordis plugin id, client
bundle registration, and `cordis.patch.yml` name are the same specifier so
dsh can import and materialize the plugin without a ModuleLoader id mismatch.
The [Release dsh plugin workflow](https://github.com/Tencent/BrowserSkill/blob/main/.github/workflows/release-dsh-plugin.yml)
publishes the package and this README to npm. Pushing a `dsh-plugin-vX.Y.Z` tag
triggers it; ordinary commits to `main` do not.
Trigger it by pushing a tag that matches `package.json`'s `version`:
1. Commit this README and any other changes intended for the release.
2. From the repository root, run the
[release script](https://github.com/Tencent/BrowserSkill/blob/main/scripts/release.mjs)
with a new stable version, replacing `<version>` below. The script updates the
CLI, extension, and DSH plugin versions, commits the version changes, and creates
their release tags:
```sh
git tag dsh-plugin-v0.1.2
git push origin dsh-plugin-v0.1.2
```
```sh
node scripts/release.mjs <version>
```
Or run the workflow from the Actions tab (`workflow_dispatch`). The job reads the
`NPM_TOKEN` secret from the `npm-publish` GitHub Environment.
3. Push the version commit to the release branch, then push the DSH plugin tag
created by the script, using the same `<version>`:
```sh
git push origin HEAD
git push origin dsh-plugin-v<version>
```
This publishes the DSH plugin. Push the CLI and extension tags separately when
those components are ready for release.
The workflow checks the version, runs typechecks and tests, builds the package,
and publishes it to npm.
You can also run the workflow manually from GitHub Actions on the intended release
ref. Both triggers require an unpublished version and the `NPM_TOKEN` secret in the
`npm-publish` GitHub Environment.
npm updates the package README only when a new version is published, including
for documentation-only changes. Published versions cannot be overwritten. See
[npm's README update rules](https://docs.npmjs.com/about-package-readme-files/).
## License
@@ -0,0 +1,90 @@
# DSH plugin development notes
For installation, usage, and configuration, see the [plugin README](../README.md).
These notes describe the current source on this branch; a published npm version may
not include later changes.
## Tool execution
The plugin runs the `bsk` CLI and parses its JSON output. The daemon and browser
extension perform the browser operations. The published package name is also the
Cordis plugin id, client bundle registration id, and `cordis.patch.yml` plugin name.
Keep these identifiers aligned so dsh can load both halves of the plugin.
## Agent skill
Beyond the tools, the plugin publishes the **`browser-skill` agent skill** through the harness's
official skill seam (`ctx.skills.register`): the catalog entry (name + routing description) is
resident in `<available_skills>`, and the body is loaded only when the model invokes the `skill`
tool. Its single source is the DSH-specific `skill/SKILL.md`, which documents only structured
`browser_*` calls and their plugin semantics. The repository-root CLI skill is intentionally not
concatenated: its command examples belong to a different execution interface and would bypass
the plugin's ownership, live observation UI, cancellation, and cleanup path if followed directly.
The build rejects internal CLI-name leakage, command-line code blocks, unknown browser tools, and
missing supported browser tools before embedding the Markdown. Registration and every pre-step
catalog snapshot are pure in-memory reads (no disk/process/daemon); compositions without the skill
seam degrade silently.
With `lazyTools: true`, the six tool schemas stay out of the system prompt until
the skill is successfully invoked. Registration happens once and lasts until the
plugin is unloaded. Repeated invocations do not register duplicate tools. Entering
a resumed conversation whose history contains a successful skill invocation also
registers the tools. Setting `lazyTools: false` registers them at plugin startup.
## Observation subscriptions and routes
- **Screenshot demand**: the client requests periodic screenshots while an observation view is
visible, using the PiP document's visibility when popped out. Hidden or collapsed views can
keep subscribing to state without requesting screenshots. The first screenshot subscriber
starts capture scheduling; the last one's departure cancels timers and skips queued captures.
An already running capture may finish. Demand applies to the service's owned sessions, not just
the selected session.
- **Wire**: `GET /bsk-observation/events` is the live synchronization channel (SSE). Every connection,
including reconnects, starts with `{ type: "snapshot", sessions, available }`, followed by
`upsert`, `remove`, `reset`, and `availability` events on the same stream. Replace local state
with each snapshot before applying subsequent events. Use `?thumbnails=0` for state-only
subscriptions or `?thumbnails=1` to request screenshots; omitting the parameter defaults to
requesting screenshots for compatibility. Closing the stream releases its screenshot demand.
`GET /bsk-observation/state` remains a one-time `{ sessions, available }` read and does not
request screenshots. It has no ordering guarantee relative to a separate SSE connection;
live clients should initialize and resynchronize from the stream's snapshot instead.
The host also serves `POST /bsk-observation/interrupt`, `POST /bsk-observation/stop`, and
`GET /bsk-observation/thumbnail/<attachmentId>` through the dsh `webServer` route seam
(dsh 0.1's Typert Remote pipeline is closed to out-of-tree packages). All commands for one
session — tool calls and frame captures alike — run through a per-session FIFO, because the
daemon accepts only one unfinished command per session.
- **Programmatic subscription**: `ObservationService.subscribe(listener, { thumbnails: false })`
subscribes to state without requesting screenshots. On an active service, the listener receives
the initial `snapshot` synchronously, before `subscribe` returns, then receives subsequent
changes. `thumbnails` defaults to `true`, so callers should choose it explicitly. The returned
unsubscribe function is safe to call repeatedly and releases this subscription's screenshot
demand. Subscribing after service disposal is a no-op.
- **Trust model**: these routes expose live screenshots (and an interrupt write), so they
replicate the browser-trust fence dsh applies to its own `/api` routes: the request Host
must be a loopback authority (`localhost`, `127.0.0.0/8`, `[::1]`), a present Origin must
match the Host, `sec-fetch-site: cross-site` is refused, and POST requires an
`application/json` body (cross-site simple requests can never satisfy that). The channel is
therefore built for **loopback-only serving** — binding the dsh web server to `0.0.0.0` and
reaching it through a LAN address will (deliberately) fail the fence; do not put these
routes behind a non-loopback reverse proxy without adding your own authentication.
## Tool results and cancellation
- **Cancellation**: aborting a tool call (`exec.signal`) kills the underlying bsk child process,
matching BrowserSkill's cooperative tool-cancellation model.
- **UI cards**: calls render as terminal cards (command line as title, output as the completed
card). Screenshots additionally attach the image itself when the host mounts an attachment store
and the active model route declares image input; otherwise the PNG path is returned.
- **Web UI toolview (browser half)**: the package is dual-face. `dsh.client` (platform `web`) ships
`lib/client.cjs`, which registers a keyed `tool.call.toolview` view for `browser_inspect`. The
custom view keeps a terminal block for every inspect action and, when a screenshot result carries
an image block, resolves the durable attachment through the client session's authorized
`readAttachment` RPC and renders it with the shared `MessageImage` thumbnail/lightbox atoms.
The other five tools keep the stock terminal card. The bundle follows the dsh client
contract: a CJS closure factory handed to `window.__ModuleLoader__.load`, platform modules
(`react`, `dsh-client-ui-*`) external, everything else inlined, CSS Modules compiled by
lightningcss.
- **Errors**: non-zero bsk exits surface the CLI's JSON error envelope (`code`, `message`, `hint`)
so the model gets the daemon's actionable guidance.
- **Long-running work** (e.g. `bsk record`) is not backgrounded via `ctx.jobs` yet — tracked as a
follow-up.
@@ -1,7 +1,7 @@
{
"name": "@wxg-prc-cpg/browser-skill-dsh-plugin",
"description": "DeepSeek Harness tool plugin that exposes BrowserSkill browser automation (browser_* tools) to the model",
"version": "0.2.0",
"version": "0.2.1",
"license": "MIT",
"publishConfig": {
"access": "public",
@@ -47,7 +47,6 @@
"client": {
"platform": "web",
"inject": [
"@deepseek-ai/dsh-client-runtime",
"@deepseek-ai/dsh-client-ui-tool",
"@deepseek-ai/dsh-client-ui-layout"
],
@@ -69,7 +68,6 @@
"peerDependencies": {
"@deepseek-ai/cordis": "^4.0.1",
"@deepseek-ai/dsh-attachment": "^0.1.0-rc.6",
"@deepseek-ai/dsh-client-runtime": "^0.1.0-rc.6",
"@deepseek-ai/dsh-client-ui-attachment": "^0.1.0-rc.6",
"@deepseek-ai/dsh-client-ui-primitives": "^0.1.0-rc.6",
"@deepseek-ai/dsh-client-ui-tool": "^0.1.0-rc.6",
@@ -82,9 +80,6 @@
"@deepseek-ai/dsh-attachment": {
"optional": true
},
"@deepseek-ai/dsh-client-runtime": {
"optional": true
},
"@deepseek-ai/dsh-client-ui-attachment": {
"optional": true
},
@@ -97,6 +97,10 @@ Continue from returned sequence cursors instead of rereading the same buffer.
- Unknown tab: list tabs instead of guessing.
- Unknown session: list owned sessions or start one; never try foreign ids.
- Timeout: inspect current state before deciding whether one longer purposeful wait is useful.
- Fill result unconfirmed: observe the field first; the page may have formatted the value. Continue
if the visible result satisfies the user's intent. Otherwise correct the remaining difference;
do not blindly repeat fill or immediately request human help. For other fill errors, follow the
returned hint and inspect current state before retrying.
- Unrecoverable failure: report the blocker and stop the owned session.
Arbitrary page-script evaluation and interaction recording are intentionally unsupported. Do not
@@ -142,6 +142,8 @@
right: 6px;
width: 16px;
height: 16px;
padding: 0;
cursor: pointer;
border-radius: 50%;
display: inline-flex;
align-items: center;
@@ -48,7 +48,13 @@ import { createPortal } from "react-dom";
import type { SessionObservation } from "../observation";
import css from "./ObservationOverlay.module.css";
import type { ObservationClientStore } from "./observation-store";
import { focusOf, statusOf, useObservationView, usePip } from "./observation-view";
import {
focusOf,
statusOf,
useObservationView,
usePip,
useThumbnailObservation,
} from "./observation-view";
import { getSidebarMode, subscribeSidebarMode } from "./sidebar-mode";
// The pure view helpers live in observation-view (shared with the sidebar
@@ -341,6 +347,7 @@ export function OverlayBody(props: {
onHeaderPointerDown,
} = props;
const [interrupting, setInterrupting] = useState(false);
const viewRef = useThumbnailObservation(store, sessions.length > 0);
const thumbId = focus?.thumbnailAttachmentId;
useEffect(() => {
@@ -373,7 +380,12 @@ export function OverlayBody(props: {
const state = !available ? "error" : focus !== undefined ? statusOf(focus) : "idle";
return (
<div className={cn(css.body, "bsk-obs")} data-state={state} data-in-pip={inPip || undefined}>
<div
ref={viewRef}
className={cn(css.body, "bsk-obs")}
data-state={state}
data-in-pip={inPip || undefined}
>
<div
className={css.header}
data-testid="obs-header"
@@ -421,9 +433,15 @@ export function OverlayBody(props: {
</div>
)}
{thumb?.status === "error" ? (
<span className={css.badge} aria-label="thumbnail failed">
<button
type="button"
className={css.badge}
aria-label="Retry thumbnail"
title="Frame unavailable — retry"
onClick={() => store.retryThumbnail(thumbId)}
>
<IconWarn size={12} />
</span>
</button>
) : null}
</div>
{sessions.length >= 2 ? (
@@ -7,6 +7,7 @@
*/
import type { ImageAttachmentRef } from "@deepseek-ai/dsh-attachment";
// Development-only types for the injected services; newer hosts no longer ship this package.
import type { ClientContext, ISessions, SessionId } from "@deepseek-ai/dsh-client-runtime/client";
// Type-only: pulls the 'shell.overlay' SlotMap merge into scope.
import type {} from "@deepseek-ai/dsh-client-ui-layout/client";
@@ -1,10 +1,9 @@
/**
* Client-side data layer for the observation overlay: initial state fetch,
* SSE increment stream (with a state refetch on every (re)open — events that
* fired while the stream was down are otherwise lost forever), on-demand
* thumbnail blob loading through the session-authorized readAttachment path,
* Client-side data layer for the observation overlay: an ordered SSE snapshot
* and increment stream (each reconnect starts with a fresh snapshot), on-demand
* thumbnail blob loading through the host's ephemeral-frame route,
* and the interrupt call. The last ready blob per session is held until its
* replacement decodes so the overlay can swap frames in place. All I/O is
* replacement loads so the overlay can swap frames in place. All I/O is
* injected so tests never touch a network.
*/
@@ -12,8 +11,6 @@ import type { ObservationEvent, SessionObservation } from "../observation";
export interface EventSourceLike {
onmessage: ((event: { data: string }) => void) | null;
/** Fires on the initial connect AND on every automatic reconnect. */
onopen?: (() => void) | null;
close(): void;
}
@@ -50,10 +47,10 @@ export interface OverlaySnapshot {
readonly available: boolean;
}
const STATE_URL = "/bsk-observation/state";
const EVENTS_URL = "/bsk-observation/events";
const INTERRUPT_URL = "/bsk-observation/interrupt";
const STOP_URL = "/bsk-observation/stop";
const THUMBNAIL_RETRY_DELAYS_MS = [1000, 3000];
function revoke(url: string | undefined): void {
if (url !== undefined && typeof URL.revokeObjectURL === "function") {
@@ -66,8 +63,10 @@ export class ObservationClientStore {
private thumbs = new Map<string, ThumbnailState>();
/** sessionId → the attachment id currently advertised for it. */
private readonly thumbBySession = new Map<string, string>();
/** sessionId → last successfully decoded frame (held across the next load). */
/** sessionId → last successfully loaded frame (held across the next load). */
private readonly lastReady = new Map<string, { attachmentId: string; url: string }>();
private readonly retryTimers = new Map<string, ReturnType<typeof setTimeout>>();
private readonly retryCounts = new Map<string, number>();
private listeners = new Set<() => void>();
private events: EventSourceLike | undefined;
private snapshot: OverlaySnapshot = {
@@ -81,6 +80,7 @@ export class ObservationClientStore {
private started = false;
/** Refcount of mounted consumers (overlay card, sidebar tab, sidebar fiber). */
private consumers = 0;
private thumbnailConsumers = 0;
constructor(private readonly deps: ObservationClientDeps) {}
@@ -133,37 +133,16 @@ export class ObservationClientStore {
return undefined;
}
/** (Re)pull the full state: initial load and every SSE (re)open. */
private refreshState(): void {
void this.deps
.fetchFn(STATE_URL)
.then(async (res) => {
if (!res.ok) return;
const body = (await res.json()) as {
sessions?: SessionObservation[];
available?: boolean;
};
const sessions = body.sessions ?? [];
this.sessions = new Map(sessions.map((s) => [s.sessionId, s]));
// Prune thumbnails of sessions that vanished while we were away.
const alive = new Set(sessions.map((s) => s.sessionId));
for (const sessionId of [...this.thumbBySession.keys()]) {
if (!alive.has(sessionId)) this.dropThumb(sessionId);
}
for (const s of sessions) this.trackThumb(s.sessionId, s.thumbnailAttachmentId);
if (typeof body.available === "boolean") this.available = body.available;
this.publish();
})
.catch(() => {});
}
/** Initial fetch + SSE subscription. Idempotent. */
start(): void {
if (this.started) return;
this.started = true;
this.refreshState();
const events = this.deps.eventSourceFactory(EVENTS_URL);
private connectEvents(): void {
const previous = this.events;
this.events = undefined;
previous?.close();
const events = this.deps.eventSourceFactory(
`${EVENTS_URL}?thumbnails=${this.thumbnailConsumers > 0 ? "1" : "0"}`,
);
this.events = events;
events.onmessage = (message) => {
if (!this.started || this.events !== events) return;
let event: ObservationEvent;
try {
event = JSON.parse(message.data) as ObservationEvent;
@@ -172,23 +151,24 @@ export class ObservationClientStore {
}
this.apply(event);
};
events.onopen = () => {
// Reconnects lose every event fired during the outage — resync.
if (this.events === events) this.refreshState();
};
this.events = events;
}
/** Every initial connection and reconnect starts with the server's snapshot. */
start(): void {
if (this.started) return;
this.started = true;
this.connectEvents();
this.publish();
}
stop(): void {
this.events?.close();
const previous = this.events;
this.events = undefined;
this.started = false;
for (const thumb of this.thumbs.values()) revoke(thumb.url);
this.thumbs.clear();
this.thumbBySession.clear();
this.lastReady.clear();
previous?.close();
this.clearThumbnails();
this.sessions.clear();
this.available = true;
this.publish();
}
@@ -209,6 +189,33 @@ export class ObservationClientStore {
if (this.consumers === 0) this.stop();
}
/** Only visible image surfaces request screenshots; metadata watchers do not. */
watchThumbnails(): () => void {
this.thumbnailConsumers += 1;
if (this.thumbnailConsumers === 1 && this.started) this.connectEvents();
let released = false;
return () => {
if (released) return;
released = true;
this.thumbnailConsumers -= 1;
if (this.thumbnailConsumers === 0 && this.started) this.connectEvents();
};
}
private forgetThumbnail(attachmentId: string): void {
clearTimeout(this.retryTimers.get(attachmentId));
this.retryTimers.delete(attachmentId);
this.retryCounts.delete(attachmentId);
revoke(this.thumbs.get(attachmentId)?.url);
this.thumbs.delete(attachmentId);
}
private clearThumbnails(): void {
for (const attachmentId of this.thumbs.keys()) this.forgetThumbnail(attachmentId);
this.thumbBySession.clear();
this.lastReady.clear();
}
/** Forget one session's tracked + held frames, revoking their blob URLs. */
private dropThumb(sessionId: string): void {
const attachmentId = this.thumbBySession.get(sessionId);
@@ -216,39 +223,45 @@ export class ObservationClientStore {
this.thumbBySession.delete(sessionId);
this.lastReady.delete(sessionId);
if (attachmentId !== undefined) {
revoke(this.thumbs.get(attachmentId)?.url);
this.thumbs.delete(attachmentId);
this.forgetThumbnail(attachmentId);
}
if (held !== undefined && held.attachmentId !== attachmentId) {
revoke(held.url);
this.thumbs.delete(held.attachmentId);
this.forgetThumbnail(held.attachmentId);
}
}
/**
* Track the frame a session currently advertises. The last *ready* blob is
* kept until the replacement decodes — dropping it on the upsert is what
* kept until the replacement loads — dropping it on the upsert is what
* made the overlay flash a placeholder between breaths.
*/
private trackThumb(sessionId: string, attachmentId: string | undefined): void {
if (attachmentId === undefined) {
this.dropThumb(sessionId);
return;
}
const previous = this.thumbBySession.get(sessionId);
if (previous === attachmentId) return;
if (attachmentId !== undefined) this.thumbBySession.set(sessionId, attachmentId);
else this.thumbBySession.delete(sessionId);
this.thumbBySession.set(sessionId, attachmentId);
const heldId = this.lastReady.get(sessionId)?.attachmentId;
if (previous !== undefined && previous !== heldId) {
revoke(this.thumbs.get(previous)?.url);
this.thumbs.delete(previous);
this.forgetThumbnail(previous);
}
}
private apply(event: ObservationEvent): void {
if (event.type === "reset") {
if (event.type === "snapshot") {
this.sessions = new Map(event.sessions.map((session) => [session.sessionId, session]));
for (const sessionId of this.thumbBySession.keys()) {
if (!this.sessions.has(sessionId)) this.dropThumb(sessionId);
}
for (const session of event.sessions) {
this.trackThumb(session.sessionId, session.thumbnailAttachmentId);
}
this.available = event.available;
} else if (event.type === "reset") {
this.sessions.clear();
for (const thumb of this.thumbs.values()) revoke(thumb.url);
this.thumbs.clear();
this.thumbBySession.clear();
this.lastReady.clear();
this.clearThumbnails();
} else if (event.type === "remove" && event.session !== undefined) {
this.sessions.delete(event.session.sessionId);
this.dropThumb(event.session.sessionId);
@@ -259,44 +272,77 @@ export class ObservationClientStore {
this.available = event.available;
}
this.publish();
if (event.type === "snapshot") {
for (const session of event.sessions) this.retryThumbnail(session.thumbnailAttachmentId);
}
}
/**
* Ensure a thumbnail load is in flight for one attachment reference. New
* frames replace the old URL only after they decode; failures keep the last
* frames replace the old URL only after they load; failures keep the last
* good frame (the caller renders `status: 'error'` as a small badge over
* the old image).
*/
ensureThumbnail(attachmentId: string | undefined): void {
if (attachmentId === undefined || this.thumbs.has(attachmentId)) return;
this.thumbs.set(attachmentId, { status: "loading" });
if (
!this.started ||
attachmentId === undefined ||
this.sessionOf(attachmentId) === undefined ||
this.thumbs.has(attachmentId)
)
return;
const loading: ThumbnailState = { status: "loading" };
this.thumbs.set(attachmentId, loading);
this.publish();
this.deps.loadImage(attachmentId).then(
(url) => {
if (!this.thumbs.has(attachmentId)) {
if (this.thumbs.get(attachmentId) !== loading) {
// Replaced or removed while loading: never resurrect (or leak) it.
revoke(url);
return;
}
this.retryCounts.delete(attachmentId);
this.thumbs.set(attachmentId, { status: "ready", url });
const sessionId = this.sessionOf(attachmentId);
if (sessionId !== undefined) {
const previous = this.lastReady.get(sessionId);
if (previous !== undefined && previous.attachmentId !== attachmentId) {
revoke(previous.url);
this.thumbs.delete(previous.attachmentId);
this.forgetThumbnail(previous.attachmentId);
}
this.lastReady.set(sessionId, { attachmentId, url });
}
this.publish();
},
() => {
this.thumbs.set(attachmentId, { status: "error" });
if (this.thumbs.get(attachmentId) !== loading) return;
const failed: ThumbnailState = { status: "error" };
this.thumbs.set(attachmentId, failed);
const attempts = this.retryCounts.get(attachmentId) ?? 0;
const delay = THUMBNAIL_RETRY_DELAYS_MS[attempts];
if (delay !== undefined) {
this.retryCounts.set(attachmentId, attempts + 1);
this.retryTimers.set(
attachmentId,
setTimeout(() => {
this.retryTimers.delete(attachmentId);
if (this.thumbs.get(attachmentId) !== failed) return;
this.thumbs.delete(attachmentId);
this.ensureThumbnail(attachmentId);
}, delay),
);
}
this.publish();
},
);
}
/** Explicit retry (also used on a fresh connection after an outage). */
retryThumbnail(attachmentId: string | undefined): void {
if (attachmentId === undefined || this.thumbs.get(attachmentId)?.status !== "error") return;
this.forgetThumbnail(attachmentId);
this.ensureThumbnail(attachmentId);
}
/** Interrupt the current or named session's in-flight call. */
async interrupt(sessionId?: string): Promise<boolean> {
try {

Some files were not shown because too many files have changed in this diff Show More