mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
Compare commits
96
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
731139954b | ||
|
|
90b0ff301b | ||
|
|
e74f354e90 | ||
|
|
05c036518a | ||
|
|
020fb3212e | ||
|
|
0822c53c65 | ||
|
|
4450fbe28c | ||
|
|
a3bb81bfab | ||
|
|
caea98cd52 | ||
|
|
187b088013 | ||
|
|
5046b5ab44 | ||
|
|
e8137b42cc | ||
|
|
cb5dc148d5 | ||
|
|
3c5f838c56 | ||
|
|
9337bd557b | ||
|
|
162a31ae92 | ||
|
|
2f84234a31 | ||
|
|
72c10aab75 | ||
|
|
1eeb55af02 | ||
|
|
4992a54864 | ||
|
|
8b195eb0df | ||
|
|
eae4db1729 | ||
|
|
e961395be9 | ||
|
|
a6d1d02acc | ||
|
|
5ebe609c40 | ||
|
|
321494d93f | ||
|
|
f2184e74c5 | ||
|
|
eeb633d68e | ||
|
|
6d181105f2 | ||
|
|
ebad0ed1b5 | ||
|
|
498f8fb4a7 | ||
|
|
cef59137e4 | ||
|
|
f70978bfcf | ||
|
|
1498cd425c | ||
|
|
e831d65db3 | ||
|
|
47ac947f18 | ||
|
|
69d64c4eb1 | ||
|
|
99c997d6d5 | ||
|
|
93853ebc65 | ||
|
|
248bb90a2d | ||
|
|
6e6c178434 | ||
|
|
386ef91c24 | ||
|
|
0dc11b30ea | ||
|
|
759eebd4b6 | ||
|
|
fd3ce202a1 | ||
|
|
618bd6a0df | ||
|
|
c455a09fc3 | ||
|
|
b6be356e40 | ||
|
|
945bf1523d | ||
|
|
a928a9fb76 | ||
|
|
0b2d7e95cc | ||
|
|
556bea1645 | ||
|
|
0302d5592b | ||
|
|
eb34ec0dbf | ||
|
|
43067b6bfe | ||
|
|
6cc0d38489 | ||
|
|
0b71963344 | ||
|
|
59e2c3224e | ||
|
|
d70df753f6 | ||
|
|
66ae555e40 | ||
|
|
fee7c0983c | ||
|
|
51473c01ff | ||
|
|
49744233f7 | ||
|
|
95571f8b70 | ||
|
|
8344e311fa | ||
|
|
63338cd98e | ||
|
|
bd193ceb5d | ||
|
|
5ed828736f | ||
|
|
ee620acf64 | ||
|
|
05a36075b8 | ||
|
|
1bd13d5c3c | ||
|
|
59d6533eff | ||
|
|
536d3a9844 | ||
|
|
a77353bce8 | ||
|
|
91621b2545 | ||
|
|
641d934337 | ||
|
|
a4fed136e3 | ||
|
|
3c77ef7b01 | ||
|
|
3b21f2f0a7 | ||
|
|
405433ab65 | ||
|
|
58c7f38a05 | ||
|
|
e0c88295c7 | ||
|
|
dc36e38cb9 | ||
|
|
d5de9b0154 | ||
|
|
995061514c | ||
|
|
406bb21b7e | ||
|
|
b1b1ae1d48 | ||
|
|
8822bba9a5 | ||
|
|
c74c119a82 | ||
|
|
b801bc1f62 | ||
|
|
bf59753aa3 | ||
|
|
a86cbf724f | ||
|
|
76d447caa0 | ||
|
|
14e5bbe4fc | ||
|
|
bef722107f | ||
|
|
6800f8fdfe |
@@ -37,6 +37,35 @@ jobs:
|
||||
- name: Run Rust tests
|
||||
run: cargo test --workspace --locked
|
||||
|
||||
windows-daemon:
|
||||
name: Windows daemon, installer and cancellation regression tests
|
||||
runs-on: windows-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Run Windows named-pipe tests
|
||||
run: cargo test -p bsk --lib --locked daemon::ipc::windows::tests
|
||||
|
||||
- name: Run Windows process liveness tests
|
||||
run: cargo test -p bsk --lib --locked daemon::lockfile::tests
|
||||
|
||||
- name: Run Windows update tests
|
||||
run: cargo test -p bsk --lib --locked cli::update::tests
|
||||
|
||||
- name: Run Windows update and cancellation process tests
|
||||
run: cargo test -p bsk --test windows_update --test windows_parent_cancel --locked
|
||||
|
||||
- name: Run installer tests (Windows PowerShell 5.1)
|
||||
shell: powershell
|
||||
run: ./scripts/install-windows.test.ps1
|
||||
|
||||
- name: Run installer tests (PowerShell 7)
|
||||
shell: pwsh
|
||||
run: ./scripts/install-windows.test.ps1
|
||||
|
||||
frontend:
|
||||
name: Frontend lint, typecheck, tests, build
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
@@ -7,6 +7,10 @@ dist/
|
||||
.output/
|
||||
.wxt/
|
||||
.web-extension-id
|
||||
.pnpm-store/
|
||||
|
||||
# Git worktrees
|
||||
.worktrees/
|
||||
|
||||
# Logs
|
||||
*.log
|
||||
|
||||
+109
@@ -0,0 +1,109 @@
|
||||
# Changelog
|
||||
|
||||
All notable changes to BrowserSkill will be documented in this file.
|
||||
The format is based on [Keep a Changelog](https://keepachangelog.com/).
|
||||
|
||||
Starting from 0.2.0, CLI / Extension / DSH Plugin share the same version number.
|
||||
|
||||
## [0.2.1] - 2026-09-09
|
||||
|
||||
### Changed
|
||||
- DSH Plugin: clarified installation, configuration, profile restart, and upgrade instructions
|
||||
|
||||
### Fixed
|
||||
- CLI: reliable Windows self-updates, including executable replacement and daemon restart
|
||||
- CLI: Windows named-pipe connection continuity and stale daemon process detection
|
||||
- Windows installer: PowerShell 5.1 compatibility, path quoting, and staged executable replacement
|
||||
- CLI / DSH Plugin: cancellation propagation and upload/download cleanup when the parent process exits
|
||||
- Extension: tab ownership and session cleanup when agent tabs fail to open or borrowed tabs are returned or closed
|
||||
- Extension: navigation recovery when other extensions restrict Chrome DevTools Protocol access
|
||||
- Extension: background input filling, automatic append behavior, and verification of field values before reporting success
|
||||
- Extension: match observed form state to the correct element after page changes
|
||||
- Extension: UUID generation fallback for recording and overlays on insecure origins
|
||||
- Browser locale matching for regional language variants
|
||||
- DSH Plugin: observation state synchronization, thumbnail capture lifecycle, and stale preview cleanup
|
||||
- DSH Plugin: removed the obsolete client runtime host dependency
|
||||
|
||||
## [0.2.0] - 2026-09-02
|
||||
|
||||
### Added
|
||||
- File transfer: upload and download support across CLI, Extension, and DSH Plugin
|
||||
- File transfer: drag-and-drop upload (`drop-to-upload`)
|
||||
- VOM semantic graph, name enrichment, and hover perception modules
|
||||
- VOM hover probing (opt-in via `observe` parameter)
|
||||
- DSH Plugin: browser tool parity with CLI commands
|
||||
- Edge Add-ons automated publishing in CI
|
||||
- Protocol upgrade reminder when CLI / Extension protocol versions differ
|
||||
- Browser evaluation harness (`evals/browser/`)
|
||||
- Unified release script (`scripts/release.mjs`)
|
||||
|
||||
### Changed
|
||||
- **Version scheme**: all three components now share the same semver
|
||||
- VOM rendering algorithm optimizations
|
||||
- Leaner SKILL.md agent instructions
|
||||
- DSH Plugin: simplified browser commands
|
||||
- Borrow confirmation UX — proactive focus and longer timeout
|
||||
- PiP window now has a close button
|
||||
|
||||
### Fixed
|
||||
- Screenshot media type detection (was hard-coded to `image/png`)
|
||||
- DSH Plugin session lifecycle stability
|
||||
- DSH Plugin Cordis package ID mismatch
|
||||
- Observation thumbnail media type sniffing
|
||||
- Upload/download race conditions and layout bypass issues
|
||||
- VOM repeated name and safety policy issues
|
||||
|
||||
---
|
||||
|
||||
*Previous releases used independent version numbers per component.*
|
||||
|
||||
## CLI 0.1.11 / Extension 0.1.7 / DSH Plugin 0.1.2 — 2026-08-26 ~ 2026-08-29
|
||||
|
||||
### Added
|
||||
- DSH Plugin sidebar integration, session lifecycle, and archive cleanup
|
||||
- Recorder iframe and OOPIF support
|
||||
|
||||
### Changed
|
||||
- VOM functional refactor
|
||||
|
||||
### Fixed
|
||||
- Recorder safety policy and bug fixes
|
||||
|
||||
## CLI 0.1.10 / Extension 0.1.6 — 2026-08-08
|
||||
|
||||
### Added
|
||||
- VOM observation recording and settled-state detection
|
||||
- Record overlay timer
|
||||
|
||||
### Fixed
|
||||
- Browser keepalive disconnect handling
|
||||
|
||||
## CLI 0.1.9 / Extension 0.1.5 — 2026-07-29
|
||||
|
||||
### Added
|
||||
- CLI auto-update mechanism
|
||||
- Trace v3 protocol and recorder
|
||||
|
||||
### Fixed
|
||||
- MV3 keepalive disconnect
|
||||
|
||||
## CLI 0.1.8 / Extension 0.1.4 — 2026-07-22
|
||||
|
||||
### Added
|
||||
- More browser interaction actions
|
||||
|
||||
### Fixed
|
||||
- Windows named-pipe hash-only path issue
|
||||
|
||||
## CLI 0.1.7 / Extension 0.1.3 — 2026-07-07
|
||||
|
||||
Initial public release pair.
|
||||
|
||||
## CLI 0.1.6 — 2026-06-30
|
||||
|
||||
### Fixed
|
||||
- Minor CLI fixes
|
||||
|
||||
## CLI 0.1.5 / Extension 0.1.2 — 2026-06-22
|
||||
|
||||
First tagged releases.
|
||||
Generated
+3
-2
@@ -134,9 +134,10 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "bsk"
|
||||
version = "0.1.11"
|
||||
version = "0.2.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"base64",
|
||||
"bsk-protocol",
|
||||
"clap",
|
||||
"console",
|
||||
@@ -168,7 +169,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "bsk-protocol"
|
||||
version = "0.1.11"
|
||||
version = "0.2.1"
|
||||
dependencies = [
|
||||
"schemars",
|
||||
"semver",
|
||||
|
||||
+3
-2
@@ -7,7 +7,7 @@ members = [
|
||||
default-members = ["crates/bsk-cli"]
|
||||
|
||||
[workspace.package]
|
||||
version = "0.1.11"
|
||||
version = "0.2.1"
|
||||
edition = "2024"
|
||||
rust-version = "1.85"
|
||||
license = "MIT"
|
||||
@@ -36,13 +36,14 @@ flate2 = "1.1.9"
|
||||
tar = "0.4.46"
|
||||
zip = { version = "7.2.0", default-features = false, features = ["deflate"] }
|
||||
sha2 = "0.11.0"
|
||||
base64 = "0.22"
|
||||
tempfile = "3"
|
||||
nix = { version = "0.29", features = ["signal", "process"] }
|
||||
rand = "0.8"
|
||||
uuid = { version = "1", features = ["v4", "serde"] }
|
||||
|
||||
# Workspace crates (version required for `cargo publish` of dependents)
|
||||
bsk-protocol = { version = "0.1.9", path = "crates/bsk-protocol" }
|
||||
bsk-protocol = { version = "0.2.1", path = "crates/bsk-protocol" }
|
||||
|
||||
[profile.release]
|
||||
strip = "symbols"
|
||||
|
||||
@@ -47,14 +47,17 @@ extension.
|
||||
|
||||
## Quick Start
|
||||
|
||||
### 1. Install the CLI and skill
|
||||
|
||||
<details open>
|
||||
<summary><b>Install with your Agent (recommended)</b></summary>
|
||||
|
||||
<br>
|
||||
|
||||
Already using Cursor, Claude Code, Codex, or another shell-capable agent? Just
|
||||
copy this one line and send it to your agent — it will install the CLI and skill
|
||||
for you, then walk you through loading the extension:
|
||||
Already using Cursor, Claude Code, Codex, WorkBuddy, or another shell-capable
|
||||
agent? Just copy this one line and send it to your agent — it will install the
|
||||
CLI and skill for you. You will install the browser extension yourself in step 2
|
||||
below.
|
||||
|
||||
```text
|
||||
Set up browser-skill on this machine by following https://raw.githubusercontent.com/Tencent/BrowserSkill/main/AGENT_INSTALL.md
|
||||
@@ -67,10 +70,9 @@ Set up browser-skill on this machine by following https://raw.githubusercontent.
|
||||
|
||||
<br>
|
||||
|
||||
Install the CLI, then install the extension from the [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi)
|
||||
or [Edge Add-ons](https://microsoftedge.microsoft.com/addons/detail/browserskill/emacgiaaaiojkkpkddmmdfhmokgmnikg).
|
||||
Install the CLI and skill with the commands below.
|
||||
|
||||
#### 1. Install the `bsk` CLI
|
||||
#### Install the `bsk` CLI
|
||||
|
||||
**macOS / Linux** (recommended — installs to `~/.local/bin`):
|
||||
|
||||
@@ -90,18 +92,7 @@ Verify the binary:
|
||||
bsk --version
|
||||
```
|
||||
|
||||
#### 2. Install the browser extension
|
||||
|
||||
Install BrowserSkill from your browser's store:
|
||||
|
||||
| Browser | Store listing |
|
||||
| --- | --- |
|
||||
| Chrome | [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi) |
|
||||
| Microsoft Edge | [Edge Add-ons](https://microsoftedge.microsoft.com/addons/detail/browserskill/emacgiaaaiojkkpkddmmdfhmokgmnikg) |
|
||||
|
||||
On other Chromium-based browsers, install the Chrome Web Store build.
|
||||
|
||||
#### 3. Install the skill
|
||||
#### Install the skill
|
||||
|
||||
BrowserSkill ships a skill that teaches your agent harness how to use `bsk`. For
|
||||
these harnesses, install it in one step:
|
||||
@@ -136,6 +127,21 @@ dedicated plugin instead — see [DeepSeek Harness plugin](#deepseek-harness-plu
|
||||
|
||||
</details>
|
||||
|
||||
### 2. Install the browser extension
|
||||
|
||||
Due to current browser security policies, you still need to manually install
|
||||
the extension for your browser.
|
||||
Open the store link for your browser and confirm the extension installation:
|
||||
|
||||
| Browser | Store listing |
|
||||
| --- | --- |
|
||||
| Chrome | [Chrome Web Store](https://chromewebstore.google.com/detail/browserskill/hhcmgoofomhgciiibhipgmgkgnoenaoi) |
|
||||
| Microsoft Edge | [Edge Add-ons](https://microsoftedge.microsoft.com/addons/detail/browserskill/emacgiaaaiojkkpkddmmdfhmokgmnikg) |
|
||||
|
||||
On other Chromium-based browsers, install the Chrome Web Store build.
|
||||
|
||||
### 3. Start using BrowserSkill
|
||||
|
||||
Start a new Agent session and write a prompt that needs the browser, for example:
|
||||
|
||||
```text
|
||||
@@ -147,20 +153,26 @@ Start a new Agent session and write a prompt that needs the browser, for example
|
||||
Using [DeepSeek Harness](https://github.com/deepseek-ai/deepseek-harness) (`dsh`)?
|
||||
BrowserSkill ships a first-class dsh plugin on npm as
|
||||
[`@wxg-prc-cpg/browser-skill-dsh-plugin`](https://www.npmjs.com/package/@wxg-prc-cpg/browser-skill-dsh-plugin).
|
||||
It injects native `browser_*` tools (no shelling out to `bsk`) and a live Web UI
|
||||
overlay of each Agent Window.
|
||||
It gives the agent native `browser_*` tools and a live view of its browser sessions
|
||||
in the Web UI. The plugin runs `bsk` on the agent's behalf.
|
||||
|
||||
Add it to a dsh profile, then start that profile:
|
||||
Install the `bsk` CLI and connect the browser extension first. Then add the plugin
|
||||
to a dsh profile and start it (replace `web` with your profile name):
|
||||
|
||||
```sh
|
||||
dsh plugin --profile web add @wxg-prc-cpg/browser-skill-dsh-plugin
|
||||
dsh --profile web
|
||||
```
|
||||
|
||||
The plugin carries its own copy of the skill, so `bsk install-skill` is not needed
|
||||
for dsh — but the `bsk` CLI and the browser extension are still prerequisites. See
|
||||
the [plugin README](packages/dsh-plugin-browserskill/README.md) for the tool list,
|
||||
configuration, and the observation overlay.
|
||||
The plugin includes the `browser-skill` skill, so `bsk install-skill` is not needed
|
||||
for dsh. Installed plugins do not update automatically. To upgrade this plugin:
|
||||
|
||||
```sh
|
||||
dsh plugin --profile web update @wxg-prc-cpg/browser-skill-dsh-plugin --latest
|
||||
```
|
||||
|
||||
Restart the profile after upgrading. See the
|
||||
[plugin README](packages/dsh-plugin-browserskill/README.md) for usage and configuration.
|
||||
|
||||
## How It Works
|
||||
|
||||
|
||||
+29
-19
@@ -36,12 +36,14 @@ BrowserSkill 由两个本地运行组件组成:`bsk` CLI/daemon 和浏览器
|
||||
|
||||
## 快速开始
|
||||
|
||||
### 1. 安装 CLI 和 skill
|
||||
|
||||
<details open>
|
||||
<summary><b>让 Agent 帮你安装(推荐)</b></summary>
|
||||
|
||||
<br>
|
||||
|
||||
已经在用 Cursor、Claude Code、Codex 或其他支持 Shell 的 Agent?只需复制下面这句话发给 Agent,它会帮你安装 CLI 和 skill,并引导你加载浏览器扩展:
|
||||
已经在用 Cursor、Claude Code、Codex、WorkBuddy 或其他支持 Shell 的 Agent?只需复制下面这句话发给 Agent,它会帮你安装 CLI 和 skill。浏览器扩展仍需你按下方第 2 步自行安装。
|
||||
|
||||
```text
|
||||
按照 https://raw.githubusercontent.com/Tencent/BrowserSkill/main/AGENT_INSTALL.md 的说明,在本机安装并配置 browser-skill
|
||||
@@ -54,10 +56,9 @@ BrowserSkill 由两个本地运行组件组成:`bsk` CLI/daemon 和浏览器
|
||||
|
||||
<br>
|
||||
|
||||
先安装 CLI,再从 [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi)
|
||||
或 [Edge 加载项商店](https://microsoftedge.microsoft.com/addons/detail/browserskill/emacgiaaaiojkkpkddmmdfhmokgmnikg) 安装浏览器扩展。
|
||||
按以下命令安装 CLI 和 skill。
|
||||
|
||||
#### 1. 安装 `bsk` CLI
|
||||
#### 安装 `bsk` CLI
|
||||
|
||||
**macOS / Linux**(推荐,安装到 `~/.local/bin`):
|
||||
|
||||
@@ -77,18 +78,7 @@ irm https://raw.githubusercontent.com/Tencent/BrowserSkill/main/install.ps1 | ie
|
||||
bsk --version
|
||||
```
|
||||
|
||||
#### 2. 安装浏览器扩展
|
||||
|
||||
在对应浏览器的商店安装 BrowserSkill:
|
||||
|
||||
| 浏览器 | 商店页面 |
|
||||
| --- | --- |
|
||||
| Chrome | [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi) |
|
||||
| Microsoft Edge | [Edge 加载项商店](https://microsoftedge.microsoft.com/addons/detail/browserskill/emacgiaaaiojkkpkddmmdfhmokgmnikg) |
|
||||
|
||||
其他基于 Chromium 的浏览器,安装 Chrome Web Store 版本即可。
|
||||
|
||||
#### 3. 安装 skill
|
||||
#### 安装 skill
|
||||
|
||||
BrowserSkill 自带 skill,用于教 Agent harness 如何使用 `bsk`。以下 harness 可一键安装:
|
||||
|
||||
@@ -117,6 +107,20 @@ bsk install-skill
|
||||
|
||||
</details>
|
||||
|
||||
### 2. 安装浏览器扩展
|
||||
|
||||
目前由于浏览器的安全策略,您还需要手动安装对应的浏览器扩展。
|
||||
点击对应浏览器的商店链接,并在浏览器中确认安装扩展:
|
||||
|
||||
| 浏览器 | 商店页面 |
|
||||
| --- | --- |
|
||||
| Chrome | [Chrome Web Store](https://chromewebstore.google.com/detail/browserskill/hhcmgoofomhgciiibhipgmgkgnoenaoi) |
|
||||
| Microsoft Edge | [Edge 加载项商店](https://microsoftedge.microsoft.com/addons/detail/browserskill/emacgiaaaiojkkpkddmmdfhmokgmnikg) |
|
||||
|
||||
其他基于 Chromium 的浏览器,安装 Chrome Web Store 版本即可。
|
||||
|
||||
### 3. 开始使用 BrowserSkill
|
||||
|
||||
启动一个新的 Agent 会话,写一条需要使用浏览器的 prompt,例如:
|
||||
|
||||
```text
|
||||
@@ -125,16 +129,22 @@ bsk install-skill
|
||||
|
||||
## DeepSeek Harness 插件
|
||||
|
||||
在用 [DeepSeek Harness](https://github.com/deepseek-ai/deepseek-harness)(`dsh`)?BrowserSkill 提供了官方 dsh 插件,已发布到 npm:[`@wxg-prc-cpg/browser-skill-dsh-plugin`](https://www.npmjs.com/package/@wxg-prc-cpg/browser-skill-dsh-plugin)。它会注入原生 `browser_*` 工具(无需再通过 Shell 调用 `bsk`),并在 Web UI 中实时观察每个 Agent Window。
|
||||
在用 [DeepSeek Harness](https://github.com/deepseek-ai/deepseek-harness)(`dsh`)?BrowserSkill 提供了官方 dsh 插件,已发布到 npm:[`@wxg-prc-cpg/browser-skill-dsh-plugin`](https://www.npmjs.com/package/@wxg-prc-cpg/browser-skill-dsh-plugin)。它为 Agent 提供原生 `browser_*` 工具,由插件代为调用 `bsk`,并在 Web UI 中实时展示浏览器会话。
|
||||
|
||||
把它装进某个 dsh profile,然后启动该 profile:
|
||||
先安装 `bsk` CLI 并连接浏览器扩展,再将插件装进 dsh profile 并启动(将 `web` 替换为你的 profile 名称):
|
||||
|
||||
```sh
|
||||
dsh plugin --profile web add @wxg-prc-cpg/browser-skill-dsh-plugin
|
||||
dsh --profile web
|
||||
```
|
||||
|
||||
插件自带 skill,所以在 dsh 下无需执行 `bsk install-skill`;但 `bsk` CLI 和浏览器扩展仍是前置条件。工具清单、配置项与观察浮层见[插件 README](packages/dsh-plugin-browserskill/README.md)。
|
||||
插件自带 `browser-skill` skill,所以在 dsh 下无需执行 `bsk install-skill`。已安装的插件不会自动更新;升级此插件请运行:
|
||||
|
||||
```sh
|
||||
dsh plugin --profile web update @wxg-prc-cpg/browser-skill-dsh-plugin --latest
|
||||
```
|
||||
|
||||
升级后重启该 profile。用法与配置见[插件 README](packages/dsh-plugin-browserskill/README.md)。
|
||||
|
||||
## 工作原理
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# BrowserSkill — Privacy Policy
|
||||
|
||||
**Last updated:** May 25, 2026
|
||||
**Last updated:** August 20, 2026
|
||||
|
||||
This Privacy Policy describes how the **BrowserSkill** browser extension (the "Extension") handles information when you install and use it. BrowserSkill is published as part of the open-source [BrowserSkill](https://github.com/Tencent/BrowserSkill) project. The source code is publicly auditable.
|
||||
|
||||
@@ -14,7 +14,7 @@ BrowserSkill is a local automation bridge that lets AI coding agents (such as Cu
|
||||
|
||||
## 2. Single Purpose
|
||||
|
||||
The Extension's single purpose is to expose browser automation primitives (navigation, DOM observation, screenshots, clicks, form filling, tab management) to a locally running BrowserSkill daemon over a WebSocket connection on `127.0.0.1`, so that an AI agent invoked by the user can interact with web pages on the user's behalf.
|
||||
The Extension's single purpose is to expose browser automation primitives (navigation, DOM observation, screenshots, clicks, form filling, task-scoped file transfer, and tab management) to a locally running BrowserSkill daemon over a WebSocket connection on `127.0.0.1`, so that an AI agent invoked by the user can interact with web pages on the user's behalf.
|
||||
|
||||
## 3. Data the Extension Accesses
|
||||
|
||||
@@ -26,6 +26,7 @@ Depending on the commands the user (via their AI agent) sends to the local daemo
|
||||
| **User input simulated by the agent** | Mouse clicks, keystrokes, and form values that the AI agent dispatches through the Chrome DevTools Protocol (CDP). | Required to perform automation actions the user has asked the agent to do. |
|
||||
| **Tab and window metadata** | Tab IDs, URLs, titles, window IDs of the Agent Window and any tabs the user explicitly authorizes. | Required to target automation commands at the correct tab/window. |
|
||||
| **Local extension storage** | A randomly generated 8-character instance ID and an optional user-supplied label. | Used so the local daemon can recognize this browser instance across reconnects. No personal data is stored. |
|
||||
| **File transfers requested by the agent** | Local files explicitly supplied to `bsk upload`, and the file created by a single `bsk download` action. | Required to attach a task file to a web page or return a browser-generated download to the invoking local agent. |
|
||||
| **OS notifications** | Permission to display a system notification when the agent requests to "borrow" one of the user's existing tabs. | Required to obtain explicit, per-tab user consent before the agent touches any pre-existing tab. |
|
||||
|
||||
## 4. Data the Extension Does **Not** Collect
|
||||
@@ -34,7 +35,7 @@ BrowserSkill does **not**:
|
||||
|
||||
- Send any data to remote servers, the Extension's authors, or any third party.
|
||||
- Call any LLM, AI, or cloud API. The Extension contains no API keys, model identifiers, or remote endpoints.
|
||||
- Read or transmit cookies, browsing history, bookmarks, downloads, saved passwords, or autofill data.
|
||||
- Read or transmit cookies, browsing history, bookmarks, saved passwords, or autofill data. It observes only the download initiated by an active `bsk download` call, not download history generally.
|
||||
- Use webcam, microphone, geolocation, or any device sensor.
|
||||
- Include analytics, telemetry, crash reporting, advertising SDKs, or fingerprinting code.
|
||||
- Track users across websites or across sessions.
|
||||
@@ -50,6 +51,7 @@ The Extension requests the following Chrome permissions. Each is used solely for
|
||||
- **`alarms`** — Periodically wake the service worker to keep the local WebSocket connection alive.
|
||||
- **`idle`** — Detect when the device returns from idle/locked so the Extension can promptly re-establish the local WebSocket connection after the machine wakes. No idle data is stored or transmitted.
|
||||
- **`notifications`** — Show a system notification to obtain user approval before the agent borrows a user-owned tab.
|
||||
- **`downloads`** — Correlate and route the one browser download initiated by an active `bsk download` command. If that claimed transaction fails, BrowserSkill cancels an in-progress file or removes its completed temporary browser file. It is not used to enumerate download history or alter unclaimed downloads.
|
||||
- **`storage`** — Persist a random instance ID and optional label in `chrome.storage.local`.
|
||||
- **Host permission `<all_urls>`** — Inject a small status overlay (showing "Agent Active") on pages controlled by the agent, and enable automation across whatever sites the user directs the agent to. The Extension does **not** read or transmit page content from sites the agent is not actively driving.
|
||||
|
||||
@@ -61,6 +63,7 @@ All Extension activity stays on the user's local device. The only network traffi
|
||||
|
||||
- The instance ID and optional label persist in `chrome.storage.local` until the user uninstalls the Extension or clears extension storage.
|
||||
- Page content, screenshots, DOM snapshots, and other observed data are returned to the local daemon in response to commands and are **not retained by the Extension**. They live only as long as the agent's tool call.
|
||||
- Upload and download bytes are staged by the local daemon in a private, session-scoped directory. Download staging is removed after it is copied to the requested destination. Upload staging is retained until the session ends so a later form submission can still read the attached file. Remaining staging is removed when the session ends or disconnects, or when the daemon next starts after a crash.
|
||||
|
||||
## 8. User Control
|
||||
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<title>BrowserSkill file-transfer fixture</title>
|
||||
<style>
|
||||
body { font: 16px system-ui; max-width: 760px; margin: 48px auto; }
|
||||
.actions { display: flex; gap: 12px; align-items: center; margin: 24px 0; }
|
||||
button, a { padding: 10px 16px; border: 1px solid #777; border-radius: 8px; }
|
||||
a[aria-disabled="true"] { opacity: .45; pointer-events: none; }
|
||||
#drop { padding: 36px; border: 3px dashed #777; border-radius: 12px; text-align: center; }
|
||||
#drop.dragging { border-color: #2563eb; background: #eff6ff; }
|
||||
img { display: block; max-width: 480px; max-height: 360px; margin-top: 20px; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Upload then download the same image</h1>
|
||||
<p id="status">No image selected</p>
|
||||
<div class="actions">
|
||||
<button id="upload" type="button">Upload through file input</button>
|
||||
<input id="file" type="file" accept="image/png" hidden />
|
||||
<a id="download" aria-disabled="true">Download uploaded image</a>
|
||||
</div>
|
||||
<div id="drop" role="button" tabindex="0">Drop an image here</div>
|
||||
<img id="preview" alt="Uploaded image preview" hidden />
|
||||
<script>
|
||||
const input = document.querySelector('#file');
|
||||
const upload = document.querySelector('#upload');
|
||||
const drop = document.querySelector('#drop');
|
||||
const download = document.querySelector('#download');
|
||||
const preview = document.querySelector('#preview');
|
||||
const status = document.querySelector('#status');
|
||||
let objectUrl;
|
||||
|
||||
window.__bskTransferFixture = { events: [], result: null };
|
||||
|
||||
function accept(file, mechanism) {
|
||||
if (!file) return;
|
||||
if (objectUrl) URL.revokeObjectURL(objectUrl);
|
||||
objectUrl = URL.createObjectURL(file);
|
||||
preview.src = objectUrl;
|
||||
preview.hidden = false;
|
||||
download.href = objectUrl;
|
||||
download.download = file.name;
|
||||
download.removeAttribute('aria-disabled');
|
||||
status.textContent = `Ready via ${mechanism}: ${file.name} (${file.size} bytes)`;
|
||||
window.__bskTransferFixture.result = {
|
||||
mechanism,
|
||||
name: file.name,
|
||||
size: file.size,
|
||||
type: file.type,
|
||||
hasPathProperty: 'path' in file,
|
||||
};
|
||||
}
|
||||
|
||||
upload.addEventListener('click', () => input.click());
|
||||
input.addEventListener('change', () => accept(input.files[0], 'input'));
|
||||
|
||||
for (const type of ['dragenter', 'dragover']) {
|
||||
drop.addEventListener(type, event => {
|
||||
const types = [...event.dataTransfer.types];
|
||||
window.__bskTransferFixture.events.push({ type, types });
|
||||
if (!types.includes('Files')) return;
|
||||
event.preventDefault();
|
||||
drop.classList.add('dragging');
|
||||
});
|
||||
}
|
||||
drop.addEventListener('dragleave', () => drop.classList.remove('dragging'));
|
||||
drop.addEventListener('drop', event => {
|
||||
window.__bskTransferFixture.events.push({
|
||||
type: 'drop',
|
||||
types: [...event.dataTransfer.types],
|
||||
fileCount: event.dataTransfer.files.length,
|
||||
});
|
||||
event.preventDefault();
|
||||
drop.classList.remove('dragging');
|
||||
accept(event.dataTransfer.files[0], 'drop');
|
||||
});
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@browser-skill/extension",
|
||||
"version": "0.1.7",
|
||||
"version": "0.2.1",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
|
||||
@@ -254,6 +254,69 @@ describe("ChromiumCdp", () => {
|
||||
expect(cdp.isAttached(7)).toBe(false);
|
||||
});
|
||||
|
||||
it.each([
|
||||
false,
|
||||
true,
|
||||
])("waits for detach before reconnecting a returned tab (detach rejects: %s)", async (rejectDetach) => {
|
||||
const { api } = fakeApi();
|
||||
const attached = new Set<number>();
|
||||
let finishDetach!: () => void;
|
||||
vi.mocked(api.attach).mockImplementation(async ({ tabId }) => {
|
||||
if (attached.has(tabId!)) throw new Error("Another debugger is already attached");
|
||||
attached.add(tabId!);
|
||||
});
|
||||
vi.mocked(api.detach).mockImplementation(async ({ tabId }) => {
|
||||
attached.delete(tabId!);
|
||||
});
|
||||
vi.mocked(api.detach).mockImplementationOnce(
|
||||
({ tabId }) =>
|
||||
new Promise<void>((resolve, reject) => {
|
||||
finishDetach = () => {
|
||||
attached.delete(tabId!);
|
||||
if (rejectDetach) reject(new Error("tab already detached"));
|
||||
else resolve();
|
||||
};
|
||||
}),
|
||||
);
|
||||
const cdp = new ChromiumCdp(api);
|
||||
cdp.trackSessionTab("aa11", 7);
|
||||
await cdp.ensureAttached(7);
|
||||
const returning = cdp.releaseSessionTab("aa11", 7);
|
||||
await vi.waitFor(() => expect(api.detach).toHaveBeenCalledOnce());
|
||||
let duplicateDetachFinished = false;
|
||||
const duplicateDetach = cdp.detach(7).then(() => {
|
||||
duplicateDetachFinished = true;
|
||||
});
|
||||
|
||||
cdp.trackSessionTab("bb22", 7);
|
||||
// Register rejection handlers immediately; the pre-fix driver rejects
|
||||
// both commands while the browser still has the previous attachment.
|
||||
const pending = Promise.allSettled([
|
||||
cdp.send(7, "Runtime.evaluate", { expression: "document.title" }),
|
||||
cdp.send(7, "DOM.getDocument"),
|
||||
]);
|
||||
await cdp.send(8, "DOM.getDocument");
|
||||
const finishedEarly = duplicateDetachFinished;
|
||||
finishDetach();
|
||||
await Promise.all([returning, duplicateDetach]);
|
||||
|
||||
expect(await pending).toEqual([
|
||||
{ status: "fulfilled", value: { ok: true } },
|
||||
{ status: "fulfilled", value: { ok: true } },
|
||||
]);
|
||||
expect(finishedEarly).toBe(false);
|
||||
expect(vi.mocked(api.attach).mock.calls.filter(([target]) => target.tabId === 7)).toHaveLength(
|
||||
2,
|
||||
);
|
||||
expect(cdp.isAttached(7)).toBe(true);
|
||||
expect(cdp.isAttached(8)).toBe(true);
|
||||
await cdp.detachSession("bb22");
|
||||
expect(api.detach).toHaveBeenCalledTimes(2);
|
||||
expect(cdp.isAttached(7)).toBe(false);
|
||||
expect(attached.has(7)).toBe(false);
|
||||
expect(cdp.isAttached(8)).toBe(true);
|
||||
});
|
||||
|
||||
it("detachAll() iterates every cached tab", async () => {
|
||||
const { api } = fakeApi();
|
||||
const cdp = new ChromiumCdp(api);
|
||||
@@ -651,6 +714,155 @@ describe("ChromiumCdp", () => {
|
||||
expect(cdp.networkEntriesSince(33, 0, 50, 1000).entries[0].url).toBeUndefined();
|
||||
});
|
||||
|
||||
it("releases only the returning session's claim on a tab", async () => {
|
||||
const { api } = fakeApi();
|
||||
const cdp = new ChromiumCdp(api);
|
||||
await cdp.ensureAttached(1);
|
||||
await cdp.ensureAttached(2);
|
||||
cdp.trackSessionTab("aa11", 1);
|
||||
cdp.trackSessionTab("bb22", 1);
|
||||
cdp.trackSessionTab("aa11", 2);
|
||||
|
||||
await cdp.releaseSessionTab("unknown", 1);
|
||||
await cdp.releaseSessionTab("aa11", 1);
|
||||
expect(api.detach).not.toHaveBeenCalled();
|
||||
|
||||
await cdp.releaseSessionTab("bb22", 1);
|
||||
await cdp.releaseSessionTab("bb22", 1);
|
||||
expect(api.detach).toHaveBeenCalledExactlyOnceWith({ tabId: 1 });
|
||||
expect(cdp.isAttached(2)).toBe(true);
|
||||
|
||||
await cdp.detachSession("aa11");
|
||||
expect(api.detach).toHaveBeenCalledTimes(2);
|
||||
expect(api.detach).toHaveBeenLastCalledWith({ tabId: 2 });
|
||||
});
|
||||
|
||||
it.each([
|
||||
false,
|
||||
true,
|
||||
])("releases a tab with an attachment in flight (new owner: %s)", async (newOwner) => {
|
||||
const { api } = fakeApi();
|
||||
let finishAttach!: () => void;
|
||||
vi.mocked(api.attach).mockImplementationOnce(
|
||||
() =>
|
||||
new Promise<void>((resolve) => {
|
||||
finishAttach = resolve;
|
||||
}),
|
||||
);
|
||||
const cdp = new ChromiumCdp(api);
|
||||
cdp.trackSessionTab("aa11", 1);
|
||||
const attaching = cdp.ensureAttached(1);
|
||||
const releasing = cdp.releaseSessionTab("aa11", 1);
|
||||
if (newOwner) cdp.trackSessionTab("bb22", 1);
|
||||
finishAttach();
|
||||
await Promise.all([attaching, releasing]);
|
||||
|
||||
expect(cdp.isAttached(1)).toBe(newOwner);
|
||||
expect(api.detach).toHaveBeenCalledTimes(newOwner ? 0 : 1);
|
||||
});
|
||||
|
||||
it("stops accepting dialogs on a returned tab even when another session keeps CDP attached", async () => {
|
||||
const { api, onEvent } = fakeApi();
|
||||
let controlled = true;
|
||||
const cdp = new ChromiumCdp(api, { shouldAutoAcceptDialog: () => controlled });
|
||||
cdp.trackSessionTab("aa11", 1);
|
||||
cdp.trackSessionTab("bb22", 1);
|
||||
await cdp.ensureAttached(1);
|
||||
onEvent.fire({ tabId: 1 }, "Page.javascriptDialogOpening", {
|
||||
type: "confirm",
|
||||
message: "before return",
|
||||
});
|
||||
await vi.waitFor(() => expect(cdp.dialogsSince(1, 0)).toHaveLength(1));
|
||||
vi.mocked(api.sendCommand).mockClear();
|
||||
|
||||
controlled = false;
|
||||
await cdp.releaseSessionTab("aa11", 1);
|
||||
expect(cdp.isAttached(1)).toBe(true);
|
||||
onEvent.fire({ tabId: 1 }, "Page.javascriptDialogOpening", {
|
||||
type: "confirm",
|
||||
message: "after return",
|
||||
});
|
||||
await Promise.resolve();
|
||||
expect(api.sendCommand).not.toHaveBeenCalled();
|
||||
expect(cdp.dialogsSince(1, 0)).toHaveLength(1);
|
||||
|
||||
controlled = true;
|
||||
onEvent.fire({ tabId: 1 }, "Page.javascriptDialogOpening", {
|
||||
type: "alert",
|
||||
message: "borrowed again",
|
||||
});
|
||||
await vi.waitFor(() => expect(cdp.dialogsSince(1, 0)).toHaveLength(2));
|
||||
expect(api.sendCommand).toHaveBeenCalledExactlyOnceWith(
|
||||
{ tabId: 1 },
|
||||
"Page.handleJavaScriptDialog",
|
||||
{ accept: true },
|
||||
);
|
||||
});
|
||||
|
||||
it("ignores dialog events after detach, including a pending eligibility check", async () => {
|
||||
const { api, onEvent } = fakeApi();
|
||||
let resolveEligibility!: (value: boolean) => void;
|
||||
const shouldAutoAcceptDialog = vi.fn(
|
||||
() =>
|
||||
new Promise<boolean>((resolve) => {
|
||||
resolveEligibility = resolve;
|
||||
}),
|
||||
);
|
||||
const cdp = new ChromiumCdp(api, { shouldAutoAcceptDialog });
|
||||
await cdp.ensureAttached(1);
|
||||
onEvent.fire({ tabId: 1 }, "Page.javascriptDialogOpening", { type: "alert" });
|
||||
await cdp.detach(1);
|
||||
resolveEligibility(true);
|
||||
await Promise.resolve();
|
||||
onEvent.fire({ tabId: 1 }, "Page.javascriptDialogOpening", { type: "alert" });
|
||||
|
||||
expect(shouldAutoAcceptDialog).toHaveBeenCalledOnce();
|
||||
expect(api.sendCommand).not.toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
"Page.handleJavaScriptDialog",
|
||||
expect.anything(),
|
||||
);
|
||||
expect(cdp.dialogsSince(1, 0)).toEqual([]);
|
||||
});
|
||||
|
||||
it("does not accept dialogs when the tab's current scope cannot be determined", async () => {
|
||||
const { api, onEvent } = fakeApi();
|
||||
const shouldAutoAcceptDialog = vi.fn(async () => {
|
||||
throw new Error("tab is gone");
|
||||
});
|
||||
const cdp = new ChromiumCdp(api, { shouldAutoAcceptDialog });
|
||||
await cdp.ensureAttached(1);
|
||||
vi.mocked(api.sendCommand).mockClear();
|
||||
|
||||
onEvent.fire({ tabId: 1 }, "Page.javascriptDialogOpening", { type: "confirm" });
|
||||
await Promise.resolve();
|
||||
|
||||
expect(shouldAutoAcceptDialog).toHaveBeenCalledWith(1);
|
||||
expect(api.sendCommand).not.toHaveBeenCalled();
|
||||
expect(cdp.dialogsSince(1, 0)).toEqual([]);
|
||||
});
|
||||
|
||||
it("does not restore dialog state when an acceptance completes after detach", async () => {
|
||||
const { api, onEvent } = fakeApi();
|
||||
const cdp = new ChromiumCdp(api);
|
||||
await cdp.ensureAttached(1);
|
||||
let finishAccept!: () => void;
|
||||
vi.mocked(api.sendCommand).mockImplementationOnce(
|
||||
() =>
|
||||
new Promise<void>((resolve) => {
|
||||
finishAccept = resolve;
|
||||
}),
|
||||
);
|
||||
|
||||
onEvent.fire({ tabId: 1 }, "Page.javascriptDialogOpening", { type: "alert" });
|
||||
await cdp.detach(1);
|
||||
finishAccept();
|
||||
await Promise.resolve();
|
||||
|
||||
expect(cdp.dialogsSince(1, 0)).toEqual([]);
|
||||
expect(cdp.dialogCursor(1)).toBe(0);
|
||||
});
|
||||
|
||||
it("detachSession only detaches tabs no other session owns", async () => {
|
||||
const { api } = fakeApi();
|
||||
const cdp = new ChromiumCdp(api);
|
||||
|
||||
@@ -165,6 +165,7 @@ export class ChromiumCdp {
|
||||
private readonly api: CdpDebuggerApi;
|
||||
private readonly attachedTabs = new Set<number>();
|
||||
private readonly attachInFlight = new Map<number, Promise<void>>();
|
||||
private readonly detachInFlight = new Map<number, Promise<void>>();
|
||||
private readonly tabOwners = new Map<number, Set<string>>();
|
||||
private readonly dialogBuffers = new Map<number, JavaScriptDialogInfo[]>();
|
||||
private readonly dialogSequences = new Map<number, number>();
|
||||
@@ -182,7 +183,13 @@ export class ChromiumCdp {
|
||||
private networkSubscription: { dispose(): void } | null = null;
|
||||
private frameTargetSubscription: { dispose(): void } | null = null;
|
||||
|
||||
constructor(api: CdpDebuggerApi = chromeDebuggerApi) {
|
||||
constructor(
|
||||
api: CdpDebuggerApi = chromeDebuggerApi,
|
||||
private readonly options: {
|
||||
/** CDP observation alone does not authorize dismissing user dialogs. */
|
||||
shouldAutoAcceptDialog?: (tabId: number) => boolean | Promise<boolean>;
|
||||
} = {},
|
||||
) {
|
||||
this.api = api;
|
||||
this.bindAutoDetach();
|
||||
this.bindDialogHandler();
|
||||
@@ -193,6 +200,10 @@ export class ChromiumCdp {
|
||||
|
||||
/** Attach to `tabId` if we haven't already in this driver. */
|
||||
async ensureAttached(tabId: number): Promise<void> {
|
||||
// Returning a tab clears the cache before Chrome finishes detaching.
|
||||
// New observers must wait before opening the next connection to that tab.
|
||||
const detaching = this.detachInFlight.get(tabId);
|
||||
if (detaching) await detaching;
|
||||
if (this.attachedTabs.has(tabId)) return;
|
||||
const existing = this.attachInFlight.get(tabId);
|
||||
if (existing) {
|
||||
@@ -425,6 +436,11 @@ export class ChromiumCdp {
|
||||
|
||||
/** Detach if attached; never throws. */
|
||||
async detach(tabId: number): Promise<void> {
|
||||
const existing = this.detachInFlight.get(tabId);
|
||||
if (existing) {
|
||||
await existing;
|
||||
return;
|
||||
}
|
||||
this.attachInFlight.delete(tabId);
|
||||
if (!this.attachedTabs.has(tabId)) return;
|
||||
this.attachedTabs.delete(tabId);
|
||||
@@ -432,13 +448,19 @@ export class ChromiumCdp {
|
||||
this.clearConsoleState(tabId);
|
||||
this.clearNetworkState(tabId);
|
||||
this.clearFrameState(tabId);
|
||||
try {
|
||||
await this.api.detach({ tabId });
|
||||
} catch (err) {
|
||||
// Tab may already be gone — Chrome auto-detaches on close. Log
|
||||
// at debug so production builds aren't noisy.
|
||||
console.debug("[bsk cdp] detach failed (likely tab already closed)", err);
|
||||
}
|
||||
const detach = (async () => {
|
||||
try {
|
||||
await this.api.detach({ tabId });
|
||||
} catch (err) {
|
||||
// Tab may already be gone — Chrome auto-detaches on close. Log
|
||||
// at debug so production builds aren't noisy.
|
||||
console.debug("[bsk cdp] detach failed (likely tab already closed)", err);
|
||||
}
|
||||
})().finally(() => {
|
||||
this.detachInFlight.delete(tabId);
|
||||
});
|
||||
this.detachInFlight.set(tabId, detach);
|
||||
await detach;
|
||||
}
|
||||
|
||||
/** True iff `ensureAttached(tabId)` has succeeded since the last detach. */
|
||||
@@ -453,6 +475,17 @@ export class ChromiumCdp {
|
||||
this.tabOwners.set(tabId, owners);
|
||||
}
|
||||
|
||||
/** Release one session's claim, preserving attachments still used by another. */
|
||||
async releaseSessionTab(sessionId: string, tabId: number): Promise<void> {
|
||||
const owners = this.tabOwners.get(tabId);
|
||||
if (!owners?.delete(sessionId) || owners.size > 0) return;
|
||||
this.tabOwners.delete(tabId);
|
||||
// An observation may still be attaching when the tab is returned. Wait
|
||||
// for it so detach cannot miss the attachment or remove a new owner's claim.
|
||||
await this.attachInFlight.get(tabId)?.catch(() => {});
|
||||
if (!this.tabOwners.has(tabId)) await this.detach(tabId);
|
||||
}
|
||||
|
||||
/** Subscribe to all CDP events. Returned disposable removes the listener. */
|
||||
onEvent(handler: (source: CdpDebuggee, method: string, params: unknown) => void): {
|
||||
dispose(): void;
|
||||
@@ -639,13 +672,23 @@ export class ChromiumCdp {
|
||||
}
|
||||
|
||||
private async onJavaScriptDialogOpening(tabId: number, params: unknown): Promise<void> {
|
||||
if (!this.attachedTabs.has(tabId) && !this.attachInFlight.has(tabId)) return;
|
||||
const parsed = parseDialogOpeningParams(params);
|
||||
try {
|
||||
if (
|
||||
this.options.shouldAutoAcceptDialog &&
|
||||
!(await this.options.shouldAutoAcceptDialog(tabId))
|
||||
) {
|
||||
return;
|
||||
}
|
||||
// The tab may have been returned while its current scope was checked.
|
||||
if (!this.attachedTabs.has(tabId) && !this.attachInFlight.has(tabId)) return;
|
||||
const handleParams: { accept: boolean; promptText?: string } = { accept: true };
|
||||
if (parsed.type === "prompt") {
|
||||
handleParams.promptText = parsed.defaultPrompt ?? "";
|
||||
}
|
||||
await this.api.sendCommand({ tabId }, "Page.handleJavaScriptDialog", handleParams);
|
||||
if (!this.attachedTabs.has(tabId) && !this.attachInFlight.has(tabId)) return;
|
||||
const sequence = (this.dialogSequences.get(tabId) ?? 0) + 1;
|
||||
this.dialogSequences.set(tabId, sequence);
|
||||
this.appendDialog(tabId, {
|
||||
@@ -835,15 +878,9 @@ export class ChromiumCdp {
|
||||
|
||||
/** Detach tabs only when no other live session has claimed them. */
|
||||
async detachSession(sessionId: string): Promise<void> {
|
||||
const tabsToDetach: number[] = [];
|
||||
for (const [tabId, owners] of this.tabOwners) {
|
||||
owners.delete(sessionId);
|
||||
if (owners.size === 0) {
|
||||
this.tabOwners.delete(tabId);
|
||||
tabsToDetach.push(tabId);
|
||||
}
|
||||
}
|
||||
await Promise.all(tabsToDetach.map((tabId) => this.detach(tabId)));
|
||||
await Promise.all(
|
||||
Array.from(this.tabOwners.keys(), (tabId) => this.releaseSessionTab(sessionId, tabId)),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -23,7 +23,7 @@ describe("BorrowConfirmationOverlay", () => {
|
||||
id: "req-1",
|
||||
isActiveTab: true,
|
||||
tabTitle: "Example",
|
||||
timeoutMs: 5000,
|
||||
timeoutMs: 60_000,
|
||||
onAllow,
|
||||
onDeny,
|
||||
},
|
||||
@@ -32,7 +32,7 @@ describe("BorrowConfirmationOverlay", () => {
|
||||
);
|
||||
|
||||
expect(screen.getByText("允许借用标签页?")).toBeTruthy();
|
||||
for (let i = 0; i < 5; i++) {
|
||||
for (let i = 0; i < 60; i++) {
|
||||
await act(async () => {
|
||||
await vi.advanceTimersByTimeAsync(1000);
|
||||
});
|
||||
@@ -66,7 +66,7 @@ describe("BorrowConfirmationOverlay", () => {
|
||||
id: "req-fallback",
|
||||
isActiveTab: true,
|
||||
tabTitle: "Example",
|
||||
timeoutMs: 5000,
|
||||
timeoutMs: 60_000,
|
||||
onAllow,
|
||||
onDeny,
|
||||
},
|
||||
@@ -74,7 +74,7 @@ describe("BorrowConfirmationOverlay", () => {
|
||||
/>,
|
||||
);
|
||||
|
||||
for (let i = 0; i < 5; i++) {
|
||||
for (let i = 0; i < 60; i++) {
|
||||
await act(async () => {
|
||||
await vi.advanceTimersByTimeAsync(1000);
|
||||
});
|
||||
|
||||
@@ -45,10 +45,32 @@ function portHarness() {
|
||||
|
||||
describe("RecordFrameAgent", () => {
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals();
|
||||
document.documentElement.removeAttribute(RECORD_DOCUMENT_ATTRIBUTE);
|
||||
document.body.replaceChildren();
|
||||
});
|
||||
|
||||
it("starts when crypto.randomUUID is unavailable", async () => {
|
||||
const harness = portHarness();
|
||||
const getRandomValues = vi.fn((bytes: Uint8Array) => {
|
||||
bytes.fill(0x11);
|
||||
return bytes;
|
||||
});
|
||||
vi.stubGlobal("crypto", { getRandomValues });
|
||||
vi.stubGlobal("chrome", {
|
||||
runtime: {
|
||||
connect: vi.fn(() => harness.port),
|
||||
sendMessage: vi.fn(),
|
||||
},
|
||||
});
|
||||
const agent = new RecordFrameAgent();
|
||||
|
||||
await expect(
|
||||
agent.start({ type: RECORD_FRAME_START, requestId: "rec-http", startedAtMs: 10 }),
|
||||
).resolves.toEqual({ ok: true });
|
||||
expect(getRandomValues).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("keeps a failed stop retryable and flushes the final dirty fill", async () => {
|
||||
const harness = portHarness();
|
||||
const sendMessage = vi
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { createRandomUuid } from "@/lib/random-uuid";
|
||||
import {
|
||||
isAcceptedRecordStepAck,
|
||||
RECORD_STEP,
|
||||
@@ -18,7 +19,7 @@ export class RecordStepDelivery {
|
||||
constructor(
|
||||
requestId: string,
|
||||
send: SendRecordStepMessage = (message) => chrome.runtime.sendMessage(message),
|
||||
producerId: string = crypto.randomUUID(),
|
||||
producerId: string = createRandomUuid(),
|
||||
) {
|
||||
this.#requestId = requestId;
|
||||
this.#producerId = producerId;
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { createRandomUuid } from "@/lib/random-uuid";
|
||||
import {
|
||||
markRecordingDocument,
|
||||
type RecordingDocumentMarker,
|
||||
@@ -67,7 +68,7 @@ export class RecordFrameAgent {
|
||||
async #start(
|
||||
message: RecordFrameStartMessage,
|
||||
): Promise<{ ok: true } | { ok: false; error: string }> {
|
||||
const producerId = crypto.randomUUID();
|
||||
const producerId = createRandomUuid();
|
||||
const root = await waitForDocumentElement();
|
||||
if (this.#disposed) return { ok: false, error: "recording document is disposed" };
|
||||
const marker = markRecordingDocument(producerId, root);
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
import { i18n } from "@browser-skill/i18n";
|
||||
import { ChromiumCdp } from "@/browser-driver/chromium-cdp";
|
||||
import { ConnectionController } from "@/lib/connection-controller";
|
||||
import { DiagnosticLogger } from "@/lib/diagnostics";
|
||||
import { startHeartbeat } from "@/lib/heartbeat";
|
||||
import {
|
||||
getConnectionEnabled,
|
||||
@@ -26,7 +25,7 @@ import { recordFrameCoordinator } from "@/lib/recording/frame-coordinator";
|
||||
import { attachSessionsLiveFlag } from "@/lib/sessions-live-flag";
|
||||
import { createDisconnectCleanup } from "@/session-manager/disconnect-cleanup";
|
||||
import { attachSessionEventHandler } from "@/session-manager/event-handler";
|
||||
import { SessionManager } from "@/session-manager/manager";
|
||||
import { isAgentControlledTab, SessionManager } from "@/session-manager/manager";
|
||||
import {
|
||||
attachBorrowNotificationButtonHandler,
|
||||
attachBorrowNotificationClickHandler,
|
||||
@@ -42,26 +41,22 @@ import {
|
||||
attachRecordStepListener,
|
||||
type RecordRuntimeDeps,
|
||||
} from "@/tools/record";
|
||||
import { chromeTabsApi } from "@/tools/shared";
|
||||
import { chromeTabMutationApi } from "@/tools/tabs";
|
||||
import { detectBrowserMeta } from "@/transport/handshake";
|
||||
import type { Transport } from "@/transport/transport";
|
||||
import { WSTransport } from "@/transport/ws-transport";
|
||||
|
||||
export default defineBackground(() => {
|
||||
const diagnostics = new DiagnosticLogger();
|
||||
diagnostics.record("extension.worker.started", {
|
||||
diagnostic_build: "lumi-session-loss-v2",
|
||||
extension_version: __BSK_EXT_VERSION__,
|
||||
extension_runtime_id: chrome.runtime.id,
|
||||
daemon_ws_url: __BSK_DAEMON_WS_URL__,
|
||||
const controller = new ConnectionController();
|
||||
const transport = new WSTransport({ url: __BSK_DAEMON_WS_URL__ });
|
||||
const sessions = new SessionManager();
|
||||
const cdp = new ChromiumCdp(undefined, {
|
||||
shouldAutoAcceptDialog: async (tabId) => {
|
||||
const tab = await chrome.tabs.get(tabId);
|
||||
return sessions.findByWindowId(tab.windowId) !== null;
|
||||
},
|
||||
});
|
||||
const controller = new ConnectionController(diagnostics.sink);
|
||||
const transport = new WSTransport({
|
||||
url: __BSK_DAEMON_WS_URL__,
|
||||
diagnostics: diagnostics.sink,
|
||||
});
|
||||
diagnostics.bindTransport((frame) => transport.send(frame));
|
||||
const sessions = new SessionManager({ diagnostics: diagnostics.sink });
|
||||
const cdp = new ChromiumCdp();
|
||||
const sessionsLive = attachSessionsLiveFlag({ manager: sessions });
|
||||
let overlayGeneration = 0;
|
||||
const controlModes = new Map<string, OverlayMode>();
|
||||
@@ -92,6 +87,24 @@ export default defineBackground(() => {
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Authoritative overlay state for a specific tab. Agent Window tabs are
|
||||
* free by default; only tabs explicitly claimed through session startup,
|
||||
* `tab_create`, or `tab_borrow` receive the control overlay.
|
||||
*/
|
||||
function overlayStateForTab(tabId?: number, windowId?: number): OverlayAgentStateMessage {
|
||||
if (typeof tabId === "number" && typeof windowId === "number") {
|
||||
const ctx = sessions.findByWindowId(windowId);
|
||||
if (ctx && isAgentControlledTab(ctx, tabId)) return overlayStateForWindow(windowId);
|
||||
}
|
||||
return {
|
||||
type: OVERLAY_AGENT_STATE,
|
||||
sessionId: null,
|
||||
mode: "hidden",
|
||||
generation: overlayGeneration,
|
||||
};
|
||||
}
|
||||
|
||||
async function pushOverlayStateToTab(
|
||||
tabId: number,
|
||||
state: OverlayAgentStateMessage,
|
||||
@@ -103,13 +116,18 @@ export default defineBackground(() => {
|
||||
}
|
||||
}
|
||||
|
||||
async function pushOverlayStateForTab(tabId: number, windowId?: number): Promise<void> {
|
||||
const state = overlayStateForTab(tabId, windowId);
|
||||
await pushOverlayStateToTab(tabId, state);
|
||||
}
|
||||
|
||||
async function pushOverlayStateForWindow(windowId: number): Promise<void> {
|
||||
const state = overlayStateForWindow(windowId);
|
||||
const tabs = await chrome.tabs.query({ windowId });
|
||||
await Promise.all(
|
||||
tabs.map((tab) =>
|
||||
typeof tab.id === "number" ? pushOverlayStateToTab(tab.id, state) : Promise.resolve(),
|
||||
),
|
||||
tabs.map((tab) => {
|
||||
if (typeof tab.id !== "number") return Promise.resolve();
|
||||
return pushOverlayStateForTab(tab.id, windowId);
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
@@ -121,9 +139,6 @@ export default defineBackground(() => {
|
||||
}
|
||||
|
||||
function onOverlaySessionStateChanged(): void {
|
||||
diagnostics.record("background.sessions.changed", {
|
||||
live_session_ids: sessions.list().map((ctx) => ctx.sessionId),
|
||||
});
|
||||
void sessionsLive.syncFromManager();
|
||||
const liveSessionIds = new Set(sessions.list().map((ctx) => ctx.sessionId));
|
||||
for (const sessionId of controlModes.keys()) {
|
||||
@@ -151,17 +166,33 @@ export default defineBackground(() => {
|
||||
if (typeof tab.windowId !== "number") return;
|
||||
pushOverlayStateForAgentWindow(tab.windowId);
|
||||
});
|
||||
// Newly observed tabs are free unless the agent's own creation path has
|
||||
// already claimed their concrete id. This listener only renders that
|
||||
// state; it never infers or mutates ownership from event ordering.
|
||||
chrome.tabs.onCreated.addListener((tab) => {
|
||||
if (typeof tab.windowId !== "number" || typeof tab.id !== "number") return;
|
||||
if (!sessions.findByWindowId(tab.windowId)) return;
|
||||
void pushOverlayStateForTab(tab.id, tab.windowId);
|
||||
});
|
||||
chrome.tabs.onRemoved.addListener((tabId, removeInfo) => {
|
||||
sessions.forgetClosedTab(tabId, { isWindowClosing: removeInfo.isWindowClosing });
|
||||
});
|
||||
// Re-sync the storage.session flag on SW startup so a previous SW's
|
||||
// stale `true` does not keep waking us on every page load until the
|
||||
// first mutation (review M4/M5 round 3 m-R3-1).
|
||||
void sessionsLive.refresh();
|
||||
const cleanupAfterDisconnect = createDisconnectCleanup({
|
||||
manager: sessions,
|
||||
sessionStopDeps: { cdp },
|
||||
// Same contract as the dispatcher's `tool.session_stop`: without these
|
||||
// deps the agent-tab cleanup and the window-release path are dead code.
|
||||
sessionStopDeps: {
|
||||
cdp,
|
||||
tabManagement: { tabs: chromeTabMutationApi },
|
||||
tabsQuery: chromeTabsApi,
|
||||
},
|
||||
onSessionsChanged: () => {
|
||||
void sessionsLive.syncFromManager();
|
||||
},
|
||||
diagnostics: diagnostics.sink,
|
||||
});
|
||||
const recordDeps = {
|
||||
tabsApi: chrome.tabs,
|
||||
@@ -192,6 +223,9 @@ export default defineBackground(() => {
|
||||
recording: recordDeps,
|
||||
onSessionsChanged: onOverlaySessionStateChanged,
|
||||
onBrowserControlResumed,
|
||||
onAgentTabClaimed: (tabId, windowId) => {
|
||||
void pushOverlayStateForTab(tabId, windowId);
|
||||
},
|
||||
approveBorrow: (ctx) =>
|
||||
requestBorrowConfirmation(ctx.tabId, {
|
||||
...(ctx.signal !== undefined ? { signal: ctx.signal } : {}),
|
||||
@@ -209,7 +243,6 @@ export default defineBackground(() => {
|
||||
title: i18n.t("helpRequest.notificationTitle", { ns: "extension" }),
|
||||
body: "",
|
||||
}),
|
||||
diagnostics: diagnostics.sink,
|
||||
});
|
||||
dispatcher.start();
|
||||
if (typeof chrome.notifications?.onClicked?.addListener === "function") {
|
||||
@@ -242,7 +275,6 @@ export default defineBackground(() => {
|
||||
transport,
|
||||
cdp,
|
||||
onSessionsChanged: onOverlaySessionStateChanged,
|
||||
diagnostics: diagnostics.sink,
|
||||
});
|
||||
|
||||
// MV3 service worker keepalive + reconnect supervisor (review M4/M5
|
||||
@@ -254,7 +286,6 @@ export default defineBackground(() => {
|
||||
startKeepalive({
|
||||
transport,
|
||||
shouldConnect: () => controller.isConnectionEnabled,
|
||||
diagnostics: diagnostics.sink,
|
||||
});
|
||||
|
||||
// Application-level heartbeat (Chrome 116+): while the post-handshake
|
||||
@@ -268,19 +299,6 @@ export default defineBackground(() => {
|
||||
controller.subscribe((snap) =>
|
||||
cb(snap.state === "connected" || snap.state === "version_skew"),
|
||||
),
|
||||
diagnostics: diagnostics.sink,
|
||||
});
|
||||
|
||||
controller.subscribe((snapshot) => {
|
||||
const ready = snapshot.state === "connected" || snapshot.state === "version_skew";
|
||||
diagnostics.setTransportReady(ready);
|
||||
diagnostics.record("background.controller.snapshot", {
|
||||
state: snapshot.state,
|
||||
connection_enabled: snapshot.connectionEnabled,
|
||||
instance_id: snapshot.instanceId,
|
||||
last_error: snapshot.lastError,
|
||||
live_session_ids: sessions.list().map((ctx) => ctx.sessionId),
|
||||
});
|
||||
});
|
||||
|
||||
// Wake-driven reconnect (best effort). An MV3 service worker is killed
|
||||
@@ -289,12 +307,7 @@ export default defineBackground(() => {
|
||||
// the worker and let us reconnect immediately instead of waiting for
|
||||
// the next 30s alarm tick. They only help when the daemon is actually
|
||||
// running; a cold daemon is (re)spawned by the next `bsk` command.
|
||||
const reconnectIfNeeded = (source: string) => {
|
||||
diagnostics.record("background.wake_reconnect.requested", {
|
||||
source,
|
||||
controller_enabled: controller.isConnectionEnabled,
|
||||
transport_state: transport.state,
|
||||
});
|
||||
const reconnectIfNeeded = () => {
|
||||
if (!controller.isConnectionEnabled) return;
|
||||
if (transport.state === "connected") return;
|
||||
void transport.connect().catch((err) => {
|
||||
@@ -302,14 +315,13 @@ export default defineBackground(() => {
|
||||
});
|
||||
};
|
||||
if (typeof chrome.runtime?.onStartup?.addListener === "function") {
|
||||
chrome.runtime.onStartup.addListener(() => reconnectIfNeeded("runtime.onStartup"));
|
||||
chrome.runtime.onStartup.addListener(reconnectIfNeeded);
|
||||
}
|
||||
if (typeof chrome.idle?.onStateChanged?.addListener === "function") {
|
||||
chrome.idle.onStateChanged.addListener((state) => {
|
||||
diagnostics.record("chrome.idle.state_changed", { state });
|
||||
// "active" fires when the user returns from idle/locked — the most
|
||||
// reliable "machine just woke" signal we get.
|
||||
if (state === "active") reconnectIfNeeded("idle.active");
|
||||
if (state === "active") reconnectIfNeeded();
|
||||
});
|
||||
// Treat >60s of no input as idle so the active transition is timely.
|
||||
chrome.idle.setDetectionInterval?.(60);
|
||||
@@ -317,21 +329,14 @@ export default defineBackground(() => {
|
||||
|
||||
void (async () => {
|
||||
const connectionEnabled = await getConnectionEnabled();
|
||||
diagnostics.record("background.controller.attach_started", { connectionEnabled });
|
||||
await controller.attach(transport, detectBrowserMeta(), connectionEnabled, {
|
||||
beforeDisconnect: async () => {
|
||||
diagnostics.record("background.before_disconnect_cleanup", {
|
||||
live_session_ids: sessions.list().map((ctx) => ctx.sessionId),
|
||||
});
|
||||
const report = await cleanupAfterDisconnect();
|
||||
if (report.failures.length > 0) {
|
||||
console.warn("[browser-skill] session cleanup before disconnect was incomplete", report);
|
||||
}
|
||||
},
|
||||
onDisconnected: async () => {
|
||||
diagnostics.record("background.after_disconnect_cleanup", {
|
||||
live_session_ids: sessions.list().map((ctx) => ctx.sessionId),
|
||||
});
|
||||
const report = await cleanupAfterDisconnect();
|
||||
if (report.failures.length > 0) {
|
||||
console.warn("[browser-skill] session cleanup after disconnect was incomplete", report);
|
||||
@@ -339,7 +344,6 @@ export default defineBackground(() => {
|
||||
},
|
||||
});
|
||||
})().catch((err) => {
|
||||
diagnostics.record("background.controller.attach_failed", { error: err });
|
||||
console.error("[browser-skill] controller failed to attach", err);
|
||||
});
|
||||
|
||||
@@ -355,7 +359,11 @@ export default defineBackground(() => {
|
||||
}
|
||||
|
||||
if (msg.kind === OVERLAY_MSG_READY) {
|
||||
sendResponse(overlayStateForWindow(sender.tab?.windowId));
|
||||
// Resolve from explicit tab ownership before sending so free tabs never
|
||||
// flash the control mask.
|
||||
if (typeof sender.tab?.id === "number") {
|
||||
void pushOverlayStateForTab(sender.tab.id, sender.tab.windowId);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -420,15 +428,6 @@ export default defineBackground(() => {
|
||||
dbg.__bhDispatcher = dispatcher;
|
||||
}
|
||||
|
||||
if (typeof chrome.runtime?.onSuspend?.addListener === "function") {
|
||||
chrome.runtime.onSuspend.addListener(() => {
|
||||
diagnostics.record("extension.worker.suspending", {
|
||||
live_session_ids: sessions.list().map((ctx) => ctx.sessionId),
|
||||
transport_state: transport.state,
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
console.info("[browser-skill] background worker initialised");
|
||||
});
|
||||
|
||||
|
||||
@@ -122,8 +122,8 @@ describe("App", () => {
|
||||
|
||||
expect(screen.queryByText(/^扩展 v/)).toBeNull();
|
||||
expect(screen.queryByText(/^daemon v/)).toBeNull();
|
||||
expect(screen.getByTitle("扩展版本").textContent).toBe(EXTENSION_VERSION);
|
||||
expect(screen.getByTitle("bsk 版本").textContent).toBe(mockDaemonVersion);
|
||||
expect(screen.getByTitle("扩展版本").textContent).toBe(`Ext ${EXTENSION_VERSION}`);
|
||||
expect(screen.getByTitle("bsk 版本").textContent).toBe(`CLI ${mockDaemonVersion}`);
|
||||
expect(screen.getByText("03c3e47f")).toBeTruthy();
|
||||
|
||||
const copyButton = screen.getByRole("button", { name: "复制实例 ID" });
|
||||
|
||||
@@ -267,9 +267,9 @@ export function App() {
|
||||
data-slot="popup-meta"
|
||||
>
|
||||
<div className="flex shrink-0 items-center gap-1">
|
||||
<span title={t("popup.extensionVersionHint")}>{extensionVersion}</span>
|
||||
<span title={t("popup.daemonVersionHint")}>CLI {daemonVersion}</span>
|
||||
<span aria-hidden>/</span>
|
||||
<span title={t("popup.daemonVersionHint")}>{daemonVersion}</span>
|
||||
<span title={t("popup.extensionVersionHint")}>Ext {extensionVersion}</span>
|
||||
</div>
|
||||
<div className="flex min-w-0 items-center justify-end gap-1">
|
||||
<span className="shrink-0">{t("popup.instanceTitle")}</span>
|
||||
|
||||
@@ -2,7 +2,7 @@ import { describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
CAPTURE_SUPPRESS,
|
||||
type CaptureSuppressMessage,
|
||||
withOverlaysHiddenForCapture,
|
||||
withExtensionOverlayHidden,
|
||||
} from "../capture-suppress-bridge";
|
||||
|
||||
function recordingSendToTab(events: string[]) {
|
||||
@@ -12,12 +12,12 @@ function recordingSendToTab(events: string[]) {
|
||||
});
|
||||
}
|
||||
|
||||
describe("withOverlaysHiddenForCapture", () => {
|
||||
describe("withExtensionOverlayHidden", () => {
|
||||
it("sends begin → fn → end and returns fn's result", async () => {
|
||||
const events: string[] = [];
|
||||
const sendToTab = recordingSendToTab(events);
|
||||
|
||||
const result = await withOverlaysHiddenForCapture(
|
||||
const result = await withExtensionOverlayHidden(
|
||||
7,
|
||||
async () => {
|
||||
events.push("capture");
|
||||
@@ -37,7 +37,7 @@ describe("withOverlaysHiddenForCapture", () => {
|
||||
const sendToTab = recordingSendToTab(events);
|
||||
|
||||
await expect(
|
||||
withOverlaysHiddenForCapture(
|
||||
withExtensionOverlayHidden(
|
||||
7,
|
||||
async () => {
|
||||
events.push("capture");
|
||||
@@ -55,7 +55,7 @@ describe("withOverlaysHiddenForCapture", () => {
|
||||
});
|
||||
const fn = vi.fn(async () => "shot");
|
||||
|
||||
const result = await withOverlaysHiddenForCapture(7, fn, sendToTab);
|
||||
const result = await withExtensionOverlayHidden(7, fn, sendToTab);
|
||||
|
||||
expect(result).toBe("shot");
|
||||
expect(fn).toHaveBeenCalledTimes(1);
|
||||
@@ -69,7 +69,7 @@ describe("withOverlaysHiddenForCapture", () => {
|
||||
return { type: CAPTURE_SUPPRESS, ok: true };
|
||||
});
|
||||
|
||||
const result = await withOverlaysHiddenForCapture(7, async () => "shot", sendToTab);
|
||||
const result = await withExtensionOverlayHidden(7, async () => "shot", sendToTab);
|
||||
|
||||
expect(result).toBe("shot");
|
||||
expect(sendToTab).toHaveBeenCalledTimes(2);
|
||||
|
||||
@@ -1,85 +0,0 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { DiagnosticLogger } from "../diagnostics";
|
||||
|
||||
class MemoryStorage {
|
||||
value: Record<string, unknown> = {};
|
||||
|
||||
async get(key: string): Promise<Record<string, unknown>> {
|
||||
return { [key]: this.value[key] };
|
||||
}
|
||||
|
||||
async set(items: Record<string, unknown>): Promise<void> {
|
||||
this.value = { ...this.value, ...items };
|
||||
}
|
||||
}
|
||||
|
||||
describe("DiagnosticLogger", () => {
|
||||
it("persists evidence before transport readiness and flushes it after reconnect", async () => {
|
||||
const storage = new MemoryStorage();
|
||||
const send = vi.fn();
|
||||
const logger = new DiagnosticLogger(storage);
|
||||
logger.bindTransport(send);
|
||||
|
||||
logger.record("transport.socket.closed", { code: 1006 });
|
||||
await logger.flush();
|
||||
expect(send).not.toHaveBeenCalled();
|
||||
|
||||
logger.setTransportReady(true);
|
||||
await logger.flush();
|
||||
|
||||
expect(send).toHaveBeenCalledOnce();
|
||||
expect(send.mock.calls[0]?.[0]).toMatchObject({
|
||||
event: "browser.diagnostic",
|
||||
payload: {
|
||||
schema_version: 1,
|
||||
entries: [
|
||||
{
|
||||
event: "transport.socket.closed",
|
||||
fields: { code: 1006 },
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
expect(storage.value.bsk_diagnostic_events_v1).toEqual([]);
|
||||
});
|
||||
|
||||
it("keeps persisted evidence when the transport rejects the flush", async () => {
|
||||
const storage = new MemoryStorage();
|
||||
const logger = new DiagnosticLogger(storage);
|
||||
logger.bindTransport(() => {
|
||||
throw new Error("socket closed");
|
||||
});
|
||||
logger.record("session.stop.requested", { session_id: "abcd" });
|
||||
|
||||
logger.setTransportReady(true);
|
||||
await logger.flush();
|
||||
|
||||
expect(storage.value.bsk_diagnostic_events_v1).toMatchObject([
|
||||
{ event: "session.stop.requested", fields: { session_id: "abcd" } },
|
||||
]);
|
||||
});
|
||||
|
||||
it("does not resend an acknowledged event when send records a new event", async () => {
|
||||
const storage = new MemoryStorage();
|
||||
const batches: Array<{ payload?: { entries?: Array<{ event: string }> } }> = [];
|
||||
const logger = new DiagnosticLogger(storage);
|
||||
logger.record("dispatcher.rpc.received", { rpc_id: "one" });
|
||||
await logger.flush();
|
||||
logger.bindTransport((frame) => {
|
||||
batches.push(frame as { payload?: { entries?: Array<{ event: string }> } });
|
||||
if (batches.length === 1) {
|
||||
logger.record("dispatcher.rpc.response_sent", { rpc_id: "one" });
|
||||
}
|
||||
});
|
||||
|
||||
logger.setTransportReady(true);
|
||||
await logger.flush();
|
||||
await logger.flush();
|
||||
|
||||
expect(batches.map((batch) => batch.payload?.entries?.map((entry) => entry.event))).toEqual([
|
||||
["dispatcher.rpc.received"],
|
||||
["dispatcher.rpc.response_sent"],
|
||||
]);
|
||||
expect(storage.value.bsk_diagnostic_events_v1).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,30 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { createRandomUuid } from "../random-uuid";
|
||||
|
||||
describe("createRandomUuid", () => {
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
it("uses crypto.randomUUID when it is available", () => {
|
||||
const randomUUID = vi.fn(() => "00000000-0000-4000-8000-000000000000");
|
||||
vi.stubGlobal("crypto", { randomUUID, getRandomValues: vi.fn() });
|
||||
|
||||
expect(createRandomUuid()).toBe("00000000-0000-4000-8000-000000000000");
|
||||
expect(randomUUID).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("falls back to getRandomValues and sets UUID v4 bits", () => {
|
||||
const bytes = new Uint8Array(16).fill(0xff);
|
||||
const getRandomValues = vi.fn((target: Uint8Array) => {
|
||||
target.set(bytes);
|
||||
return target;
|
||||
});
|
||||
vi.stubGlobal("crypto", { getRandomValues });
|
||||
|
||||
const uuid = createRandomUuid();
|
||||
|
||||
expect(uuid).toBe("ffffffff-ffff-4fff-bfff-ffffffffffff");
|
||||
expect(getRandomValues).toHaveBeenCalledOnce();
|
||||
});
|
||||
});
|
||||
@@ -1,9 +1,9 @@
|
||||
/**
|
||||
* Wire protocol and background-side helper for hiding the extension's
|
||||
* in-page overlay while a screenshot is captured, so captured frames only
|
||||
* contain the page itself.
|
||||
* Wire protocol and background-side helper for temporarily excluding the
|
||||
* extension's in-page overlay. Screenshots use it to avoid captured chrome;
|
||||
* coordinate-driven native input uses it so Chrome hit-tests the page.
|
||||
*
|
||||
* The background wraps every capture in `withOverlaysHiddenForCapture`,
|
||||
* The background wraps each bounded operation in `withExtensionOverlayHidden`,
|
||||
* which sends `begin` (the content script hides the overlay host and only
|
||||
* acks once the compositor has produced an overlay-free frame) and always
|
||||
* follows up with `end`, even when the capture throws. Tabs without the
|
||||
@@ -46,7 +46,7 @@ const defaultSendToTab: CaptureSuppressSendToTab = (tabId, message) =>
|
||||
* (or any other `begin` failure) simply skips suppression — there is no
|
||||
* overlay to hide in that tab.
|
||||
*/
|
||||
export async function withOverlaysHiddenForCapture<T>(
|
||||
export async function withExtensionOverlayHidden<T>(
|
||||
tabId: number,
|
||||
fn: () => Promise<T>,
|
||||
sendToTab: CaptureSuppressSendToTab = defaultSendToTab,
|
||||
|
||||
@@ -6,7 +6,6 @@ import {
|
||||
} from "../transport/handshake";
|
||||
import type { Transport } from "../transport/transport";
|
||||
import type { ConnectionState, HandshakeResult } from "../transport/types";
|
||||
import type { DiagnosticSink } from "./diagnostics";
|
||||
import { getLabel, getOrCreateInstanceId } from "./instance-id";
|
||||
import { compareProtocol, parseProtocolMajor } from "./semver";
|
||||
|
||||
@@ -52,11 +51,6 @@ export class ConnectionController {
|
||||
private handshakeAbort: AbortController | null = null;
|
||||
private handshakeRetryTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
private suppressDisconnectRecovery = false;
|
||||
private readonly diagnostics?: DiagnosticSink;
|
||||
|
||||
constructor(diagnostics?: DiagnosticSink) {
|
||||
this.diagnostics = diagnostics;
|
||||
}
|
||||
|
||||
get isConnectionEnabled(): boolean {
|
||||
return this.connectionEnabled;
|
||||
@@ -95,20 +89,8 @@ export class ConnectionController {
|
||||
this.lifecycleHooks = lifecycleHooks;
|
||||
this.instanceId = await getOrCreateInstanceId();
|
||||
this.label = await getLabel();
|
||||
this.diagnostics?.("controller.attach", {
|
||||
instance_id: this.instanceId,
|
||||
connection_enabled: connectionEnabled,
|
||||
transport_state: transport.state,
|
||||
});
|
||||
|
||||
transport.onConnectionStateChange((s) => {
|
||||
this.diagnostics?.("controller.transport_state", {
|
||||
transport_state: s,
|
||||
controller_state: this.currentState,
|
||||
connection_generation: this.connectionGeneration,
|
||||
suppress_disconnect_recovery: this.suppressDisconnectRecovery,
|
||||
recovery_inflight: this.disconnectRecovery !== null,
|
||||
});
|
||||
if (s === "disconnected") {
|
||||
// An in-flight handshake belongs to the dead connection — cancel it so
|
||||
// a late resolution can never clobber the next attempt.
|
||||
@@ -121,9 +103,6 @@ export class ConnectionController {
|
||||
// handshake bounce); the caller owns the reconnect policy there.
|
||||
this.suppressDisconnectRecovery = false;
|
||||
} else {
|
||||
this.diagnostics?.("controller.disconnect_recovery.requested", {
|
||||
connection_generation: this.connectionGeneration,
|
||||
});
|
||||
void this.recoverFromDisconnect();
|
||||
}
|
||||
}
|
||||
@@ -182,12 +161,6 @@ export class ConnectionController {
|
||||
const generation = ++this.connectionGeneration;
|
||||
const abort = new AbortController();
|
||||
this.handshakeAbort = abort;
|
||||
this.diagnostics?.("controller.handshake.started", {
|
||||
connection_generation: generation,
|
||||
browser_name: browser.name,
|
||||
browser_version: browser.version,
|
||||
instance_id: this.instanceId,
|
||||
});
|
||||
void this.runHandshake(browser, generation, abort.signal);
|
||||
}
|
||||
|
||||
@@ -224,21 +197,11 @@ export class ConnectionController {
|
||||
}
|
||||
this.clearHandshakeRetry();
|
||||
this.lastError = null;
|
||||
this.diagnostics?.("controller.handshake.completed", {
|
||||
connection_generation: generation,
|
||||
verdict: verdict.kind,
|
||||
daemon_version: outcome.result.version,
|
||||
daemon_protocol: outcome.result.protocol_version,
|
||||
});
|
||||
this.setState(verdict.kind);
|
||||
} catch (err) {
|
||||
if (generation !== this.connectionGeneration || signal.aborted || isAbortError(err)) return;
|
||||
this.handshake = null;
|
||||
this.lastError = err instanceof Error ? err.message : String(err);
|
||||
this.diagnostics?.("controller.handshake.failed", {
|
||||
connection_generation: generation,
|
||||
error: err,
|
||||
});
|
||||
// Bounce the half-open socket deliberately; the retry timer below owns
|
||||
// the reconnect, so skip unexpected-loss recovery for this disconnect.
|
||||
this.suppressDisconnectRecovery = true;
|
||||
@@ -265,57 +228,23 @@ export class ConnectionController {
|
||||
}
|
||||
|
||||
private recoverFromDisconnect(): Promise<void> {
|
||||
if (this.disconnectRecovery) {
|
||||
this.diagnostics?.("controller.disconnect_recovery.coalesced", {
|
||||
connection_generation: this.connectionGeneration,
|
||||
});
|
||||
return this.disconnectRecovery;
|
||||
}
|
||||
if (this.disconnectRecovery) return this.disconnectRecovery;
|
||||
const recovery = (async () => {
|
||||
const recoveryGeneration = this.connectionGeneration;
|
||||
this.diagnostics?.("controller.disconnect_recovery.started", {
|
||||
recovery_generation: recoveryGeneration,
|
||||
});
|
||||
// WSTransport schedules its reconnect timer immediately after notifying
|
||||
// state listeners. Yield once, then disconnect explicitly so that timer
|
||||
// is cancelled before local session teardown begins.
|
||||
await Promise.resolve();
|
||||
if (!this.connectionEnabled || !this.transport) {
|
||||
this.diagnostics?.("controller.disconnect_recovery.skipped", {
|
||||
recovery_generation: recoveryGeneration,
|
||||
reason: "disabled_or_missing_transport",
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (!this.connectionEnabled || !this.transport) return;
|
||||
// Another path already reconnected while we yielded — nothing to do.
|
||||
if (this.currentState !== "disconnected") {
|
||||
this.diagnostics?.("controller.disconnect_recovery.skipped", {
|
||||
recovery_generation: recoveryGeneration,
|
||||
reason: "state_already_changed",
|
||||
current_state: this.currentState,
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (this.currentState !== "disconnected") return;
|
||||
await this.transport.disconnect().catch(() => {});
|
||||
try {
|
||||
this.diagnostics?.("controller.disconnect_cleanup.started", {
|
||||
recovery_generation: recoveryGeneration,
|
||||
current_generation: this.connectionGeneration,
|
||||
});
|
||||
await this.lifecycleHooks.onDisconnected?.();
|
||||
this.diagnostics?.("controller.disconnect_cleanup.completed", {
|
||||
recovery_generation: recoveryGeneration,
|
||||
current_generation: this.connectionGeneration,
|
||||
});
|
||||
} catch (err) {
|
||||
console.warn("[browser-skill] session cleanup after disconnect failed", err);
|
||||
}
|
||||
if (!this.connectionEnabled) return;
|
||||
try {
|
||||
this.diagnostics?.("controller.disconnect_reconnect.started", {
|
||||
recovery_generation: recoveryGeneration,
|
||||
current_generation: this.connectionGeneration,
|
||||
});
|
||||
await this.transport.connect();
|
||||
} catch (err) {
|
||||
this.lastError = err instanceof Error ? err.message : String(err);
|
||||
@@ -323,39 +252,22 @@ export class ConnectionController {
|
||||
}
|
||||
})();
|
||||
this.disconnectRecovery = recovery.finally(() => {
|
||||
this.diagnostics?.("controller.disconnect_recovery.completed", {
|
||||
connection_generation: this.connectionGeneration,
|
||||
current_state: this.currentState,
|
||||
});
|
||||
this.disconnectRecovery = null;
|
||||
});
|
||||
return this.disconnectRecovery;
|
||||
}
|
||||
|
||||
private cancelHandshake(): void {
|
||||
const previousGeneration = this.connectionGeneration;
|
||||
this.connectionGeneration += 1;
|
||||
this.diagnostics?.("controller.handshake.cancelled", {
|
||||
previous_generation: previousGeneration,
|
||||
next_generation: this.connectionGeneration,
|
||||
had_abort_controller: this.handshakeAbort !== null,
|
||||
});
|
||||
this.handshakeAbort?.abort();
|
||||
this.handshakeAbort = null;
|
||||
}
|
||||
|
||||
private scheduleHandshakeRetry(browser: { name: string; version: string }): void {
|
||||
if (this.handshakeRetryTimer || !this.connectionEnabled) return;
|
||||
this.diagnostics?.("controller.handshake_retry.scheduled", {
|
||||
delay_ms: HANDSHAKE_RETRY_DELAY_MS,
|
||||
connection_generation: this.connectionGeneration,
|
||||
});
|
||||
this.handshakeRetryTimer = setTimeout(() => {
|
||||
this.handshakeRetryTimer = null;
|
||||
if (!this.connectionEnabled || !this.transport) return;
|
||||
this.diagnostics?.("controller.handshake_retry.timer_fired", {
|
||||
connection_generation: this.connectionGeneration,
|
||||
});
|
||||
void this.transport.connect().catch((err) => {
|
||||
this.lastError = err instanceof Error ? err.message : String(err);
|
||||
this.setState("disconnected");
|
||||
|
||||
@@ -1,178 +0,0 @@
|
||||
import type { ProtocolFrame } from "@/transport/types";
|
||||
|
||||
const STORAGE_KEY = "bsk_diagnostic_events_v1";
|
||||
const MAX_ENTRIES = 500;
|
||||
|
||||
export type DiagnosticFields = Record<string, unknown>;
|
||||
export type DiagnosticSink = (event: string, fields?: DiagnosticFields) => void;
|
||||
|
||||
export interface DiagnosticEntry {
|
||||
id: string;
|
||||
sequence: number;
|
||||
timestamp: string;
|
||||
monotonic_ms: number;
|
||||
worker_boot_id: string;
|
||||
event: string;
|
||||
fields: DiagnosticFields;
|
||||
}
|
||||
|
||||
interface DiagnosticStorage {
|
||||
get(key: string): Promise<Record<string, unknown>>;
|
||||
set(items: Record<string, unknown>): Promise<void>;
|
||||
}
|
||||
|
||||
function defaultStorage(): DiagnosticStorage | null {
|
||||
if (typeof chrome === "undefined" || !chrome.storage?.local) return null;
|
||||
return {
|
||||
get: (key) => chrome.storage.local.get(key),
|
||||
set: (items) => chrome.storage.local.set(items),
|
||||
};
|
||||
}
|
||||
|
||||
function makeBootId(): string {
|
||||
try {
|
||||
return crypto.randomUUID();
|
||||
} catch {
|
||||
return `worker-${Date.now()}-${Math.random().toString(16).slice(2)}`;
|
||||
}
|
||||
}
|
||||
|
||||
function normalise(value: unknown): unknown {
|
||||
if (value instanceof Error) {
|
||||
return { name: value.name, message: value.message, stack: value.stack };
|
||||
}
|
||||
if (typeof value === "bigint") return value.toString();
|
||||
if (Array.isArray(value)) return value.map(normalise);
|
||||
if (value && typeof value === "object") {
|
||||
return Object.fromEntries(
|
||||
Object.entries(value as Record<string, unknown>).map(([key, child]) => [
|
||||
key,
|
||||
normalise(child),
|
||||
]),
|
||||
);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
/**
|
||||
* Persistent diagnostics for the MV3 background worker.
|
||||
*
|
||||
* Events are first written to chrome.storage.local so a worker termination or
|
||||
* dead WebSocket does not erase the evidence. Once the post-handshake link is
|
||||
* ready, pending entries are forwarded to the daemon as one
|
||||
* `browser.diagnostic` event and removed from the local queue.
|
||||
*/
|
||||
export class DiagnosticLogger {
|
||||
readonly workerBootId: string;
|
||||
readonly sink: DiagnosticSink;
|
||||
|
||||
private sequence = 0;
|
||||
private storageQueue: Promise<void> = Promise.resolve();
|
||||
private flushPromise: Promise<void> | null = null;
|
||||
private flushRequested = false;
|
||||
private send: ((frame: ProtocolFrame) => void) | null = null;
|
||||
private transportReady = false;
|
||||
private readonly storage: DiagnosticStorage | null;
|
||||
|
||||
constructor(storage: DiagnosticStorage | null = defaultStorage()) {
|
||||
this.workerBootId = makeBootId();
|
||||
this.storage = storage;
|
||||
this.sink = (event, fields = {}) => this.record(event, fields);
|
||||
}
|
||||
|
||||
bindTransport(send: (frame: ProtocolFrame) => void): void {
|
||||
this.send = send;
|
||||
}
|
||||
|
||||
setTransportReady(ready: boolean): void {
|
||||
this.transportReady = ready;
|
||||
if (ready) void this.flush();
|
||||
}
|
||||
|
||||
record(event: string, fields: DiagnosticFields = {}): void {
|
||||
const sequence = ++this.sequence;
|
||||
const entry: DiagnosticEntry = {
|
||||
id: `${this.workerBootId}:${sequence}`,
|
||||
sequence,
|
||||
timestamp: new Date().toISOString(),
|
||||
monotonic_ms: Math.round(performance.now()),
|
||||
worker_boot_id: this.workerBootId,
|
||||
event,
|
||||
fields: normalise(fields) as DiagnosticFields,
|
||||
};
|
||||
console.info("[bsk diagnostic]", entry);
|
||||
this.storageQueue = this.storageQueue
|
||||
.then(async () => {
|
||||
if (!this.storage) return;
|
||||
const stored = await this.storage.get(STORAGE_KEY);
|
||||
const previous = Array.isArray(stored[STORAGE_KEY])
|
||||
? (stored[STORAGE_KEY] as DiagnosticEntry[])
|
||||
: [];
|
||||
await this.storage.set({
|
||||
[STORAGE_KEY]: [...previous, entry].slice(-MAX_ENTRIES),
|
||||
});
|
||||
})
|
||||
.catch((error) => {
|
||||
console.warn("[bsk diagnostic] failed to persist event", error);
|
||||
});
|
||||
if (this.transportReady) void this.flush();
|
||||
}
|
||||
|
||||
flush(): Promise<void> {
|
||||
if (this.flushPromise) {
|
||||
this.flushRequested = true;
|
||||
return this.flushPromise;
|
||||
}
|
||||
const running = (async () => {
|
||||
await this.storageQueue;
|
||||
if (!this.transportReady || !this.send || !this.storage) return;
|
||||
const stored = await this.storage.get(STORAGE_KEY);
|
||||
const entries = Array.isArray(stored[STORAGE_KEY])
|
||||
? (stored[STORAGE_KEY] as DiagnosticEntry[])
|
||||
: [];
|
||||
if (entries.length === 0) return;
|
||||
try {
|
||||
this.send({
|
||||
event: "browser.diagnostic",
|
||||
payload: {
|
||||
schema_version: 1,
|
||||
worker_boot_id: this.workerBootId,
|
||||
entries,
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
console.warn("[bsk diagnostic] daemon flush failed", error);
|
||||
return;
|
||||
}
|
||||
const sentIds = new Set(entries.map((entry) => entry.id));
|
||||
// Serialize acknowledgement behind every record write that may have
|
||||
// started while send() was in progress. Without this, a concurrent
|
||||
// record can read the pre-ack queue and write an already-sent entry
|
||||
// back into storage, causing the next batch to duplicate it.
|
||||
this.storageQueue = this.storageQueue
|
||||
.then(async () => {
|
||||
const current = await this.storage?.get(STORAGE_KEY);
|
||||
const currentEntries = Array.isArray(current?.[STORAGE_KEY])
|
||||
? (current[STORAGE_KEY] as DiagnosticEntry[])
|
||||
: [];
|
||||
await this.storage?.set({
|
||||
[STORAGE_KEY]: currentEntries.filter((entry) => !sentIds.has(entry.id)),
|
||||
});
|
||||
})
|
||||
.catch((error) => {
|
||||
console.warn("[bsk diagnostic] failed to acknowledge event batch", error);
|
||||
});
|
||||
await this.storageQueue;
|
||||
})().catch((error) => {
|
||||
console.warn("[bsk diagnostic] flush failed", error);
|
||||
});
|
||||
this.flushPromise = running.finally(() => {
|
||||
this.flushPromise = null;
|
||||
if (this.flushRequested && this.transportReady) {
|
||||
this.flushRequested = false;
|
||||
void this.flush();
|
||||
}
|
||||
});
|
||||
return this.flushPromise;
|
||||
}
|
||||
}
|
||||
@@ -21,7 +21,6 @@
|
||||
* after the handshake has landed.
|
||||
*/
|
||||
|
||||
import type { DiagnosticSink } from "@/lib/diagnostics";
|
||||
import type { ProtocolFrame } from "@/transport/types";
|
||||
|
||||
export const HEARTBEAT_EVENT = "system.heartbeat";
|
||||
@@ -47,7 +46,6 @@ export interface HeartbeatOptions {
|
||||
intervalMs?: number;
|
||||
/** Inject fake timers (tests). */
|
||||
timers?: HeartbeatTimers;
|
||||
diagnostics?: DiagnosticSink;
|
||||
}
|
||||
|
||||
export interface HeartbeatHandle {
|
||||
@@ -76,9 +74,7 @@ export function startHeartbeat(options: HeartbeatOptions): HeartbeatHandle {
|
||||
const beat = () => {
|
||||
try {
|
||||
options.send({ event: HEARTBEAT_EVENT, payload: {} });
|
||||
options.diagnostics?.("heartbeat.sent");
|
||||
} catch (err) {
|
||||
options.diagnostics?.("heartbeat.send_failed", { error: err });
|
||||
// The socket raced a close between the state change and this
|
||||
// tick. The transport's own reconnect path recovers; swallow so
|
||||
// the interval keeps running (it is cleared on the disconnect
|
||||
@@ -91,14 +87,12 @@ export function startHeartbeat(options: HeartbeatOptions): HeartbeatHandle {
|
||||
if (timer !== null) {
|
||||
timers.clearInterval(timer);
|
||||
timer = null;
|
||||
options.diagnostics?.("heartbeat.timer.stopped");
|
||||
}
|
||||
};
|
||||
|
||||
const start = () => {
|
||||
if (timer !== null) return;
|
||||
timer = timers.setInterval(beat, intervalMs);
|
||||
options.diagnostics?.("heartbeat.timer.started", { interval_ms: intervalMs });
|
||||
};
|
||||
|
||||
const unsubscribe = options.onActiveChange((active) => {
|
||||
|
||||
@@ -20,7 +20,6 @@
|
||||
* SW idle threshold.
|
||||
*/
|
||||
|
||||
import type { DiagnosticSink } from "@/lib/diagnostics";
|
||||
import type { Transport } from "@/transport/transport";
|
||||
|
||||
export const KEEPALIVE_ALARM_NAME = "bh-keepalive";
|
||||
@@ -60,7 +59,6 @@ export interface KeepaliveOptions {
|
||||
alarmName?: string;
|
||||
/** Override the alarm period (tests). */
|
||||
periodMin?: number;
|
||||
diagnostics?: DiagnosticSink;
|
||||
}
|
||||
|
||||
export interface KeepaliveHandle {
|
||||
@@ -81,24 +79,11 @@ export function startKeepalive(options: KeepaliveOptions): KeepaliveHandle {
|
||||
const period = options.periodMin ?? KEEPALIVE_PERIOD_MIN;
|
||||
|
||||
const tick = async () => {
|
||||
if (options.shouldConnect && !options.shouldConnect()) {
|
||||
options.diagnostics?.("keepalive.tick.skipped", { reason: "connection_disabled" });
|
||||
return;
|
||||
}
|
||||
if (options.transport.state === "connected") {
|
||||
options.diagnostics?.("keepalive.tick.healthy", { transport_state: "connected" });
|
||||
return;
|
||||
}
|
||||
options.diagnostics?.("keepalive.reconnect.started", {
|
||||
transport_state: options.transport.state,
|
||||
});
|
||||
if (options.shouldConnect && !options.shouldConnect()) return;
|
||||
if (options.transport.state === "connected") return;
|
||||
try {
|
||||
await options.transport.connect();
|
||||
options.diagnostics?.("keepalive.reconnect.completed", {
|
||||
transport_state: options.transport.state,
|
||||
});
|
||||
} catch (err) {
|
||||
options.diagnostics?.("keepalive.reconnect.failed", { error: err });
|
||||
console.debug("[bsk keepalive] connect attempt failed", err);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -3,8 +3,10 @@
|
||||
* between the content-script control overlay and the background SW.
|
||||
*
|
||||
* Content script → background:
|
||||
* - `{ kind: "overlay.ready" }` → background replies with the authoritative
|
||||
* overlay state for the sender's window.
|
||||
* - `{ kind: "overlay.ready" }` → background decides the per-tab state first
|
||||
* (see overlayStateForTab) and pushes the authoritative overlay state for
|
||||
* that tab. It also proactively pushes on tab create and control-mode
|
||||
* change, so a tab usually receives state before it ever sends ready.
|
||||
* - `{ kind: "overlay.interrupt", sessionId }` → background asks the
|
||||
* daemon (via a `session.user_interrupt` WS event) to cancel
|
||||
* every inflight + queued tool call for that session with
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
/**
|
||||
* Generate a UUID v4 in browser contexts where randomUUID() is unavailable.
|
||||
*
|
||||
* Crypto.randomUUID() is restricted to secure contexts, while
|
||||
* crypto.getRandomValues() is available to content scripts on ordinary HTTP
|
||||
* origins. BrowserSkill records many intranet pages served over HTTP, so the
|
||||
* fallback is required for recording to work on those pages.
|
||||
*/
|
||||
export function createRandomUuid(): string {
|
||||
const cryptoApi = globalThis.crypto;
|
||||
if (typeof cryptoApi.randomUUID === "function") return cryptoApi.randomUUID();
|
||||
|
||||
const bytes = new Uint8Array(16);
|
||||
cryptoApi.getRandomValues(bytes);
|
||||
bytes[6] = (bytes[6]! & 0x0f) | 0x40;
|
||||
bytes[8] = (bytes[8]! & 0x3f) | 0x80;
|
||||
|
||||
const hex = Array.from(bytes, (byte) => byte.toString(16).padStart(2, "0")).join("");
|
||||
return [
|
||||
hex.slice(0, 8),
|
||||
hex.slice(8, 12),
|
||||
hex.slice(12, 16),
|
||||
hex.slice(16, 20),
|
||||
hex.slice(20),
|
||||
].join("-");
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
import { createRandomUuid } from "../random-uuid";
|
||||
import {
|
||||
isRecordFrameQueryMessage,
|
||||
RECORD_FRAME_PORT,
|
||||
@@ -188,7 +189,7 @@ export class RecordFrameCoordinator {
|
||||
const recording = this.#recordings.get(requestId);
|
||||
if (!recording) return true;
|
||||
recording.finishing = true;
|
||||
const commandId = crypto.randomUUID();
|
||||
const commandId = createRandomUuid();
|
||||
const results = await Promise.all(
|
||||
[...recording.agents.values()].map((agent) => this.#stopAgent(agent, requestId, commandId)),
|
||||
);
|
||||
|
||||
@@ -9,7 +9,7 @@ describe("disconnect session cleanup", () => {
|
||||
const manager = new SessionManager({
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => nextWindowId++),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
remove,
|
||||
},
|
||||
});
|
||||
@@ -41,7 +41,7 @@ describe("disconnect session cleanup", () => {
|
||||
const manager = new SessionManager({
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 100),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
remove: vi.fn(() => removeGate),
|
||||
},
|
||||
});
|
||||
|
||||
@@ -39,7 +39,7 @@ describe("attachSessionEventHandler", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 4242),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
await manager.start("aa11");
|
||||
@@ -73,7 +73,7 @@ describe("attachSessionEventHandler", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 4242),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
const ctx = await manager.start("aa11");
|
||||
@@ -87,6 +87,7 @@ describe("attachSessionEventHandler", () => {
|
||||
windowEvents: events.api,
|
||||
});
|
||||
|
||||
manager.forgetClosedTab(7, { isWindowClosing: true });
|
||||
events.emit(4242);
|
||||
for (let i = 0; i < 4; i += 1) await Promise.resolve();
|
||||
|
||||
@@ -113,7 +114,7 @@ describe("attachSessionEventHandler", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 1),
|
||||
remove: vi.fn(),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
const transport = fakeTransport();
|
||||
@@ -129,7 +130,7 @@ describe("attachSessionEventHandler", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 1),
|
||||
remove: vi.fn(),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
const transport = fakeTransport();
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import type { AgentWindowApi, AgentWindowCreateOptions } from "../agent-window";
|
||||
import { SessionManager } from "../manager";
|
||||
import { isAgentControlledTab, SessionManager } from "../manager";
|
||||
|
||||
function fakeAgentWindow(): AgentWindowApi & {
|
||||
createMock: ReturnType<typeof vi.fn>;
|
||||
@@ -13,7 +13,7 @@ function fakeAgentWindow(): AgentWindowApi & {
|
||||
return id;
|
||||
});
|
||||
const removeMock = vi.fn(async (_id: number) => {});
|
||||
const ensureActiveTabMock = vi.fn(async (_windowId: number, _url: string) => {});
|
||||
const ensureActiveTabMock = vi.fn(async (_windowId: number, _url: string) => 0);
|
||||
return {
|
||||
create: createMock,
|
||||
remove: removeMock,
|
||||
@@ -151,6 +151,76 @@ describe("SessionManager", () => {
|
||||
expect(sm.list()).toEqual([]);
|
||||
});
|
||||
|
||||
describe("explicit tab claims", () => {
|
||||
it("claims the exact home tab returned by AgentWindowApi", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow() });
|
||||
const ctx = await sm.start("aa11");
|
||||
expect(ctx.agentCreatedTabs).toEqual(new Set([0]));
|
||||
expect(isAgentControlledTab(ctx, 0)).toBe(true);
|
||||
});
|
||||
|
||||
it("keeps every other tab free without classification state", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow() });
|
||||
const ctx = await sm.start("aa11");
|
||||
expect(isAgentControlledTab(ctx, 99)).toBe(false);
|
||||
expect(ctx).not.toHaveProperty("userTabs");
|
||||
expect(ctx).not.toHaveProperty("pendingAgentTabCount");
|
||||
});
|
||||
|
||||
it("treats a committed borrow as an explicit claim", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow() });
|
||||
const ctx = await sm.start("aa11");
|
||||
const reservation = sm.tryReserveBorrow(42, ctx.sessionId);
|
||||
if ("borrowedBy" in reservation) throw new Error("unexpected borrow conflict");
|
||||
reservation.commit({ tabId: 42, originalWindowId: 7, originalIndex: 3 });
|
||||
expect(isAgentControlledTab(ctx, 42)).toBe(true);
|
||||
});
|
||||
|
||||
it("forgets an agent-created tab after Chrome removes it", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow() });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.agentCreatedTabs.add(42);
|
||||
sm.forgetClosedTab(42);
|
||||
expect(ctx.agentCreatedTabs.has(42)).toBe(false);
|
||||
expect(isAgentControlledTab(ctx, 0)).toBe(true);
|
||||
});
|
||||
|
||||
it("forgets closed borrows idempotently without affecting other tabs or sessions", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow() });
|
||||
const a = await sm.start("aa11");
|
||||
const b = await sm.start("bb22");
|
||||
a.borrowedTabs.set(42, { tabId: 42, originalWindowId: 7, originalIndex: 0 });
|
||||
a.borrowedTabs.set(43, { tabId: 43, originalWindowId: 7, originalIndex: 1 });
|
||||
b.borrowedTabs.set(44, { tabId: 44, originalWindowId: 8, originalIndex: 0 });
|
||||
|
||||
sm.forgetClosedTab(42);
|
||||
sm.forgetClosedTab(42);
|
||||
sm.forgetClosedTab(999);
|
||||
|
||||
expect(isAgentControlledTab(a, 42)).toBe(false);
|
||||
expect(sm.findBorrowingSession(42, null)).toBeNull();
|
||||
expect([...a.borrowedTabs.keys()]).toEqual([43]);
|
||||
expect([...b.borrowedTabs.keys()]).toEqual([44]);
|
||||
expect(sm.list()).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("prevents an in-flight borrow from reclaiming a closed tab", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow() });
|
||||
const ctx = await sm.start("aa11");
|
||||
const reservation = sm.tryReserveBorrow(42, ctx.sessionId);
|
||||
if ("borrowedBy" in reservation) throw new Error("unexpected borrow conflict");
|
||||
|
||||
sm.forgetClosedTab(42);
|
||||
|
||||
expect(() =>
|
||||
reservation.commit({ tabId: 42, originalWindowId: 7, originalIndex: 0 }),
|
||||
).toThrow(/reservation disappeared/);
|
||||
reservation.release();
|
||||
expect(sm.findBorrowingSession(42, null)).toBeNull();
|
||||
expect(ctx.borrowedTabs.has(42)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("findBorrowingSession", () => {
|
||||
it("returns null when no session has borrowed the tab", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow() });
|
||||
|
||||
@@ -14,8 +14,11 @@ export interface AgentWindowApi {
|
||||
* Guarantee the Agent Window has an active, CDP-navigable tab.
|
||||
* `chrome://` pages (including the New Tab page) reject `Page.navigate`,
|
||||
* so sessions bootstrap with `about:blank` instead.
|
||||
*
|
||||
* Resolves with the id of the activated (or newly created) tab, so callers
|
||||
* can track the session's home tab without re-querying Chrome.
|
||||
*/
|
||||
ensureActiveTab(windowId: number, url: string): Promise<void>;
|
||||
ensureActiveTab(windowId: number, url: string): Promise<number>;
|
||||
}
|
||||
|
||||
/** Creation hints for a new Agent Window. */
|
||||
@@ -49,15 +52,19 @@ export const chromeAgentWindowApi: AgentWindowApi = {
|
||||
// cancellation success while the Agent Window remains open.
|
||||
await chrome.windows.remove(windowId);
|
||||
},
|
||||
async ensureActiveTab(windowId: number, url: string): Promise<void> {
|
||||
async ensureActiveTab(windowId: number, url: string): Promise<number> {
|
||||
const tabs = await chrome.tabs.query({ windowId });
|
||||
const first = tabs.find((t) => typeof t.id === "number");
|
||||
if (first?.id) {
|
||||
if (first?.id !== undefined) {
|
||||
if (!first.active) {
|
||||
await chrome.tabs.update(first.id, { active: true });
|
||||
}
|
||||
return;
|
||||
return first.id;
|
||||
}
|
||||
await chrome.tabs.create({ windowId, url, active: true });
|
||||
const created = await chrome.tabs.create({ windowId, url, active: true });
|
||||
if (typeof created?.id !== "number") {
|
||||
throw new Error("[bh] chrome.tabs.create returned no tab id");
|
||||
}
|
||||
return created.id;
|
||||
},
|
||||
};
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
import type { DiagnosticSink } from "@/lib/diagnostics";
|
||||
import { handleSessionStop, type SessionStopDeps } from "@/tools/session";
|
||||
import type { SessionManager } from "./manager";
|
||||
|
||||
@@ -16,7 +15,6 @@ export interface DisconnectCleanupOptions {
|
||||
manager: SessionManager;
|
||||
sessionStopDeps?: SessionStopDeps;
|
||||
onSessionsChanged?: () => void;
|
||||
diagnostics?: DiagnosticSink;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -31,15 +29,7 @@ export function createDisconnectCleanup(options: DisconnectCleanupOptions) {
|
||||
let inFlight: Promise<DisconnectCleanupReport> | null = null;
|
||||
|
||||
return (): Promise<DisconnectCleanupReport> => {
|
||||
if (inFlight) {
|
||||
options.diagnostics?.("disconnect_cleanup.coalesced", {
|
||||
live_session_ids: options.manager.list().map((ctx) => ctx.sessionId),
|
||||
});
|
||||
return inFlight;
|
||||
}
|
||||
options.diagnostics?.("disconnect_cleanup.scheduled", {
|
||||
live_session_ids: options.manager.list().map((ctx) => ctx.sessionId),
|
||||
});
|
||||
if (inFlight) return inFlight;
|
||||
inFlight = cleanupSessions(options).finally(() => {
|
||||
inFlight = null;
|
||||
});
|
||||
@@ -52,17 +42,8 @@ async function cleanupSessions(
|
||||
): Promise<DisconnectCleanupReport> {
|
||||
const stoppedSessionIds: string[] = [];
|
||||
const failures: DisconnectCleanupFailure[] = [];
|
||||
const capturedSessions = options.manager.list();
|
||||
options.diagnostics?.("disconnect_cleanup.started", {
|
||||
captured_session_ids: capturedSessions.map((ctx) => ctx.sessionId),
|
||||
});
|
||||
|
||||
for (const ctx of capturedSessions) {
|
||||
options.diagnostics?.("disconnect_cleanup.session_stop.started", {
|
||||
session_id: ctx.sessionId,
|
||||
agent_window_id: ctx.agentWindowId,
|
||||
still_registered: options.manager.has(ctx.sessionId),
|
||||
});
|
||||
for (const ctx of options.manager.list()) {
|
||||
try {
|
||||
const result = await handleSessionStop(
|
||||
options.manager,
|
||||
@@ -81,26 +62,14 @@ async function cleanupSessions(
|
||||
continue;
|
||||
}
|
||||
stoppedSessionIds.push(ctx.sessionId);
|
||||
options.diagnostics?.("disconnect_cleanup.session_stop.completed", {
|
||||
session_id: ctx.sessionId,
|
||||
});
|
||||
} catch (err) {
|
||||
failures.push({
|
||||
sessionId: ctx.sessionId,
|
||||
message: err instanceof Error ? err.message : String(err),
|
||||
});
|
||||
options.diagnostics?.("disconnect_cleanup.session_stop.failed", {
|
||||
session_id: ctx.sessionId,
|
||||
error: err,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
options.onSessionsChanged?.();
|
||||
options.diagnostics?.("disconnect_cleanup.completed", {
|
||||
stopped_session_ids: stoppedSessionIds,
|
||||
failures,
|
||||
remaining_session_ids: options.manager.list().map((ctx) => ctx.sessionId),
|
||||
});
|
||||
return { stoppedSessionIds, failures };
|
||||
}
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
import type { DiagnosticSink } from "@/lib/diagnostics";
|
||||
import type { Transport } from "@/transport/transport";
|
||||
import type { EventFrame } from "@/transport/types";
|
||||
import type { SessionManager } from "./manager";
|
||||
@@ -25,7 +24,6 @@ export interface SessionEventHandlerOptions {
|
||||
* `chrome.storage.session` "sessions live" flag (review M4/M5 I3).
|
||||
*/
|
||||
onSessionsChanged?: () => void;
|
||||
diagnostics?: DiagnosticSink;
|
||||
}
|
||||
|
||||
function chromeWindowEvents(): WindowRemovedListener {
|
||||
@@ -52,11 +50,6 @@ export function attachSessionEventHandler(options: SessionEventHandlerOptions):
|
||||
|
||||
const onRemoved = (windowId: number): void => {
|
||||
const ctx = manager.findByWindowId(windowId);
|
||||
options.diagnostics?.("chrome.window.removed", {
|
||||
window_id: windowId,
|
||||
matched_session_id: ctx?.sessionId ?? null,
|
||||
live_session_ids: manager.list().map((entry) => entry.sessionId),
|
||||
});
|
||||
if (!ctx) return;
|
||||
const returnFailures = Array.from(ctx.borrowedTabs.keys()).map((tabId) => ({
|
||||
tab_id: tabId,
|
||||
@@ -88,16 +81,7 @@ export function attachSessionEventHandler(options: SessionEventHandlerOptions):
|
||||
};
|
||||
try {
|
||||
transport.send(event);
|
||||
options.diagnostics?.("session.window_closed.sent", {
|
||||
session_id: ctx.sessionId,
|
||||
window_id: windowId,
|
||||
});
|
||||
} catch (err) {
|
||||
options.diagnostics?.("session.window_closed.send_failed", {
|
||||
session_id: ctx.sessionId,
|
||||
window_id: windowId,
|
||||
error: err,
|
||||
});
|
||||
console.warn("[bh] could not push session.window_closed event", err);
|
||||
}
|
||||
})
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
import type { DiagnosticSink } from "@/lib/diagnostics";
|
||||
import { AGENT_WINDOW_HOME, type AgentWindowApi, chromeAgentWindowApi } from "./agent-window";
|
||||
import { RefStore } from "./ref-store";
|
||||
|
||||
@@ -7,9 +6,20 @@ export interface SessionContext {
|
||||
agentWindowId: number;
|
||||
refStore: RefStore;
|
||||
borrowedTabs: Map<number, BorrowedTab>;
|
||||
/**
|
||||
* Tabs explicitly claimed by the agent because it created them. This
|
||||
* includes the Agent Window's home tab and tabs created by `tool.tab_create`.
|
||||
* Tabs opened through Chrome UI never enter this set.
|
||||
*/
|
||||
agentCreatedTabs: Set<number>;
|
||||
createdAtMs: number;
|
||||
}
|
||||
|
||||
/** Whether this session has explicitly claimed control of `tabId`. */
|
||||
export function isAgentControlledTab(ctx: SessionContext, tabId: number): boolean {
|
||||
return ctx.agentCreatedTabs.has(tabId) || ctx.borrowedTabs.has(tabId);
|
||||
}
|
||||
|
||||
export interface BorrowedTab {
|
||||
tabId: number;
|
||||
originalWindowId: number;
|
||||
@@ -24,7 +34,6 @@ export interface BorrowReservation {
|
||||
export interface SessionManagerOptions {
|
||||
agentWindow?: AgentWindowApi;
|
||||
now?: () => number;
|
||||
diagnostics?: DiagnosticSink;
|
||||
}
|
||||
|
||||
/** Options for starting a session's Agent Window. */
|
||||
@@ -85,12 +94,10 @@ export class SessionManager {
|
||||
private readonly borrowReservations = new Map<number, string>();
|
||||
private readonly agentWindow: AgentWindowApi;
|
||||
private readonly now: () => number;
|
||||
private readonly diagnostics?: DiagnosticSink;
|
||||
|
||||
constructor(options: SessionManagerOptions = {}) {
|
||||
this.agentWindow = options.agentWindow ?? chromeAgentWindowApi;
|
||||
this.now = options.now ?? Date.now;
|
||||
this.diagnostics = options.diagnostics;
|
||||
}
|
||||
|
||||
has(sessionId: string): boolean {
|
||||
@@ -110,6 +117,19 @@ export class SessionManager {
|
||||
return Array.from(this.sessions.values());
|
||||
}
|
||||
|
||||
/**
|
||||
* Forget a tab Chrome has removed, including any uncommitted borrow.
|
||||
* Whole-window closures keep committed borrows until the window-removed
|
||||
* handler reports which user tabs could not be returned.
|
||||
*/
|
||||
forgetClosedTab(tabId: number, { isWindowClosing = false } = {}): void {
|
||||
this.borrowReservations.delete(tabId);
|
||||
for (const ctx of this.sessions.values()) {
|
||||
ctx.agentCreatedTabs.delete(tabId);
|
||||
if (!isWindowClosing) ctx.borrowedTabs.delete(tabId);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Look up whether `tabId` is currently borrowed by some *other*
|
||||
* session than the one calling. Used by M8 `tab_borrow` to refuse
|
||||
@@ -173,13 +193,6 @@ export class SessionManager {
|
||||
* daemon in the `tool.session_start` reply.
|
||||
*/
|
||||
async start(sessionId: string, opts: SessionStartOptions = {}): Promise<SessionContext> {
|
||||
this.diagnostics?.("session_manager.start.requested", {
|
||||
session_id: sessionId,
|
||||
existing_session_ids: this.list().map((ctx) => ctx.sessionId),
|
||||
focused: opts.focused ?? null,
|
||||
size: opts.size ?? null,
|
||||
signal_aborted: opts.signal?.aborted ?? false,
|
||||
});
|
||||
if (this.sessions.has(sessionId)) {
|
||||
throw new Error(`[bh] session ${sessionId} already exists`);
|
||||
}
|
||||
@@ -189,12 +202,8 @@ export class SessionManager {
|
||||
try {
|
||||
const { signal: _signal, ...createOptions } = opts;
|
||||
windowId = await this.agentWindow.create(AGENT_WINDOW_HOME, createOptions);
|
||||
this.diagnostics?.("session_manager.window.created", {
|
||||
session_id: sessionId,
|
||||
agent_window_id: windowId,
|
||||
});
|
||||
throwIfSessionStartAborted(opts.signal);
|
||||
await this.agentWindow.ensureActiveTab(windowId, AGENT_WINDOW_HOME);
|
||||
const homeTabId = await this.agentWindow.ensureActiveTab(windowId, AGENT_WINDOW_HOME);
|
||||
throwIfSessionStartAborted(opts.signal);
|
||||
|
||||
const ctx: SessionContext = {
|
||||
@@ -202,22 +211,16 @@ export class SessionManager {
|
||||
agentWindowId: windowId,
|
||||
refStore: new RefStore(),
|
||||
borrowedTabs: new Map(),
|
||||
// The home tab is the session's first explicit claim. Every other
|
||||
// tab remains free until `tab_create` or `tab_borrow` identifies it
|
||||
// by its concrete Chrome tab id.
|
||||
agentCreatedTabs: new Set([homeTabId]),
|
||||
createdAtMs: this.now(),
|
||||
};
|
||||
this.sessions.set(sessionId, ctx);
|
||||
this.windowIndex.set(windowId, sessionId);
|
||||
this.diagnostics?.("session_manager.start.committed", {
|
||||
session_id: sessionId,
|
||||
agent_window_id: windowId,
|
||||
live_session_ids: this.list().map((entry) => entry.sessionId),
|
||||
});
|
||||
return ctx;
|
||||
} catch (startupError) {
|
||||
this.diagnostics?.("session_manager.start.failed", {
|
||||
session_id: sessionId,
|
||||
agent_window_id: windowId,
|
||||
error: startupError,
|
||||
});
|
||||
if (windowId !== null) {
|
||||
try {
|
||||
await this.agentWindow.remove(windowId);
|
||||
@@ -241,28 +244,12 @@ export class SessionManager {
|
||||
options: { dropOnly?: boolean } = {},
|
||||
): Promise<SessionContext | null> {
|
||||
const ctx = this.sessions.get(sessionId);
|
||||
this.diagnostics?.("session_manager.stop.requested", {
|
||||
session_id: sessionId,
|
||||
drop_only: options.dropOnly ?? false,
|
||||
existed: ctx !== undefined,
|
||||
live_session_ids: this.list().map((entry) => entry.sessionId),
|
||||
});
|
||||
if (!ctx) return null;
|
||||
if (!options.dropOnly) {
|
||||
await this.agentWindow.remove(ctx.agentWindowId);
|
||||
this.diagnostics?.("session_manager.window.removed", {
|
||||
session_id: sessionId,
|
||||
agent_window_id: ctx.agentWindowId,
|
||||
});
|
||||
}
|
||||
this.sessions.delete(sessionId);
|
||||
this.windowIndex.delete(ctx.agentWindowId);
|
||||
this.diagnostics?.("session_manager.stop.committed", {
|
||||
session_id: sessionId,
|
||||
agent_window_id: ctx.agentWindowId,
|
||||
drop_only: options.dropOnly ?? false,
|
||||
live_session_ids: this.list().map((entry) => entry.sessionId),
|
||||
});
|
||||
return ctx;
|
||||
}
|
||||
|
||||
|
||||
@@ -127,6 +127,38 @@ describe("requestBorrowConfirmation", () => {
|
||||
expect(message.requestId.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("proactively focuses the user window before sending the overlay message", async () => {
|
||||
tabs.get.mockResolvedValueOnce({ id: 42, title: "Tab" });
|
||||
windows.getLastFocused.mockResolvedValueOnce(
|
||||
userWindowWithActiveTab({ windowId: 77, tabId: 7, url: "https://app.example/" }),
|
||||
);
|
||||
tabs.sendMessage.mockResolvedValueOnce({ type: "borrow-response", allowed: true });
|
||||
|
||||
const pending = requestBorrowConfirmation(42, {
|
||||
deps: { tabs, windows, notifications },
|
||||
});
|
||||
await vi.runAllTimersAsync();
|
||||
await pending;
|
||||
|
||||
expect(windows.update).toHaveBeenCalledWith(77, { focused: true });
|
||||
});
|
||||
|
||||
it("still proceeds when proactive window focus fails", async () => {
|
||||
tabs.get.mockResolvedValueOnce({ id: 42, title: "Tab" });
|
||||
windows.getLastFocused.mockResolvedValueOnce(
|
||||
userWindowWithActiveTab({ windowId: 77, tabId: 7, url: "https://app.example/" }),
|
||||
);
|
||||
windows.update.mockRejectedValueOnce(new Error("window gone"));
|
||||
tabs.sendMessage.mockResolvedValueOnce({ type: "borrow-response", allowed: true });
|
||||
|
||||
const pending = requestBorrowConfirmation(42, {
|
||||
deps: { tabs, windows, notifications },
|
||||
});
|
||||
await vi.runAllTimersAsync();
|
||||
|
||||
expect(await pending).toBe(true);
|
||||
});
|
||||
|
||||
it("sends confirmation to the borrow target when it is the active tab", async () => {
|
||||
tabs.get.mockResolvedValueOnce({ id: 42, title: "Active Tab" });
|
||||
windows.getLastFocused.mockResolvedValueOnce(
|
||||
|
||||
@@ -13,7 +13,7 @@ function fakeAgentWindow(ids: number[]) {
|
||||
return id;
|
||||
}),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -49,13 +49,54 @@ describe("ToolDispatcher", () => {
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
it.each([
|
||||
"tool.snapshot",
|
||||
"tool.evaluate",
|
||||
"tool.get_html",
|
||||
"tool.console",
|
||||
])("annotates Chrome extension-access failures from %s", async (method) => {
|
||||
const tab = { id: 7, windowId: 4242, active: true, url: "https://example.test" };
|
||||
vi.stubGlobal("chrome", {
|
||||
tabs: {
|
||||
get: vi.fn(async () => tab),
|
||||
query: vi.fn(async () => [tab]),
|
||||
},
|
||||
});
|
||||
const { transport, sent, deliver } = fakeTransport();
|
||||
const sessions = new SessionManager({
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 4242),
|
||||
remove: vi.fn(),
|
||||
ensureActiveTab: vi.fn(),
|
||||
},
|
||||
});
|
||||
await sessions.start("test");
|
||||
const error = new Error("Cannot access a chrome-extension:// URL of different extension");
|
||||
const cdp = {
|
||||
send: vi.fn().mockRejectedValue(error),
|
||||
ensureConsoleCapture: vi.fn().mockRejectedValue(error),
|
||||
consoleEntriesSince: vi.fn(),
|
||||
} as unknown as TestDispatcherCdp;
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions, cdp });
|
||||
dispatcher.start();
|
||||
deliver(makeRequest(method, { session_id: "test", expression: "document.title" }));
|
||||
await vi.waitFor(() => expect(sent).toHaveLength(1));
|
||||
expect(sent[0]).toMatchObject({
|
||||
error: {
|
||||
code: "cdp_failed",
|
||||
data: { reason: "cdp_extension_access_denied" },
|
||||
},
|
||||
});
|
||||
dispatcher.stop();
|
||||
});
|
||||
|
||||
it("uses the configured recording runtime for start and stop", async () => {
|
||||
const { transport, sent, deliver } = fakeTransport();
|
||||
const sessions = new SessionManager({
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 4242),
|
||||
remove: vi.fn(),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
await sessions.start("aa11");
|
||||
@@ -117,7 +158,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 4242),
|
||||
remove: vi.fn(),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions });
|
||||
@@ -140,7 +181,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create,
|
||||
remove: vi.fn(),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions });
|
||||
@@ -158,7 +199,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 4242),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
await sessions.start("aa11");
|
||||
@@ -172,6 +213,47 @@ describe("ToolDispatcher", () => {
|
||||
expect(sent[0]).toEqual({ id: "r-1", result: {} });
|
||||
});
|
||||
|
||||
it("wires the production tab APIs into tool.session_stop so a surviving user tab releases the window", async () => {
|
||||
// Regression guard for the deps that session_stop reads directly: when
|
||||
// `tabManagement.tabs` / `tabsQuery` are not injected by the dispatcher,
|
||||
// the agent-tab cleanup and the release decision silently no-op and the
|
||||
// Agent Window is closed even though a user tab is still open (issue #57).
|
||||
const removeTab = vi.fn(async () => {});
|
||||
const closeWindow = vi.fn(async () => {});
|
||||
// After cleanup the window still holds one tab (id 99) that is neither
|
||||
// the home tab nor an agent-created tab — i.e. a genuine user tab.
|
||||
vi.stubGlobal("chrome", {
|
||||
tabs: {
|
||||
remove: removeTab,
|
||||
query: vi.fn(async () => [{ id: 99, windowId: 4242, active: true }]),
|
||||
},
|
||||
});
|
||||
const { transport, sent, deliver } = fakeTransport();
|
||||
const sessions = new SessionManager({
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 4242),
|
||||
remove: closeWindow,
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
await sessions.start("aa11");
|
||||
// Register the exact tab id returned by the agent's creation path.
|
||||
sessions.get("aa11")?.agentCreatedTabs.add(7);
|
||||
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions });
|
||||
dispatcher.start();
|
||||
|
||||
deliver(makeRequest("tool.session_stop", { session_id: "aa11" }));
|
||||
await flushMicrotasks();
|
||||
|
||||
// The agent tab is closed through the real chrome.tabs surface...
|
||||
expect(removeTab).toHaveBeenCalledWith(7);
|
||||
// ...and the window is released rather than closed, so the user's tab survives.
|
||||
expect(closeWindow).not.toHaveBeenCalled();
|
||||
expect(sessions.has("aa11")).toBe(false);
|
||||
expect(sent[0]).toEqual({ id: "r-1", result: { window_released: true } });
|
||||
});
|
||||
|
||||
it("routes tool.console through the CDP console buffer", async () => {
|
||||
vi.stubGlobal("chrome", {
|
||||
tabs: {
|
||||
@@ -183,7 +265,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 4242),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
await sessions.start("aa11");
|
||||
@@ -232,13 +314,114 @@ describe("ToolDispatcher", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("bypasses and restores the control overlay for an upload trigger click", async () => {
|
||||
const sendMessage = vi.fn(async () => undefined);
|
||||
vi.stubGlobal("chrome", {
|
||||
tabs: {
|
||||
get: vi.fn(async () => ({ id: 7, windowId: 4242, active: true })),
|
||||
query: vi.fn(async () => [{ id: 7, windowId: 4242, active: true }]),
|
||||
sendMessage,
|
||||
},
|
||||
});
|
||||
const { transport, sent, deliver } = fakeTransport();
|
||||
const sessions = new SessionManager({
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 4242),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
const ctx = await sessions.start("aa11");
|
||||
ctx.refStore.set("e1", 123, { tabId: 7 });
|
||||
const send = vi.fn(async <T>(_tabId: number, method: string, params?: object) => {
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } } as T;
|
||||
}
|
||||
if (method === "DOM.getContentQuads") {
|
||||
return { quads: [[0, 0, 20, 0, 20, 20, 0, 20]] } as T;
|
||||
}
|
||||
if (method === "DOM.resolveNode") {
|
||||
return { object: { objectId: "trigger-object" } } as T;
|
||||
}
|
||||
if (method === "DOM.describeNode") return { node: { backendNodeId: 456 } } as T;
|
||||
if (method === "Runtime.callFunctionOn") {
|
||||
const declaration = (params as { functionDeclaration?: string }).functionDeclaration ?? "";
|
||||
if (declaration.includes("count: state.inputs.length")) {
|
||||
return { result: { value: { count: 1, multiple: false } } } as T;
|
||||
}
|
||||
if (declaration.includes("inputs[0]")) {
|
||||
return { result: { objectId: "input-object" } } as T;
|
||||
}
|
||||
return { result: { value: true } } as T;
|
||||
}
|
||||
if (method === "Runtime.evaluate") {
|
||||
const expression = (params as { expression?: string }).expression ?? "";
|
||||
if (expression.includes("overlayDetails")) {
|
||||
return { result: { value: { hitIndex: 0 } } } as T;
|
||||
}
|
||||
if (expression.includes("overlayHostPresent")) {
|
||||
return {
|
||||
result: {
|
||||
value: { overlayHostPresent: true, overlayHostConnected: true },
|
||||
},
|
||||
} as T;
|
||||
}
|
||||
if (expression.includes("count:")) {
|
||||
return { result: { value: { count: 1, multiple: false } } } as T;
|
||||
}
|
||||
if (expression.includes("?.inputs[0]")) {
|
||||
return { result: { objectId: "input-object" } } as T;
|
||||
}
|
||||
return { result: { value: true } } as T;
|
||||
}
|
||||
return {} as T;
|
||||
});
|
||||
const cdp = {
|
||||
send,
|
||||
detachSession: vi.fn(async () => {}),
|
||||
ensureNetworkCapture: vi.fn(async () => {}),
|
||||
networkEntriesSince: vi.fn(() => ({
|
||||
tab_id: 7,
|
||||
entries: [],
|
||||
next_since: 0,
|
||||
truncated: false,
|
||||
})),
|
||||
setDeviceMetricsOverride: vi.fn(async () => {}),
|
||||
clearDeviceMetricsOverride: vi.fn(async () => {}),
|
||||
setUserAgentOverride: vi.fn(async () => {}),
|
||||
setTouchEmulationEnabled: vi.fn(async () => {}),
|
||||
};
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions, cdp: cdp as TestDispatcherCdp });
|
||||
dispatcher.start();
|
||||
|
||||
deliver(
|
||||
makeRequest("tool.upload", {
|
||||
session_id: "aa11",
|
||||
ref: "@e1",
|
||||
files: [{ transfer_id: "tr_1", name: "test.png", staged_path: "/stage/test.png" }],
|
||||
}),
|
||||
);
|
||||
await flushMicrotasks();
|
||||
await vi.waitFor(() => expect(sent).toHaveLength(1));
|
||||
|
||||
expect(sent[0]).toMatchObject({ result: { tab_id: 7, file_names: ["test.png"] } });
|
||||
expect(sendMessage).toHaveBeenNthCalledWith(1, 7, {
|
||||
type: "bh-automation-bypass",
|
||||
enabled: true,
|
||||
});
|
||||
expect(sendMessage).toHaveBeenNthCalledWith(2, 7, {
|
||||
type: "bh-automation-bypass",
|
||||
enabled: false,
|
||||
});
|
||||
});
|
||||
|
||||
it("detaches CDP state before stopping a session", async () => {
|
||||
const { transport, sent, deliver } = fakeTransport();
|
||||
const sessions = new SessionManager({
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 4242),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
await sessions.start("aa11");
|
||||
@@ -274,7 +457,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 4242),
|
||||
remove,
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
await sessions.start("aa11");
|
||||
@@ -322,7 +505,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 1),
|
||||
remove: vi.fn(),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions });
|
||||
@@ -342,7 +525,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 1),
|
||||
remove: vi.fn(),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions });
|
||||
@@ -366,7 +549,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 1),
|
||||
remove: vi.fn(),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions });
|
||||
@@ -383,7 +566,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 1),
|
||||
remove: vi.fn(),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions });
|
||||
@@ -400,7 +583,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 1),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
const onSessionsChanged = vi.fn();
|
||||
@@ -417,16 +600,32 @@ describe("ToolDispatcher", () => {
|
||||
});
|
||||
|
||||
it("invokes onBrowserControlResumed for browser-control tools but not passive reads", async () => {
|
||||
vi.stubGlobal("chrome", {
|
||||
tabs: {
|
||||
create: vi.fn(async (props: chrome.tabs.CreateProperties) => ({
|
||||
id: 7,
|
||||
windowId: props.windowId,
|
||||
url: props.url,
|
||||
active: props.active,
|
||||
})),
|
||||
},
|
||||
});
|
||||
const { transport, sent, deliver } = fakeTransport();
|
||||
const sessions = new SessionManager({
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 1),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
const onBrowserControlResumed = vi.fn();
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions, onBrowserControlResumed });
|
||||
const onAgentTabClaimed = vi.fn();
|
||||
const dispatcher = new ToolDispatcher({
|
||||
transport,
|
||||
sessions,
|
||||
onBrowserControlResumed,
|
||||
onAgentTabClaimed,
|
||||
});
|
||||
dispatcher.start();
|
||||
|
||||
deliver(makeRequest("tool.session_start", { session_id: "aa11" }));
|
||||
@@ -443,6 +642,7 @@ describe("ToolDispatcher", () => {
|
||||
});
|
||||
await flushMicrotasks();
|
||||
expect(onBrowserControlResumed).toHaveBeenCalledWith("aa11");
|
||||
expect(onAgentTabClaimed).toHaveBeenCalledWith(7, 1);
|
||||
});
|
||||
|
||||
it("reasserts remembered hover before follow-up work and releases only after actions", async () => {
|
||||
@@ -456,7 +656,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 1),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
const order: string[] = [];
|
||||
@@ -538,7 +738,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 1),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
const cdp = {
|
||||
@@ -588,6 +788,135 @@ describe("ToolDispatcher", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ tab_id: 7, code: "not_found" },
|
||||
{ tab_id: undefined, code: "invalid_params" },
|
||||
])("keeps hover when tab_return is rejected with $code", async ({ tab_id, code }) => {
|
||||
const sendMessage = vi.fn(async () => undefined);
|
||||
vi.stubGlobal("chrome", {
|
||||
tabs: {
|
||||
sendMessage,
|
||||
get: vi.fn(async () => ({ id: 7, windowId: 100, active: true })),
|
||||
query: vi.fn(async () => [{ id: 7, windowId: 100, active: true }]),
|
||||
},
|
||||
});
|
||||
const { transport, sent, deliver } = fakeTransport();
|
||||
const sessions = new SessionManager({
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 100),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
await sessions.start("aa11");
|
||||
const cdp = {
|
||||
send: vi.fn(async <T>() => ({}) as T),
|
||||
detachSession: vi.fn(async () => {}),
|
||||
releaseSessionTab: vi.fn(async () => {}),
|
||||
};
|
||||
const dispatcher = new ToolDispatcher({
|
||||
transport,
|
||||
sessions,
|
||||
cdp: cdp as unknown as TestDispatcherCdp,
|
||||
});
|
||||
const helpers = dispatcher as unknown as {
|
||||
rememberHover: (sessionId: string, result: unknown) => unknown;
|
||||
setHoverBypass: (sessionId: string, tabId: number, enabled: boolean) => Promise<void>;
|
||||
withHoverReassert: <T>(
|
||||
params: { session_id: string; tab_id?: number },
|
||||
work: () => Promise<T>,
|
||||
) => Promise<T>;
|
||||
};
|
||||
helpers.rememberHover("aa11", { tab_id: 7, x: 10, y: 20 });
|
||||
await helpers.setHoverBypass("aa11", 7, true);
|
||||
dispatcher.start();
|
||||
deliver(makeRequest("tool.tab_return", { session_id: "aa11", tab_id }));
|
||||
await vi.waitFor(() => expect(sent).toHaveLength(1));
|
||||
|
||||
expect(sent[0]).toMatchObject({ error: { code } });
|
||||
expect(cdp.releaseSessionTab).not.toHaveBeenCalled();
|
||||
expect(sendMessage).not.toHaveBeenCalledWith(7, expect.objectContaining({ enabled: false }));
|
||||
await helpers.withHoverReassert({ session_id: "aa11", tab_id: 7 }, async () => ({}));
|
||||
expect(cdp.send).toHaveBeenCalledExactlyOnceWith(7, "Input.dispatchMouseEvent", {
|
||||
type: "mouseMoved",
|
||||
x: 10,
|
||||
y: 20,
|
||||
});
|
||||
dispatcher.stop();
|
||||
});
|
||||
|
||||
it("releases the returned tab's hover before moving it and leaves other tabs alone", async () => {
|
||||
const sendMessage = vi.fn(async () => undefined);
|
||||
const move = vi.fn(async () => {
|
||||
expect(sendMessage).toHaveBeenCalledWith(7, expect.objectContaining({ enabled: false }));
|
||||
return { id: 7, windowId: 200, index: 4 };
|
||||
});
|
||||
vi.stubGlobal("chrome", {
|
||||
tabs: { sendMessage, move },
|
||||
windows: { get: vi.fn(async () => ({ id: 200 })) },
|
||||
});
|
||||
const { transport, sent, deliver } = fakeTransport();
|
||||
const sessions = new SessionManager({
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 100),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
const ctx = await sessions.start("aa11");
|
||||
ctx.borrowedTabs.set(7, { tabId: 7, originalWindowId: 200, originalIndex: 4 });
|
||||
const cdp = {
|
||||
send: vi.fn(async <T>() => ({}) as T),
|
||||
detachSession: vi.fn(async () => {}),
|
||||
releaseSessionTab: vi.fn(async () => {}),
|
||||
};
|
||||
const dispatcher = new ToolDispatcher({
|
||||
transport,
|
||||
sessions,
|
||||
cdp: cdp as unknown as TestDispatcherCdp,
|
||||
});
|
||||
const helpers = dispatcher as unknown as {
|
||||
rememberHover: (sessionId: string, result: unknown) => unknown;
|
||||
setHoverBypass: (sessionId: string, tabId: number, enabled: boolean) => Promise<void>;
|
||||
withHoverReassert: <T>(
|
||||
params: { session_id: string; tab_id?: number },
|
||||
work: () => Promise<T>,
|
||||
) => Promise<T>;
|
||||
};
|
||||
for (const [sessionId, tabId] of [
|
||||
["aa11", 7],
|
||||
["aa11", 8],
|
||||
["bb22", 9],
|
||||
] as const) {
|
||||
helpers.rememberHover(sessionId, { tab_id: tabId, x: 10, y: 20 });
|
||||
await helpers.setHoverBypass(sessionId, tabId, true);
|
||||
}
|
||||
dispatcher.start();
|
||||
deliver(makeRequest("tool.tab_return", { session_id: "aa11", tab_id: 7 }));
|
||||
await vi.waitFor(() => expect(sent).toHaveLength(1));
|
||||
|
||||
expect(sent[0]).toMatchObject({ result: { tab_id: 7, returned_to_window_id: 200 } });
|
||||
expect(cdp.releaseSessionTab).toHaveBeenCalledExactlyOnceWith("aa11", 7);
|
||||
await helpers.withHoverReassert({ session_id: "aa11", tab_id: 7 }, async () => ({}));
|
||||
expect(cdp.send).not.toHaveBeenCalled();
|
||||
for (const [sessionId, tabId] of [
|
||||
["aa11", 8],
|
||||
["bb22", 9],
|
||||
] as const) {
|
||||
expect(sendMessage).not.toHaveBeenCalledWith(
|
||||
tabId,
|
||||
expect.objectContaining({ enabled: false }),
|
||||
);
|
||||
await helpers.withHoverReassert({ session_id: sessionId, tab_id: tabId }, async () => ({}));
|
||||
expect(cdp.send).toHaveBeenLastCalledWith(tabId, "Input.dispatchMouseEvent", {
|
||||
type: "mouseMoved",
|
||||
x: 10,
|
||||
y: 20,
|
||||
});
|
||||
}
|
||||
dispatcher.stop();
|
||||
});
|
||||
|
||||
it("transfers hover overlay bypass ownership between sessions", async () => {
|
||||
vi.stubGlobal("chrome", {
|
||||
tabs: {
|
||||
@@ -599,7 +928,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 1),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions });
|
||||
@@ -636,7 +965,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 4242),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
await sessions.start("aa11");
|
||||
@@ -688,7 +1017,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 1),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions });
|
||||
@@ -722,7 +1051,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 4242),
|
||||
remove: vi.fn(),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
transport.send = () => {
|
||||
@@ -744,7 +1073,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 5555),
|
||||
remove,
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
transport.send = () => {
|
||||
@@ -772,7 +1101,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(() => createPromise),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions });
|
||||
@@ -823,7 +1152,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 1),
|
||||
remove: vi.fn(),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions });
|
||||
@@ -844,7 +1173,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(() => createPromise),
|
||||
remove: vi.fn(),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions });
|
||||
|
||||
@@ -20,7 +20,7 @@ function fakeAgentWindow(ids: number[]) {
|
||||
return id;
|
||||
}),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { RpcError } from "@/transport/types";
|
||||
import { cdpError, classifyCdpError } from "../errors";
|
||||
|
||||
const denied = "Cannot access a chrome-extension:// URL of different extension";
|
||||
|
||||
describe("CDP extension access errors", () => {
|
||||
it("classifies both Chrome Error instances and string failures", () => {
|
||||
for (const error of [new Error(denied), denied]) {
|
||||
expect(cdpError(error)).toEqual({
|
||||
code: "cdp_failed",
|
||||
message: denied,
|
||||
data: { reason: "cdp_extension_access_denied" },
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
it("preserves transfer outcome and cleanup instructions when CDP was denied", () => {
|
||||
const error: RpcError = {
|
||||
code: "cdp_failed",
|
||||
message: denied,
|
||||
data: {
|
||||
reason: "transfer_outcome_unknown",
|
||||
effect_state: "unknown",
|
||||
cleanup_state: "failed",
|
||||
},
|
||||
};
|
||||
expect(classifyCdpError(error)).toBe(error);
|
||||
});
|
||||
|
||||
it("leaves other error codes and unrelated debugger failures unchanged", () => {
|
||||
for (const error of [
|
||||
{ code: "permission_denied", message: denied },
|
||||
{ code: "cdp_failed", message: "Another debugger is already attached" },
|
||||
] satisfies RpcError[]) {
|
||||
expect(classifyCdpError(error)).toBe(error);
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -12,7 +12,7 @@ function fakeAgentWindow(ids: number[]) {
|
||||
return id;
|
||||
}),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,988 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { SessionManager } from "@/session-manager/manager";
|
||||
import { type DownloadsApi, handleDownload } from "../download";
|
||||
import { captureBrowserDownload } from "../download-capture";
|
||||
import { uploadThroughFileDrop } from "../file-drop-transaction";
|
||||
import { uploadThroughActivatedFileInput } from "../file-input-transaction";
|
||||
import type { ResolvedActionTarget } from "../interaction";
|
||||
import type { CdpRunner } from "../shared";
|
||||
import { handleUpload } from "../upload";
|
||||
|
||||
function sessions() {
|
||||
return new SessionManager({
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 100),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function tabsApi() {
|
||||
return {
|
||||
get: vi.fn(
|
||||
async (tabId: number) => ({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab,
|
||||
),
|
||||
query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
|
||||
};
|
||||
}
|
||||
|
||||
function fakeEvent<T extends (...args: never[]) => unknown>() {
|
||||
const listeners = new Set<T>();
|
||||
return {
|
||||
addListener: (listener: T) => listeners.add(listener),
|
||||
removeListener: (listener: T) => listeners.delete(listener),
|
||||
emit: (...args: Parameters<T>) => {
|
||||
for (const listener of listeners) listener(...args);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function actionTarget(frameId?: string, sessionId?: string): ResolvedActionTarget {
|
||||
return {
|
||||
tab: { tabId: 4, windowId: 100, active: true },
|
||||
backendNodeId: 123,
|
||||
cdpTarget: { tabId: 4, ...(sessionId ? { sessionId } : {}) },
|
||||
...(frameId ? { frameId } : {}),
|
||||
usedRef: "e3",
|
||||
};
|
||||
}
|
||||
|
||||
function withoutOverlay<T>(operation: () => Promise<T>): Promise<T> {
|
||||
return operation();
|
||||
}
|
||||
|
||||
function uploadCdp(
|
||||
options: {
|
||||
inputCount?: number;
|
||||
multiple?: boolean;
|
||||
chooser?: { frameId?: string; backendNodeId?: number; mode?: string };
|
||||
pendingResolve?: boolean;
|
||||
} = {},
|
||||
) {
|
||||
const calls: Array<{ method: string; params?: object }> = [];
|
||||
let cdpEvent: Parameters<NonNullable<CdpRunner["onEvent"]>>[0] | undefined;
|
||||
const send = vi.fn(async (_tabId: number, method: string, params?: object) => {
|
||||
calls.push({ method, params });
|
||||
if (method === "DOM.scrollIntoViewIfNeeded") return {};
|
||||
if (method === "Page.setInterceptFileChooserDialog") return {};
|
||||
if (method === "Page.getLayoutMetrics")
|
||||
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
|
||||
if (method === "DOM.getContentQuads") return { quads: [[0, 0, 20, 0, 20, 20, 0, 20]] };
|
||||
if (method === "DOM.resolveNode") {
|
||||
if (options.pendingResolve) return new Promise<never>(() => {});
|
||||
return { object: { objectId: "trigger-object" } };
|
||||
}
|
||||
if (method === "Runtime.callFunctionOn") {
|
||||
const declaration = (params as { functionDeclaration?: string }).functionDeclaration ?? "";
|
||||
if (declaration.includes("Object.defineProperty")) return { result: { value: true } };
|
||||
if (declaration.includes("count: state.inputs.length")) {
|
||||
return {
|
||||
result: {
|
||||
value: {
|
||||
count: options.inputCount ?? 1,
|
||||
multiple: options.multiple ?? true,
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
if (declaration.includes("inputs[0]")) {
|
||||
return { result: { objectId: "input-object" } };
|
||||
}
|
||||
return { result: { value: true } };
|
||||
}
|
||||
if (method === "DOM.describeNode") return { node: { backendNodeId: 456 } };
|
||||
if (
|
||||
method === "Input.dispatchMouseEvent" &&
|
||||
(params as { type?: string }).type === "mousePressed" &&
|
||||
options.chooser
|
||||
) {
|
||||
cdpEvent?.({ tabId: 4 }, "Page.fileChooserOpened", options.chooser);
|
||||
}
|
||||
return {};
|
||||
});
|
||||
const cdp: CdpRunner = {
|
||||
send: send as unknown as CdpRunner["send"],
|
||||
onEvent: (handler) => {
|
||||
cdpEvent = handler;
|
||||
return { dispose: vi.fn() };
|
||||
},
|
||||
};
|
||||
return {
|
||||
calls,
|
||||
emitChooser: (event: { frameId?: string; backendNodeId?: number; mode?: string }) =>
|
||||
cdpEvent?.({ tabId: 4 }, "Page.fileChooserOpened", event),
|
||||
cdp,
|
||||
};
|
||||
}
|
||||
|
||||
function dropCdp(
|
||||
options: {
|
||||
targetContainsPoint?: boolean;
|
||||
dispatchError?: (type: string) => Error | Promise<never> | undefined;
|
||||
onCall?: (method: string) => void;
|
||||
} = {},
|
||||
) {
|
||||
const calls: Array<{ method: string; params?: object }> = [];
|
||||
const detach = vi.fn(async () => {});
|
||||
const send = vi.fn(async (_tabId: number, method: string, params?: object) => {
|
||||
calls.push({ method, params });
|
||||
options.onCall?.(method);
|
||||
if (method === "DOM.scrollIntoViewIfNeeded") return {};
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
|
||||
}
|
||||
if (method === "DOM.getContentQuads") {
|
||||
return { quads: [[10, 20, 210, 20, 210, 120, 10, 120]] };
|
||||
}
|
||||
if (method === "DOM.getNodeForLocation") return { backendNodeId: 124 };
|
||||
if (method === "DOM.resolveNode") {
|
||||
const id = (params as { backendNodeId?: number }).backendNodeId;
|
||||
return { object: { objectId: id === 123 ? "drop-target" : "hit-target" } };
|
||||
}
|
||||
if (method === "Runtime.callFunctionOn") {
|
||||
return { result: { value: options.targetContainsPoint ?? true } };
|
||||
}
|
||||
if (method === "Input.dispatchDragEvent") {
|
||||
const type = (params as { type?: string }).type ?? "";
|
||||
const error = options.dispatchError?.(type);
|
||||
if (error instanceof Promise) return error;
|
||||
if (error) throw error;
|
||||
return {};
|
||||
}
|
||||
if (method === "Runtime.releaseObjectGroup") return {};
|
||||
throw new Error(`unexpected drop CDP command ${method}`);
|
||||
});
|
||||
const cdp: CdpRunner = {
|
||||
send: send as unknown as CdpRunner["send"],
|
||||
detach,
|
||||
};
|
||||
return { calls, cdp, detach };
|
||||
}
|
||||
|
||||
describe("file transfer tools", () => {
|
||||
it("captures the file input activated by the requested click and injects only staged paths", async () => {
|
||||
const manager = sessions();
|
||||
const ctx = await manager.start("s1");
|
||||
ctx.refStore.set("e3", 123, { tabId: 4 });
|
||||
const { cdp, calls } = uploadCdp();
|
||||
|
||||
const result = await handleUpload(
|
||||
manager,
|
||||
{
|
||||
session_id: "s1",
|
||||
ref: "@e3",
|
||||
files: [
|
||||
{ transfer_id: "tr_1", name: "one.png", staged_path: "/private/stage/one" },
|
||||
{ transfer_id: "tr_2", name: "two.png", staged_path: "/private/stage/two" },
|
||||
],
|
||||
},
|
||||
{ cdp, tabsApi: tabsApi() },
|
||||
);
|
||||
|
||||
expect(result).toMatchObject({ tab_id: 4, file_names: ["one.png", "two.png"] });
|
||||
expect(calls[0]).toEqual({
|
||||
method: "Page.setInterceptFileChooserDialog",
|
||||
params: { enabled: true },
|
||||
});
|
||||
expect(calls).toContainEqual({
|
||||
method: "Page.setInterceptFileChooserDialog",
|
||||
params: { enabled: false, cancel: true },
|
||||
});
|
||||
expect(calls).toContainEqual({
|
||||
method: "DOM.setFileInputFiles",
|
||||
params: { files: ["/private/stage/one", "/private/stage/two"], backendNodeId: 456 },
|
||||
});
|
||||
});
|
||||
|
||||
it("fails immediately when the trigger does not activate a file input", async () => {
|
||||
const { cdp } = uploadCdp({ inputCount: 0 });
|
||||
const result = await uploadThroughActivatedFileInput({
|
||||
cdp,
|
||||
actionTarget: actionTarget(),
|
||||
files: ["/private/stage/one"],
|
||||
timeoutMs: 100,
|
||||
trigger: async () => ({ tab_id: 4, x: 10, y: 10 }),
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({
|
||||
code: "unsupported",
|
||||
data: { reason: "file_input_not_activated", phase: "resolve_input" },
|
||||
});
|
||||
});
|
||||
|
||||
it("fails before clicking when chooser interception is unavailable", async () => {
|
||||
const trigger = vi.fn(async () => ({ tab_id: 4, x: 10, y: 10 }));
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async (_tabId: number, method: string) => {
|
||||
if (method === "Page.setInterceptFileChooserDialog") {
|
||||
throw new Error("method unavailable");
|
||||
}
|
||||
return {};
|
||||
}) as CdpRunner["send"],
|
||||
};
|
||||
|
||||
const result = await uploadThroughActivatedFileInput({
|
||||
cdp,
|
||||
actionTarget: actionTarget(),
|
||||
files: ["/private/stage/one"],
|
||||
timeoutMs: 100,
|
||||
trigger,
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({
|
||||
code: "cdp_failed",
|
||||
data: { effect_state: "none", phase: "arm_interception" },
|
||||
});
|
||||
expect(trigger).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("uses an exact chooser event as an independent input-location signal", async () => {
|
||||
const { cdp, calls, emitChooser } = uploadCdp({ inputCount: 0 });
|
||||
const result = await uploadThroughActivatedFileInput({
|
||||
cdp,
|
||||
actionTarget: actionTarget("f1"),
|
||||
files: ["/private/stage/one"],
|
||||
timeoutMs: 100,
|
||||
trigger: async () => {
|
||||
emitChooser({ frameId: "f1", backendNodeId: 789, mode: "selectSingle" });
|
||||
return { tab_id: 4, x: 10, y: 10 };
|
||||
},
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({ multiple: false });
|
||||
expect(calls).toContainEqual({
|
||||
method: "DOM.setFileInputFiles",
|
||||
params: { files: ["/private/stage/one"], backendNodeId: 789 },
|
||||
});
|
||||
});
|
||||
|
||||
it("reports a File System Access picker without waiting for a timeout", async () => {
|
||||
const { cdp, emitChooser } = uploadCdp({
|
||||
inputCount: 0,
|
||||
});
|
||||
const result = await uploadThroughActivatedFileInput({
|
||||
cdp,
|
||||
actionTarget: actionTarget("f1"),
|
||||
files: ["/private/stage/one"],
|
||||
timeoutMs: 100,
|
||||
trigger: async () => {
|
||||
emitChooser({ frameId: "f1", mode: "selectSingle" });
|
||||
return { tab_id: 4, x: 10, y: 10 };
|
||||
},
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({
|
||||
code: "unsupported",
|
||||
message: "upload trigger invoked a non-input file picker",
|
||||
data: { reason: "file_input_not_activated", phase: "resolve_input" },
|
||||
});
|
||||
});
|
||||
|
||||
it("bounds a stuck file-input probe", async () => {
|
||||
const { cdp } = uploadCdp({ pendingResolve: true });
|
||||
|
||||
const result = await uploadThroughActivatedFileInput({
|
||||
cdp,
|
||||
actionTarget: actionTarget(),
|
||||
files: ["/private/stage/one"],
|
||||
timeoutMs: 5,
|
||||
trigger: vi.fn(),
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({
|
||||
code: "timeout",
|
||||
data: { reason: "file_input_probe_failed", phase: "arm_input_probe" },
|
||||
});
|
||||
});
|
||||
|
||||
it("marks a timed-out file assignment unknown and detaches browser state", async () => {
|
||||
const fixture = uploadCdp();
|
||||
const originalSend = fixture.cdp.send;
|
||||
const detach = vi.fn(async () => {});
|
||||
fixture.cdp.detach = detach;
|
||||
fixture.cdp.send = vi.fn((tabId: number, method: string, params?: object) => {
|
||||
if (method === "DOM.setFileInputFiles") return new Promise<never>(() => {});
|
||||
return originalSend(tabId, method, params);
|
||||
}) as CdpRunner["send"];
|
||||
|
||||
const result = await uploadThroughActivatedFileInput({
|
||||
cdp: fixture.cdp,
|
||||
actionTarget: actionTarget(),
|
||||
files: ["/private/stage/one"],
|
||||
timeoutMs: 10,
|
||||
trigger: async () => ({ tab_id: 4, x: 10, y: 10 }),
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({
|
||||
code: "timeout",
|
||||
data: { effect_state: "unknown", phase: "set_files" },
|
||||
});
|
||||
expect(detach).toHaveBeenCalledWith(4);
|
||||
});
|
||||
|
||||
it("delivers staged files through one explicit native drop without clicking", async () => {
|
||||
const manager = sessions();
|
||||
const ctx = await manager.start("s1");
|
||||
ctx.refStore.set("e3", 123, { tabId: 4 });
|
||||
const lifecycle: string[] = [];
|
||||
const { cdp, calls } = dropCdp({ onCall: (method) => lifecycle.push(method) });
|
||||
const overlayPhases: string[] = [];
|
||||
|
||||
const result = await handleUpload(
|
||||
manager,
|
||||
{
|
||||
session_id: "s1",
|
||||
ref: "@e3",
|
||||
mode: "drop",
|
||||
files: [{ transfer_id: "tr_1", name: "one.png", staged_path: "/stage/one.png" }],
|
||||
},
|
||||
{
|
||||
cdp,
|
||||
tabsApi: tabsApi(),
|
||||
sendToTab: vi.fn(async (_tabId, message) => {
|
||||
overlayPhases.push(message.phase);
|
||||
lifecycle.push(`overlay:${message.phase}`);
|
||||
}),
|
||||
},
|
||||
);
|
||||
|
||||
expect(result).toMatchObject({ tab_id: 4, used_ref: "e3", file_names: ["one.png"] });
|
||||
expect(calls.filter((call) => call.method === "Input.dispatchDragEvent")).toEqual([
|
||||
{
|
||||
method: "Input.dispatchDragEvent",
|
||||
params: {
|
||||
type: "dragEnter",
|
||||
x: 110,
|
||||
y: 70,
|
||||
data: { items: [], files: ["/stage/one.png"], dragOperationsMask: 1 },
|
||||
},
|
||||
},
|
||||
{
|
||||
method: "Input.dispatchDragEvent",
|
||||
params: {
|
||||
type: "dragOver",
|
||||
x: 110,
|
||||
y: 70,
|
||||
data: { items: [], files: ["/stage/one.png"], dragOperationsMask: 1 },
|
||||
},
|
||||
},
|
||||
{
|
||||
method: "Input.dispatchDragEvent",
|
||||
params: {
|
||||
type: "drop",
|
||||
x: 110,
|
||||
y: 70,
|
||||
data: { items: [], files: ["/stage/one.png"], dragOperationsMask: 1 },
|
||||
},
|
||||
},
|
||||
]);
|
||||
expect(calls.some((call) => call.method === "Input.dispatchMouseEvent")).toBe(false);
|
||||
expect(overlayPhases).toEqual(["begin", "end"]);
|
||||
expect(lifecycle.indexOf("DOM.getContentQuads")).toBeLessThan(
|
||||
lifecycle.indexOf("overlay:begin"),
|
||||
);
|
||||
expect(lifecycle.indexOf("overlay:begin")).toBeLessThan(
|
||||
lifecycle.indexOf("DOM.getNodeForLocation"),
|
||||
);
|
||||
expect(lifecycle.lastIndexOf("Input.dispatchDragEvent")).toBeLessThan(
|
||||
lifecycle.indexOf("overlay:end"),
|
||||
);
|
||||
expect(lifecycle.indexOf("overlay:end")).toBeLessThan(
|
||||
lifecycle.indexOf("Runtime.releaseObjectGroup"),
|
||||
);
|
||||
});
|
||||
|
||||
it("fails closed before dragging when another element owns the action point", async () => {
|
||||
const { cdp, calls } = dropCdp({ targetContainsPoint: false });
|
||||
const result = await uploadThroughFileDrop({
|
||||
cdp,
|
||||
actionTarget: actionTarget(),
|
||||
files: ["/stage/one.png"],
|
||||
timeoutMs: 100,
|
||||
withOverlayHidden: withoutOverlay,
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({
|
||||
code: "permission_denied",
|
||||
data: {
|
||||
reason: "file_drop_target_unavailable",
|
||||
mechanism: "drop",
|
||||
effect_state: "none",
|
||||
phase: "resolve_target",
|
||||
},
|
||||
});
|
||||
expect(calls.some((call) => call.method === "Input.dispatchDragEvent")).toBe(false);
|
||||
});
|
||||
|
||||
it("dispatches an OOPIF file drop in the child target's local coordinates", async () => {
|
||||
const rootCalls: Array<{ method: string; params?: object }> = [];
|
||||
const childCalls: Array<{ method: string; params?: object }> = [];
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async (_tabId, method, params) => {
|
||||
rootCalls.push({ method, params });
|
||||
if (method === "DOM.scrollIntoViewIfNeeded") return {};
|
||||
if (method === "DOM.getBoxModel") {
|
||||
return { model: { content: [100, 100, 300, 100, 300, 200, 100, 200] } };
|
||||
}
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 800, clientHeight: 600 } };
|
||||
}
|
||||
throw new Error(`unexpected root command ${method}`);
|
||||
}) as CdpRunner["send"],
|
||||
sendToTarget: vi.fn(async (_target, method, params) => {
|
||||
childCalls.push({ method, params });
|
||||
if (method === "DOM.scrollIntoViewIfNeeded") return {};
|
||||
if (method === "DOM.getContentQuads") {
|
||||
return { quads: [[10, 20, 210, 20, 210, 120, 10, 120]] };
|
||||
}
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 400, clientHeight: 300 } };
|
||||
}
|
||||
if (method === "DOM.getNodeForLocation") {
|
||||
return { backendNodeId: 124, frameId: "child" };
|
||||
}
|
||||
if (method === "DOM.resolveNode") {
|
||||
const id = (params as { backendNodeId?: number }).backendNodeId;
|
||||
return { object: { objectId: id === 123 ? "drop-target" : "hit-target" } };
|
||||
}
|
||||
if (method === "Runtime.callFunctionOn") return { result: { value: true } };
|
||||
if (method === "Input.dispatchDragEvent" || method === "Runtime.releaseObjectGroup") {
|
||||
return {};
|
||||
}
|
||||
throw new Error(`unexpected child command ${method}`);
|
||||
}) as CdpRunner["sendToTarget"],
|
||||
getFrameGraph: vi.fn(async () => ({
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "child",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 10,
|
||||
target: { tabId: 4, sessionId: "child-session" },
|
||||
},
|
||||
],
|
||||
})),
|
||||
};
|
||||
|
||||
const result = await uploadThroughFileDrop({
|
||||
cdp,
|
||||
actionTarget: actionTarget("child", "child-session"),
|
||||
files: ["/stage/one.png"],
|
||||
timeoutMs: 100,
|
||||
withOverlayHidden: withoutOverlay,
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({ x: 110, y: 70 });
|
||||
expect(
|
||||
childCalls
|
||||
.filter((call) => call.method === "Input.dispatchDragEvent")
|
||||
.map((call) => call.params),
|
||||
).toEqual([
|
||||
expect.objectContaining({ type: "dragEnter", x: 110, y: 70 }),
|
||||
expect.objectContaining({ type: "dragOver", x: 110, y: 70 }),
|
||||
expect.objectContaining({ type: "drop", x: 110, y: 70 }),
|
||||
]);
|
||||
expect(rootCalls.some((call) => call.method === "Input.dispatchDragEvent")).toBe(false);
|
||||
});
|
||||
|
||||
it("reports a missing native drag command as unsupported before commit", async () => {
|
||||
const { cdp } = dropCdp({
|
||||
dispatchError: (type) =>
|
||||
type === "dragEnter" ? new Error("'Input.dispatchDragEvent' wasn't found") : undefined,
|
||||
});
|
||||
const result = await uploadThroughFileDrop({
|
||||
cdp,
|
||||
actionTarget: actionTarget(),
|
||||
files: ["/stage/one.png"],
|
||||
timeoutMs: 100,
|
||||
withOverlayHidden: withoutOverlay,
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({
|
||||
code: "unsupported",
|
||||
data: {
|
||||
reason: "upload_mechanism_unsupported",
|
||||
effect_state: "none",
|
||||
phase: "drag_enter",
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("cancels an unresolved drag before returning a pre-commit timeout", async () => {
|
||||
const { cdp, calls } = dropCdp({
|
||||
dispatchError: (type) => (type === "dragEnter" ? new Promise<never>(() => {}) : undefined),
|
||||
});
|
||||
const result = await uploadThroughFileDrop({
|
||||
cdp,
|
||||
actionTarget: actionTarget(),
|
||||
files: ["/stage/one.png"],
|
||||
timeoutMs: 10,
|
||||
withOverlayHidden: withoutOverlay,
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({
|
||||
code: "timeout",
|
||||
data: { reason: "file_drop_failed", effect_state: "none", phase: "drag_enter" },
|
||||
});
|
||||
expect(calls).toContainEqual({
|
||||
method: "Input.dispatchDragEvent",
|
||||
params: expect.objectContaining({ type: "dragCancel" }),
|
||||
});
|
||||
});
|
||||
|
||||
it("marks an unresolved drop unknown and detaches browser state", async () => {
|
||||
const { cdp, detach } = dropCdp({
|
||||
dispatchError: (type) => (type === "drop" ? new Promise<never>(() => {}) : undefined),
|
||||
});
|
||||
const result = await uploadThroughFileDrop({
|
||||
cdp,
|
||||
actionTarget: actionTarget(),
|
||||
files: ["/stage/one.png"],
|
||||
timeoutMs: 10,
|
||||
withOverlayHidden: withoutOverlay,
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({
|
||||
code: "timeout",
|
||||
data: { reason: "file_drop_failed", effect_state: "unknown", phase: "drop" },
|
||||
});
|
||||
expect(detach).toHaveBeenCalledWith(4);
|
||||
});
|
||||
|
||||
it("routes one exact-target download through a browser-relative capability", async () => {
|
||||
const manager = sessions();
|
||||
const ctx = await manager.start("s1");
|
||||
ctx.refStore.set("e3", 123, { tabId: 4 });
|
||||
const onCreated = fakeEvent<(item: chrome.downloads.DownloadItem) => void>();
|
||||
const onChanged = fakeEvent<(delta: chrome.downloads.DownloadDelta) => void>();
|
||||
const onDeterminingFilename =
|
||||
fakeEvent<
|
||||
(
|
||||
item: chrome.downloads.DownloadItem,
|
||||
suggest: (suggestion?: chrome.downloads.DownloadFilenameSuggestion) => void,
|
||||
) => void | true
|
||||
>();
|
||||
const initial = {
|
||||
id: 9,
|
||||
url: "https://example.test/result.zip",
|
||||
finalUrl: "https://example.test/result.zip",
|
||||
filename: "result.zip",
|
||||
state: "in_progress",
|
||||
fileSize: -1,
|
||||
totalBytes: 12,
|
||||
mime: "application/zip",
|
||||
danger: "safe",
|
||||
} as chrome.downloads.DownloadItem;
|
||||
const completed = {
|
||||
...initial,
|
||||
filename: "/profile/Downloads/BrowserSkill/tr_1/result.zip",
|
||||
state: "complete",
|
||||
fileSize: 12,
|
||||
} as chrome.downloads.DownloadItem;
|
||||
const downloads: DownloadsApi = {
|
||||
onCreated,
|
||||
onChanged,
|
||||
onDeterminingFilename,
|
||||
search: vi.fn(async () => [completed]),
|
||||
cancel: vi.fn(async () => {}),
|
||||
removeFile: vi.fn(async () => {}),
|
||||
};
|
||||
let cdpEvent: Parameters<NonNullable<CdpRunner["onEvent"]>>[0] | undefined;
|
||||
let suggested: chrome.downloads.DownloadFilenameSuggestion | undefined;
|
||||
const send = vi.fn(async (_tabId: number, method: string, params?: object) => {
|
||||
if (method === "Page.getLayoutMetrics")
|
||||
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
|
||||
if (method === "DOM.getContentQuads") return { quads: [[0, 0, 20, 0, 20, 20, 0, 20]] };
|
||||
if (
|
||||
method === "Input.dispatchMouseEvent" &&
|
||||
(params as { type?: string }).type === "mousePressed"
|
||||
) {
|
||||
cdpEvent?.({ tabId: 4 }, "Page.downloadWillBegin", {
|
||||
url: initial.url,
|
||||
suggestedFilename: "result.zip",
|
||||
});
|
||||
await new Promise<void>((resolve) => {
|
||||
onDeterminingFilename.emit(initial, (value) => {
|
||||
suggested = value;
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
onCreated.emit(completed);
|
||||
}
|
||||
return {};
|
||||
});
|
||||
const cdp: CdpRunner = {
|
||||
send: send as unknown as CdpRunner["send"],
|
||||
onEvent: (handler) => {
|
||||
cdpEvent = handler;
|
||||
return { dispose: vi.fn() };
|
||||
},
|
||||
};
|
||||
|
||||
const result = await handleDownload(
|
||||
manager,
|
||||
{ session_id: "s1", ref: "@e3", browser_relative_dir: "BrowserSkill/tr_1" },
|
||||
{ cdp, tabsApi: tabsApi(), downloads },
|
||||
);
|
||||
|
||||
expect(suggested).toEqual({
|
||||
filename: "BrowserSkill/tr_1/result.zip",
|
||||
conflictAction: "overwrite",
|
||||
});
|
||||
expect(result).toMatchObject({
|
||||
tab_id: 4,
|
||||
suggested_filename: "result.zip",
|
||||
byte_size: 12,
|
||||
browser_path: completed.filename,
|
||||
});
|
||||
});
|
||||
|
||||
it("does not claim a download without an intent from the exact target", async () => {
|
||||
const onCreated = fakeEvent<(item: chrome.downloads.DownloadItem) => void>();
|
||||
const onChanged = fakeEvent<(delta: chrome.downloads.DownloadDelta) => void>();
|
||||
const onDeterminingFilename =
|
||||
fakeEvent<
|
||||
(
|
||||
item: chrome.downloads.DownloadItem,
|
||||
suggest: (suggestion?: chrome.downloads.DownloadFilenameSuggestion) => void,
|
||||
) => void | true
|
||||
>();
|
||||
const downloads: DownloadsApi = {
|
||||
onCreated,
|
||||
onChanged,
|
||||
onDeterminingFilename,
|
||||
search: vi.fn(async () => []),
|
||||
cancel: vi.fn(async () => {}),
|
||||
removeFile: vi.fn(async () => {}),
|
||||
};
|
||||
let cdpEvent: Parameters<NonNullable<CdpRunner["onEvent"]>>[0] | undefined;
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async () => ({})) as unknown as CdpRunner["send"],
|
||||
onEvent: (handler) => {
|
||||
cdpEvent = handler;
|
||||
return { dispose: vi.fn() };
|
||||
},
|
||||
};
|
||||
const unrelated = {
|
||||
id: 17,
|
||||
url: "https://example.test/unrelated.zip",
|
||||
finalUrl: "https://example.test/unrelated.zip",
|
||||
filename: "unrelated.zip",
|
||||
state: "in_progress",
|
||||
} as chrome.downloads.DownloadItem;
|
||||
let defaultSuggestionCalled = false;
|
||||
|
||||
const result = await captureBrowserDownload({
|
||||
cdp,
|
||||
target: { tabId: 4, sessionId: "expected-child" },
|
||||
downloads,
|
||||
browserRelativeDir: "BrowserSkill/tr_1",
|
||||
timeoutMs: 5,
|
||||
trigger: async () => {
|
||||
cdpEvent?.({ tabId: 4, sessionId: "other-child" }, "Page.downloadWillBegin", {
|
||||
url: unrelated.url,
|
||||
suggestedFilename: unrelated.filename,
|
||||
});
|
||||
onDeterminingFilename.emit(unrelated, (suggestion) => {
|
||||
defaultSuggestionCalled = suggestion === undefined;
|
||||
});
|
||||
return { tab_id: 4, x: 10, y: 10 };
|
||||
},
|
||||
});
|
||||
|
||||
expect(defaultSuggestionCalled).toBe(true);
|
||||
expect(downloads.cancel).not.toHaveBeenCalled();
|
||||
expect(result).toMatchObject({
|
||||
code: "cdp_failed",
|
||||
data: { reason: "download_capture_failed" },
|
||||
});
|
||||
});
|
||||
|
||||
it("correlates a filename candidate that arrives before the CDP intent", async () => {
|
||||
const onCreated = fakeEvent<(item: chrome.downloads.DownloadItem) => void>();
|
||||
const onChanged = fakeEvent<(delta: chrome.downloads.DownloadDelta) => void>();
|
||||
const onDeterminingFilename =
|
||||
fakeEvent<
|
||||
(
|
||||
item: chrome.downloads.DownloadItem,
|
||||
suggest: (suggestion?: chrome.downloads.DownloadFilenameSuggestion) => void,
|
||||
) => void | true
|
||||
>();
|
||||
const initial = {
|
||||
id: 21,
|
||||
url: "https://example.test/candidate-first.bin",
|
||||
finalUrl: "https://example.test/candidate-first.bin",
|
||||
filename: "candidate-first.bin",
|
||||
state: "in_progress",
|
||||
fileSize: -1,
|
||||
totalBytes: 4,
|
||||
bytesReceived: 0,
|
||||
} as chrome.downloads.DownloadItem;
|
||||
const complete = {
|
||||
...initial,
|
||||
state: "complete",
|
||||
fileSize: 4,
|
||||
} as chrome.downloads.DownloadItem;
|
||||
const downloads: DownloadsApi = {
|
||||
onCreated,
|
||||
onChanged,
|
||||
onDeterminingFilename,
|
||||
search: vi.fn(async () => [complete]),
|
||||
cancel: vi.fn(async () => {}),
|
||||
removeFile: vi.fn(async () => {}),
|
||||
};
|
||||
let cdpEvent: Parameters<NonNullable<CdpRunner["onEvent"]>>[0] | undefined;
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async () => ({})) as CdpRunner["send"],
|
||||
onEvent: (handler) => {
|
||||
cdpEvent = handler;
|
||||
return { dispose: vi.fn() };
|
||||
},
|
||||
};
|
||||
let suggestion: chrome.downloads.DownloadFilenameSuggestion | undefined;
|
||||
|
||||
const result = await captureBrowserDownload({
|
||||
cdp,
|
||||
target: { tabId: 4 },
|
||||
downloads,
|
||||
browserRelativeDir: "BrowserSkill/tr_21",
|
||||
timeoutMs: 1_000,
|
||||
trigger: async () => {
|
||||
const suggested = new Promise<void>((resolve) => {
|
||||
onDeterminingFilename.emit(initial, (value) => {
|
||||
suggestion = value;
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
cdpEvent?.({ tabId: 4 }, "Page.downloadWillBegin", {
|
||||
url: initial.url,
|
||||
suggestedFilename: initial.filename,
|
||||
});
|
||||
await suggested;
|
||||
onCreated.emit(complete);
|
||||
return { tab_id: 4, x: 10, y: 10 };
|
||||
},
|
||||
});
|
||||
|
||||
expect(suggestion).toEqual({
|
||||
filename: "BrowserSkill/tr_21/candidate-first.bin",
|
||||
conflictAction: "overwrite",
|
||||
});
|
||||
expect(result).toMatchObject({ item: { id: 21, state: "complete" } });
|
||||
});
|
||||
|
||||
it("removes a completed download when the final size exceeds the limit", async () => {
|
||||
const onCreated = fakeEvent<(item: chrome.downloads.DownloadItem) => void>();
|
||||
const onChanged = fakeEvent<(delta: chrome.downloads.DownloadDelta) => void>();
|
||||
const onDeterminingFilename =
|
||||
fakeEvent<
|
||||
(
|
||||
item: chrome.downloads.DownloadItem,
|
||||
suggest: (suggestion?: chrome.downloads.DownloadFilenameSuggestion) => void,
|
||||
) => void | true
|
||||
>();
|
||||
const initial = {
|
||||
id: 22,
|
||||
url: "https://example.test/oversized.bin",
|
||||
finalUrl: "https://example.test/oversized.bin",
|
||||
filename: "oversized.bin",
|
||||
state: "in_progress",
|
||||
fileSize: -1,
|
||||
totalBytes: -1,
|
||||
bytesReceived: 0,
|
||||
} as chrome.downloads.DownloadItem;
|
||||
const complete = {
|
||||
...initial,
|
||||
filename: "/profile/Downloads/BrowserSkill/tr_22/oversized.bin",
|
||||
state: "complete",
|
||||
fileSize: 8,
|
||||
totalBytes: 8,
|
||||
bytesReceived: 8,
|
||||
} as chrome.downloads.DownloadItem;
|
||||
const downloads: DownloadsApi = {
|
||||
onCreated,
|
||||
onChanged,
|
||||
onDeterminingFilename,
|
||||
search: vi.fn(async () => [complete]),
|
||||
cancel: vi.fn(async () => {}),
|
||||
removeFile: vi.fn(async () => {}),
|
||||
};
|
||||
let cdpEvent: Parameters<NonNullable<CdpRunner["onEvent"]>>[0] | undefined;
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async () => ({})) as CdpRunner["send"],
|
||||
onEvent: (handler) => {
|
||||
cdpEvent = handler;
|
||||
return { dispose: vi.fn() };
|
||||
},
|
||||
};
|
||||
|
||||
const result = await captureBrowserDownload({
|
||||
cdp,
|
||||
target: { tabId: 4 },
|
||||
downloads,
|
||||
browserRelativeDir: "BrowserSkill/tr_22",
|
||||
maxByteSize: 4,
|
||||
timeoutMs: 1_000,
|
||||
trigger: async () => {
|
||||
cdpEvent?.({ tabId: 4 }, "Page.downloadWillBegin", {
|
||||
url: initial.url,
|
||||
suggestedFilename: initial.filename,
|
||||
});
|
||||
await new Promise<void>((resolve) => {
|
||||
onDeterminingFilename.emit(initial, () => resolve());
|
||||
});
|
||||
onCreated.emit(initial);
|
||||
onChanged.emit({ id: initial.id, state: { current: "complete" } });
|
||||
return { tab_id: 4, x: 10, y: 10 };
|
||||
},
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({
|
||||
code: "cdp_failed",
|
||||
data: { reason: "download_capture_failed", effect_state: "committed" },
|
||||
});
|
||||
expect(downloads.removeFile).toHaveBeenCalledWith(initial.id);
|
||||
expect(downloads.cancel).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("reconciles a download that completes while cancellation is being requested", async () => {
|
||||
const onCreated = fakeEvent<(item: chrome.downloads.DownloadItem) => void>();
|
||||
const onChanged = fakeEvent<(delta: chrome.downloads.DownloadDelta) => void>();
|
||||
const onDeterminingFilename =
|
||||
fakeEvent<
|
||||
(
|
||||
item: chrome.downloads.DownloadItem,
|
||||
suggest: (suggestion?: chrome.downloads.DownloadFilenameSuggestion) => void,
|
||||
) => void | true
|
||||
>();
|
||||
const initial = {
|
||||
id: 23,
|
||||
url: "https://example.test/racing.bin",
|
||||
finalUrl: "https://example.test/racing.bin",
|
||||
filename: "racing.bin",
|
||||
state: "in_progress",
|
||||
fileSize: -1,
|
||||
totalBytes: -1,
|
||||
bytesReceived: 0,
|
||||
} as chrome.downloads.DownloadItem;
|
||||
const complete = {
|
||||
...initial,
|
||||
filename: "/profile/Downloads/BrowserSkill/tr_23/racing.bin",
|
||||
state: "complete",
|
||||
fileSize: 4,
|
||||
totalBytes: 4,
|
||||
bytesReceived: 4,
|
||||
} as chrome.downloads.DownloadItem;
|
||||
const downloads: DownloadsApi = {
|
||||
onCreated,
|
||||
onChanged,
|
||||
onDeterminingFilename,
|
||||
search: vi.fn().mockResolvedValueOnce([initial]).mockResolvedValueOnce([complete]),
|
||||
cancel: vi.fn(async () => {
|
||||
throw new Error("download already complete");
|
||||
}),
|
||||
removeFile: vi.fn(async () => {}),
|
||||
};
|
||||
let cdpEvent: Parameters<NonNullable<CdpRunner["onEvent"]>>[0] | undefined;
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async () => ({})) as CdpRunner["send"],
|
||||
onEvent: (handler) => {
|
||||
cdpEvent = handler;
|
||||
return { dispose: vi.fn() };
|
||||
},
|
||||
};
|
||||
|
||||
const result = await captureBrowserDownload({
|
||||
cdp,
|
||||
target: { tabId: 4 },
|
||||
downloads,
|
||||
browserRelativeDir: "BrowserSkill/tr_23",
|
||||
timeoutMs: 80,
|
||||
trigger: async () => {
|
||||
cdpEvent?.({ tabId: 4 }, "Page.downloadWillBegin", {
|
||||
url: initial.url,
|
||||
suggestedFilename: initial.filename,
|
||||
});
|
||||
await new Promise<void>((resolve) => {
|
||||
onDeterminingFilename.emit(initial, () => resolve());
|
||||
});
|
||||
onCreated.emit(initial);
|
||||
return { tab_id: 4, x: 10, y: 10 };
|
||||
},
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({
|
||||
code: "cdp_failed",
|
||||
data: { reason: "download_capture_failed", effect_state: "committed" },
|
||||
});
|
||||
expect(downloads.cancel).toHaveBeenCalledWith(initial.id);
|
||||
expect(downloads.removeFile).toHaveBeenCalledWith(initial.id);
|
||||
expect(result).not.toMatchObject({ data: { cleanup_state: "failed" } });
|
||||
});
|
||||
|
||||
it("rejects ambiguous attribution without cancelling either unclaimed download", async () => {
|
||||
const onCreated = fakeEvent<(item: chrome.downloads.DownloadItem) => void>();
|
||||
const onChanged = fakeEvent<(delta: chrome.downloads.DownloadDelta) => void>();
|
||||
const onDeterminingFilename =
|
||||
fakeEvent<
|
||||
(
|
||||
item: chrome.downloads.DownloadItem,
|
||||
suggest: (suggestion?: chrome.downloads.DownloadFilenameSuggestion) => void,
|
||||
) => void | true
|
||||
>();
|
||||
const downloads: DownloadsApi = {
|
||||
onCreated,
|
||||
onChanged,
|
||||
onDeterminingFilename,
|
||||
search: vi.fn(async () => []),
|
||||
cancel: vi.fn(async () => {}),
|
||||
removeFile: vi.fn(async () => {}),
|
||||
};
|
||||
let cdpEvent: Parameters<NonNullable<CdpRunner["onEvent"]>>[0] | undefined;
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async () => ({})) as CdpRunner["send"],
|
||||
onEvent: (handler) => {
|
||||
cdpEvent = handler;
|
||||
return { dispose: vi.fn() };
|
||||
},
|
||||
};
|
||||
const defaults: number[] = [];
|
||||
const candidate = (id: number) =>
|
||||
({
|
||||
id,
|
||||
url: "https://example.test/same.bin",
|
||||
finalUrl: "https://example.test/same.bin",
|
||||
filename: "same.bin",
|
||||
state: "in_progress",
|
||||
}) as chrome.downloads.DownloadItem;
|
||||
|
||||
const result = await captureBrowserDownload({
|
||||
cdp,
|
||||
target: { tabId: 4 },
|
||||
downloads,
|
||||
browserRelativeDir: "BrowserSkill/tr_ambiguous",
|
||||
timeoutMs: 100,
|
||||
trigger: async () => {
|
||||
cdpEvent?.({ tabId: 4 }, "Page.downloadWillBegin", {
|
||||
url: "https://example.test/same.bin",
|
||||
suggestedFilename: "same.bin",
|
||||
});
|
||||
for (const id of [31, 32]) {
|
||||
onDeterminingFilename.emit(candidate(id), (value) => {
|
||||
if (value === undefined) defaults.push(id);
|
||||
});
|
||||
}
|
||||
return { tab_id: 4, x: 10, y: 10 };
|
||||
},
|
||||
});
|
||||
|
||||
expect(defaults.sort()).toEqual([31, 32]);
|
||||
expect(downloads.cancel).not.toHaveBeenCalled();
|
||||
expect(result).toMatchObject({
|
||||
code: "cdp_failed",
|
||||
data: { effect_state: "unknown", phase: "attribution" },
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,573 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { SessionManager } from "@/session-manager/manager";
|
||||
import type { CdpRunner } from "@/tools/shared";
|
||||
import { handleFill } from "../interaction";
|
||||
|
||||
interface ScriptParams {
|
||||
functionDeclaration: string;
|
||||
arguments?: Array<{ value: unknown }>;
|
||||
}
|
||||
|
||||
async function setup(markup = '<input value="old">') {
|
||||
document.body.innerHTML = markup;
|
||||
const element = document.body.firstElementChild as HTMLInputElement;
|
||||
const manager = new SessionManager({
|
||||
agentWindow: {
|
||||
create: async () => 100,
|
||||
remove: async () => {},
|
||||
ensureActiveTab: async () => 4,
|
||||
},
|
||||
});
|
||||
const ctx = await manager.start("aa11");
|
||||
ctx.refStore.set("e1", 12, { tabId: 4 });
|
||||
const script = vi.fn(async (params: ScriptParams): Promise<unknown> => {
|
||||
try {
|
||||
const fn = new Function(`return (${params.functionDeclaration})`)();
|
||||
return {
|
||||
result: {
|
||||
value: await fn.apply(
|
||||
element,
|
||||
(params.arguments ?? []).map((arg) => arg.value),
|
||||
),
|
||||
},
|
||||
};
|
||||
} catch {
|
||||
return { exceptionDetails: { text: "test page script failed" } };
|
||||
}
|
||||
});
|
||||
// Number/email controls have a browser caret even though their selection
|
||||
// getters return null. Model that caret instead of assuming it is at the end.
|
||||
const hiddenCaret = { start: 0, end: 0 };
|
||||
const moveToEnd = vi.fn(async () => {
|
||||
if (typeof element.selectionStart === "number") {
|
||||
element.setSelectionRange(element.value.length, element.value.length);
|
||||
} else {
|
||||
hiddenCaret.start = hiddenCaret.end = element.value.length;
|
||||
}
|
||||
});
|
||||
const insert = vi.fn(async (text: string) => {
|
||||
const focused = document.activeElement as HTMLInputElement;
|
||||
if (focused.readOnly || focused.disabled) return;
|
||||
const start = focused.selectionStart ?? hiddenCaret.start;
|
||||
const end = focused.selectionEnd ?? hiddenCaret.end;
|
||||
const value = focused.value.slice(0, start) + text + focused.value.slice(end);
|
||||
focused.value = focused.maxLength >= 0 ? value.slice(0, focused.maxLength) : value;
|
||||
hiddenCaret.start = hiddenCaret.end = Math.min(start + text.length, focused.value.length);
|
||||
focused.dispatchEvent(new Event("input", { bubbles: true }));
|
||||
});
|
||||
const release = vi.fn(async () => ({}));
|
||||
const send = vi.fn(async (_tabId: number, method: string, params?: object) => {
|
||||
switch (method) {
|
||||
case "DOM.describeNode":
|
||||
return {
|
||||
node: {
|
||||
nodeName: element.tagName,
|
||||
attributes: Array.from(element.attributes).flatMap((attr) => [attr.name, attr.value]),
|
||||
},
|
||||
};
|
||||
case "DOM.scrollIntoViewIfNeeded":
|
||||
return {};
|
||||
case "DOM.focus":
|
||||
element.focus();
|
||||
return {};
|
||||
case "DOM.resolveNode":
|
||||
return { object: { objectId: "fill-target" } };
|
||||
case "Runtime.callFunctionOn":
|
||||
return script(params as ScriptParams);
|
||||
case "Input.insertText":
|
||||
await insert((params as { text: string }).text);
|
||||
return {};
|
||||
case "Input.dispatchKeyEvent":
|
||||
if ((params as { commands?: string[] }).commands?.includes("moveToEndOfDocument")) {
|
||||
await moveToEnd();
|
||||
}
|
||||
return {};
|
||||
case "Runtime.releaseObject":
|
||||
return release();
|
||||
default:
|
||||
throw new Error(`unexpected ${method}`);
|
||||
}
|
||||
});
|
||||
return {
|
||||
element,
|
||||
hiddenCaret,
|
||||
moveToEnd,
|
||||
script,
|
||||
insert,
|
||||
release,
|
||||
send,
|
||||
fill: (value = "hello", clearBefore = true, signal?: AbortSignal) =>
|
||||
handleFill(
|
||||
manager,
|
||||
{ session_id: "aa11", ref: "e1", value, clear_before: clearBefore },
|
||||
{
|
||||
cdp: { send: send as CdpRunner["send"] },
|
||||
tabsApi: {
|
||||
get: async (id) => ({ id, windowId: 100, active: true }) as chrome.tabs.Tab,
|
||||
query: async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab],
|
||||
},
|
||||
signal,
|
||||
},
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
document.body.innerHTML = "";
|
||||
});
|
||||
|
||||
describe("fill result verification", () => {
|
||||
it.each([
|
||||
"readonly",
|
||||
"disabled",
|
||||
])("preserves a %s field before rejecting it", async (attribute) => {
|
||||
const h = await setup(`<input ${attribute} value="old">`);
|
||||
expect(await h.fill()).toMatchObject({
|
||||
code: "invalid_params",
|
||||
data: { reason: "target_not_fillable" },
|
||||
});
|
||||
expect(h.element.value).toBe("old");
|
||||
expect(h.insert).not.toHaveBeenCalled();
|
||||
expect(h.release).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it.each([
|
||||
"checkbox",
|
||||
"radio",
|
||||
"range",
|
||||
"date",
|
||||
"file",
|
||||
])("rejects an unsupported %s input before clearing", async (type) => {
|
||||
const h = await setup(`<input type="${type}">`);
|
||||
const before = h.element.value;
|
||||
expect(await h.fill()).toMatchObject({
|
||||
code: "invalid_params",
|
||||
data: { reason: "target_not_fillable" },
|
||||
});
|
||||
expect(h.element.value).toBe(before);
|
||||
expect(h.insert).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("rejects an overlong value before clearing", async () => {
|
||||
const h = await setup('<input maxlength="3" value="old">');
|
||||
expect(await h.fill()).toMatchObject({
|
||||
code: "invalid_params",
|
||||
message: /maxlength/,
|
||||
data: { reason: "fill_value_invalid" },
|
||||
});
|
||||
expect(h.element.value).toBe("old");
|
||||
expect(h.insert).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each([
|
||||
true,
|
||||
false,
|
||||
])("classifies a DOM.focus rejection on a disabled=%s target", async (disabled) => {
|
||||
const h = await setup(`<input ${disabled ? "disabled" : ""} value="old">`);
|
||||
const send = h.send.getMockImplementation()!;
|
||||
h.send.mockImplementation(async (tabId, method, params) => {
|
||||
if (method === "DOM.focus") throw new Error("Element is not focusable");
|
||||
return send(tabId, method, params);
|
||||
});
|
||||
expect(await h.fill()).toMatchObject({
|
||||
code: disabled ? "invalid_params" : "cdp_failed",
|
||||
data: { reason: disabled ? "target_not_fillable" : "fill_failed" },
|
||||
});
|
||||
expect(h.element.value).toBe("old");
|
||||
expect(h.insert).not.toHaveBeenCalled();
|
||||
expect(h.release).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("fills a background document without requiring foreground browser focus", async () => {
|
||||
const h = await setup();
|
||||
vi.spyOn(document, "hasFocus").mockReturnValue(false);
|
||||
expect(await h.fill()).toMatchObject({ value_length: 5 });
|
||||
expect(h.element.value).toBe("hello");
|
||||
});
|
||||
|
||||
it("restores focus once after clearing without typing into another field", async () => {
|
||||
const h = await setup();
|
||||
const other = document.createElement("input");
|
||||
document.body.append(other);
|
||||
h.element.addEventListener("input", () => other.focus());
|
||||
expect(await h.fill()).toMatchObject({ value_length: 5 });
|
||||
expect(h.element.value).toBe("hello");
|
||||
expect(other.value).toBe("");
|
||||
expect(h.send.mock.calls.filter(([, method]) => method === "DOM.focus")).toHaveLength(2);
|
||||
});
|
||||
|
||||
it.each([
|
||||
"focus",
|
||||
"value",
|
||||
])("rechecks %s after restoring focus, without retrying again", async (action) => {
|
||||
const h = await setup();
|
||||
const other = document.createElement("input");
|
||||
document.body.append(other);
|
||||
h.element.addEventListener(
|
||||
"input",
|
||||
() => {
|
||||
other.focus();
|
||||
h.element.addEventListener("focus", () => {
|
||||
if (action === "focus") queueMicrotask(() => other.focus());
|
||||
else h.element.value = "changed";
|
||||
});
|
||||
},
|
||||
{ once: true },
|
||||
);
|
||||
expect(await h.fill()).toMatchObject({
|
||||
code: "cdp_failed",
|
||||
data: { reason: action === "focus" ? "fill_focus_lost" : "fill_target_changed" },
|
||||
});
|
||||
expect(other.value).toBe("");
|
||||
expect(h.insert).not.toHaveBeenCalled();
|
||||
expect(h.send.mock.calls.filter(([, method]) => method === "DOM.focus")).toHaveLength(2);
|
||||
});
|
||||
|
||||
it.each([
|
||||
"remove",
|
||||
"readonly",
|
||||
"restore",
|
||||
"microtask-restore",
|
||||
])("stops when clearing causes the page to %s the target", async (action) => {
|
||||
const h = await setup();
|
||||
h.element.addEventListener("input", () => {
|
||||
if (action === "remove") h.element.remove();
|
||||
if (action === "readonly") h.element.readOnly = true;
|
||||
if (action === "restore") h.element.value = "restored";
|
||||
if (action === "microtask-restore")
|
||||
queueMicrotask(() => {
|
||||
h.element.value = "restored";
|
||||
});
|
||||
});
|
||||
expect(await h.fill()).toMatchObject({
|
||||
code: "cdp_failed",
|
||||
data: { reason: "fill_target_changed" },
|
||||
});
|
||||
expect(h.insert).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("fails when insertText is acknowledged without changing the value", async () => {
|
||||
const h = await setup();
|
||||
h.insert.mockImplementation(async () => {});
|
||||
expect(await h.fill()).toMatchObject({
|
||||
code: "cdp_failed",
|
||||
data: { reason: "fill_value_mismatch" },
|
||||
});
|
||||
expect(h.release).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("detects truncation when maxlength changes after preparation", async () => {
|
||||
const h = await setup();
|
||||
h.element.addEventListener(
|
||||
"input",
|
||||
() => {
|
||||
h.element.maxLength = 3;
|
||||
},
|
||||
{ once: true },
|
||||
);
|
||||
expect(await h.fill()).toMatchObject({
|
||||
code: "cdp_failed",
|
||||
data: { reason: "fill_value_mismatch" },
|
||||
});
|
||||
expect(h.element.value).toBe("hel");
|
||||
});
|
||||
|
||||
it.each([
|
||||
"input",
|
||||
"change",
|
||||
"microtask-change",
|
||||
])("detects a value reverted by the %s handler", async (event) => {
|
||||
const h = await setup();
|
||||
h.element.addEventListener(event === "microtask-change" ? "change" : event, () => {
|
||||
if (event === "microtask-change")
|
||||
queueMicrotask(() => {
|
||||
h.element.value = "";
|
||||
});
|
||||
else h.element.value = "";
|
||||
});
|
||||
expect(await h.fill()).toMatchObject({ code: "cdp_failed", message: /expected value/ });
|
||||
});
|
||||
|
||||
it("compares content rather than length without including field values in the error", async () => {
|
||||
const h = await setup('<input type="password">');
|
||||
h.element.addEventListener("change", () => {
|
||||
h.element.value = "other-secret";
|
||||
});
|
||||
const result = await h.fill("given-secret");
|
||||
expect(result).toMatchObject({ code: "cdp_failed", message: /expected value/ });
|
||||
expect(JSON.stringify(result)).not.toMatch(/given-secret|other-secret/);
|
||||
});
|
||||
|
||||
it("does not report success for a target replaced by a change handler", async () => {
|
||||
const h = await setup();
|
||||
h.element.addEventListener("change", () => h.element.replaceWith(h.element.cloneNode()));
|
||||
expect(await h.fill()).toMatchObject({
|
||||
code: "cdp_failed",
|
||||
data: { reason: "fill_target_changed" },
|
||||
});
|
||||
});
|
||||
|
||||
it.each([
|
||||
1, 2, 3, 4, 5,
|
||||
])("rejects script exceptions at phase %s and releases the target", async (phase) => {
|
||||
const h = await setup();
|
||||
vi.spyOn(document, "hasFocus").mockReturnValue(false);
|
||||
const run = h.script.getMockImplementation()!;
|
||||
let count = 0;
|
||||
h.script.mockImplementation(async (params) =>
|
||||
++count === phase ? { exceptionDetails: { text: "page secret" } } : run(params),
|
||||
);
|
||||
const result = await h.fill();
|
||||
expect(result).toMatchObject({ code: "cdp_failed", data: { reason: "fill_failed" } });
|
||||
expect(JSON.stringify(result)).not.toContain("page secret");
|
||||
if (phase <= 3) expect(h.insert).not.toHaveBeenCalled();
|
||||
expect(h.release).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it.each([1, 2, 3, 5])("rejects missing results at phase %s", async (phase) => {
|
||||
const h = await setup();
|
||||
vi.spyOn(document, "hasFocus").mockReturnValue(false);
|
||||
const run = h.script.getMockImplementation()!;
|
||||
let count = 0;
|
||||
h.script.mockImplementation(async (params) =>
|
||||
++count === phase ? { result: { type: "undefined" } } : run(params),
|
||||
);
|
||||
expect(await h.fill()).toMatchObject({ code: "cdp_failed" });
|
||||
if (phase <= 3) expect(h.insert).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each(["input", "change"])("checks state after nested microtasks from %s", async (event) => {
|
||||
const h = await setup();
|
||||
h.element.addEventListener(event, () => {
|
||||
queueMicrotask(() =>
|
||||
queueMicrotask(() => {
|
||||
h.element.value = "reverted";
|
||||
}),
|
||||
);
|
||||
});
|
||||
expect(await h.fill()).toMatchObject({ code: "cdp_failed" });
|
||||
if (event === "input") expect(h.insert).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each([
|
||||
"<input>",
|
||||
'<input type="password">',
|
||||
"<textarea></textarea>",
|
||||
])("verifies successful text in %s", async (markup) => {
|
||||
const h = await setup(markup);
|
||||
expect(await h.fill("你好🙂")).toMatchObject({ value_length: 4 });
|
||||
expect(h.element.value).toBe("你好🙂");
|
||||
expect(h.release).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("verifies append against the old value plus the requested text", async () => {
|
||||
const h = await setup();
|
||||
h.element.focus();
|
||||
h.element.setSelectionRange(3, 3);
|
||||
expect(await h.fill("!", false)).toMatchObject({ value_length: 4 });
|
||||
expect(h.element.value).toBe("old!");
|
||||
});
|
||||
|
||||
it.each([
|
||||
["<textarea></textarea>", "a\r\nb", "a\nb"],
|
||||
["<input>", "a\nb", "a b"],
|
||||
["<input>", "a\r\nb", "a b"],
|
||||
[
|
||||
'<input type="email" multiple>',
|
||||
" a@example.com , b@example.com ",
|
||||
"a@example.com,b@example.com",
|
||||
],
|
||||
])("accepts native normalization for %s", async (markup, requested, normalized) => {
|
||||
if (markup.startsWith("<textarea")) {
|
||||
// Happy DOM does not implement Chrome's textarea line-ending normalization.
|
||||
const setter = Object.getOwnPropertyDescriptor(HTMLTextAreaElement.prototype, "value")!.set!;
|
||||
vi.spyOn(HTMLTextAreaElement.prototype, "value", "set").mockImplementation(function (
|
||||
this: HTMLTextAreaElement,
|
||||
value,
|
||||
) {
|
||||
setter.call(this, value.replace(/\r\n?/g, "\n"));
|
||||
});
|
||||
}
|
||||
const h = await setup(markup);
|
||||
h.insert.mockImplementation(async () => {
|
||||
h.element.value = normalized;
|
||||
});
|
||||
expect(await h.fill(requested)).toMatchObject({ value_length: normalized.length });
|
||||
});
|
||||
|
||||
it("does not count an editable padding break as an extra typed character", async () => {
|
||||
const h = await setup('<div contenteditable="true" tabindex="0"></div>');
|
||||
// These two layout/editability properties are missing in Happy DOM.
|
||||
let innerText = "";
|
||||
Object.defineProperty(h.element, "isContentEditable", { get: () => true });
|
||||
Object.defineProperty(h.element, "innerText", { get: () => innerText });
|
||||
h.insert.mockImplementation(async () => {
|
||||
h.element.innerHTML = "one<div><br></div>";
|
||||
innerText = "one\n\n";
|
||||
});
|
||||
expect(await h.fill("one\n")).toMatchObject({ value_length: 4 });
|
||||
});
|
||||
|
||||
it.each([
|
||||
[0, 0],
|
||||
[1, 1],
|
||||
[0, 3],
|
||||
])("appends after moving selection %s:%s to the end", async (start, end) => {
|
||||
const h = await setup();
|
||||
h.element.focus();
|
||||
h.element.setSelectionRange(start, end);
|
||||
expect(await h.fill("🙂", false)).toMatchObject({ value_length: 5 });
|
||||
expect(h.element.value).toBe("old🙂");
|
||||
});
|
||||
|
||||
it.each(
|
||||
["number", "email"].flatMap((type) =>
|
||||
[
|
||||
[0, 0],
|
||||
[1, 1],
|
||||
[0, 2],
|
||||
].map(([start, end]) => ({ type, start, end })),
|
||||
),
|
||||
)("appends to $type from selection $start:$end without using its unsupported API", async ({
|
||||
type,
|
||||
start,
|
||||
end,
|
||||
}) => {
|
||||
const h = await setup(`<input type="${type}" value="12">`);
|
||||
Object.assign(h.hiddenCaret, { start, end });
|
||||
vi.spyOn(h.element, "selectionStart", "get").mockReturnValue(null);
|
||||
vi.spyOn(h.element, "selectionEnd", "get").mockReturnValue(null);
|
||||
const selection = vi.spyOn(h.element, "setSelectionRange").mockImplementation(() => {
|
||||
throw new DOMException("unsupported", "InvalidStateError");
|
||||
});
|
||||
expect(await h.fill("3", false)).toMatchObject({ value_length: 3 });
|
||||
expect(h.element.value).toBe("123");
|
||||
expect(selection).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each([
|
||||
"number",
|
||||
"email",
|
||||
])("does not report successful %s append when the caret command is ignored", async (type) => {
|
||||
const h = await setup(`<input type="${type}" value="12">`);
|
||||
vi.spyOn(h.element, "selectionStart", "get").mockReturnValue(null);
|
||||
vi.spyOn(h.element, "selectionEnd", "get").mockReturnValue(null);
|
||||
h.moveToEnd.mockResolvedValueOnce(undefined);
|
||||
expect(await h.fill("3", false)).toMatchObject({
|
||||
code: "cdp_failed",
|
||||
data: { reason: "fill_value_mismatch" },
|
||||
});
|
||||
expect(h.element.value).toBe("312");
|
||||
expect(h.insert).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("rejects invalid number text before clearing", async () => {
|
||||
const h = await setup('<input type="number" value="12">');
|
||||
expect(await h.fill("abc")).toMatchObject({
|
||||
code: "invalid_params",
|
||||
data: { reason: "fill_value_invalid" },
|
||||
});
|
||||
expect(h.element.value).toBe("12");
|
||||
expect(h.insert).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("reports page formatting as unconfirmed without retrying the write", async () => {
|
||||
const h = await setup();
|
||||
h.element.addEventListener("change", () => {
|
||||
h.element.value = h.element.value.toUpperCase();
|
||||
});
|
||||
expect(await h.fill()).toMatchObject({
|
||||
code: "cdp_failed",
|
||||
data: { reason: "fill_value_mismatch" },
|
||||
});
|
||||
expect(h.element.value).toBe("HELLO");
|
||||
expect(h.insert).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it.each([
|
||||
["focus", false],
|
||||
["focus", true],
|
||||
["value", false],
|
||||
["value", true],
|
||||
["cancel", false],
|
||||
["cancel", true],
|
||||
["throw", false],
|
||||
["throw", true],
|
||||
] as const)("stops when positioning triggers %s (clear=%s) and releases the key", async (action, clearBefore) => {
|
||||
const h = await setup();
|
||||
vi.spyOn(document, "hasFocus").mockReturnValue(false);
|
||||
const controller = new AbortController();
|
||||
const other = document.createElement("input");
|
||||
document.body.append(other);
|
||||
const send = h.send.getMockImplementation()!;
|
||||
h.send.mockImplementation(async (tabId, method, params) => {
|
||||
const reply = await send(tabId, method, params);
|
||||
if (
|
||||
method === "Input.dispatchKeyEvent" &&
|
||||
(params as { type: string }).type === "rawKeyDown"
|
||||
) {
|
||||
if (action === "focus") other.focus();
|
||||
if (action === "value") h.element.value = "changed";
|
||||
if (action === "cancel") controller.abort();
|
||||
if (action === "throw") throw new Error("dispatch failed");
|
||||
}
|
||||
return reply;
|
||||
});
|
||||
const result = await h.fill("!", clearBefore, controller.signal);
|
||||
expect(result).toMatchObject({ code: action === "cancel" ? "cancelled" : "cdp_failed" });
|
||||
expect(h.insert).not.toHaveBeenCalled();
|
||||
expect(other.value).toBe("");
|
||||
expect(h.send).toHaveBeenCalledWith(
|
||||
4,
|
||||
"Input.dispatchKeyEvent",
|
||||
expect.objectContaining({ type: "keyUp" }),
|
||||
);
|
||||
expect(h.release).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it.each([true, false])("verifies an empty request with clear_before=%s", async (clearBefore) => {
|
||||
const h = await setup();
|
||||
expect(await h.fill("", clearBefore)).toMatchObject({ value_length: clearBefore ? 0 : 3 });
|
||||
expect(h.element.value).toBe(clearBefore ? "" : "old");
|
||||
expect(h.insert).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("cancels after insertion without sending change events", async () => {
|
||||
const h = await setup();
|
||||
const controller = new AbortController();
|
||||
h.insert.mockImplementation(async () => controller.abort());
|
||||
expect(await h.fill("hello", true, controller.signal)).toMatchObject({ code: "cancelled" });
|
||||
expect(h.script).toHaveBeenCalledTimes(2);
|
||||
expect(h.release).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("cancels after resolving the remote object without clearing the field", async () => {
|
||||
const h = await setup();
|
||||
const controller = new AbortController();
|
||||
const send = h.send.getMockImplementation()!;
|
||||
h.send.mockImplementation(async (tabId, method, params) => {
|
||||
const result = await send(tabId, method, params);
|
||||
if (method === "DOM.resolveNode") controller.abort();
|
||||
return result;
|
||||
});
|
||||
expect(await h.fill("hello", true, controller.signal)).toMatchObject({ code: "cancelled" });
|
||||
expect(h.element.value).toBe("old");
|
||||
expect(h.script).not.toHaveBeenCalled();
|
||||
expect(h.release).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("honors cancellation during verification", async () => {
|
||||
const h = await setup();
|
||||
const controller = new AbortController();
|
||||
h.element.addEventListener("change", () => controller.abort());
|
||||
expect(await h.fill("hello", true, controller.signal)).toMatchObject({ code: "cancelled" });
|
||||
expect(h.release).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("keeps the verified outcome when releasing the remote object fails", async () => {
|
||||
const h = await setup();
|
||||
h.release.mockRejectedValueOnce(new Error("target closed"));
|
||||
expect(await h.fill()).toMatchObject({ value_length: 5 });
|
||||
});
|
||||
});
|
||||
@@ -169,6 +169,7 @@ describe("frame geometry projection", () => {
|
||||
expect(geometry).toMatchObject({
|
||||
topBounds: { x: 100, y: 100, width: 200, height: 100 },
|
||||
actionPoint: { x: 200, y: 150 },
|
||||
targetActionPoint: { x: 100, y: 50 },
|
||||
});
|
||||
});
|
||||
|
||||
@@ -261,6 +262,7 @@ describe("frame geometry projection", () => {
|
||||
expect(geometry).toMatchObject({
|
||||
topBounds: { x: 100, y: 100, width: 200, height: 100 },
|
||||
actionPoint: { x: 200, y: 150 },
|
||||
targetActionPoint: { x: 200, y: 150 },
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -21,7 +21,7 @@ function fakeAgentWindow(ids: number[]) {
|
||||
return id;
|
||||
}),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -456,30 +456,6 @@ describe("handleClick", () => {
|
||||
expect(res).toMatchObject({ code: "not_found", data: { reason: "selector_not_found" } });
|
||||
});
|
||||
|
||||
it("returns invalid_selector when Chrome rejects non-CSS selector syntax", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
await sm.start("aa11");
|
||||
const fake = makeFakeCdp({
|
||||
"DOM.getDocument": () => ({ root: { nodeId: 1 } }),
|
||||
"DOM.querySelector": () => {
|
||||
throw new Error('{"code":-32000,"message":"DOM Error while querying"}');
|
||||
},
|
||||
});
|
||||
const res = await handleClick(
|
||||
sm,
|
||||
{ session_id: "aa11", selector: "text=下午 12~18点" },
|
||||
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
||||
);
|
||||
expect(res).toMatchObject({
|
||||
code: "invalid_params",
|
||||
message: "selector is not valid CSS: text=下午 12~18点",
|
||||
data: {
|
||||
reason: "invalid_selector",
|
||||
selector: "text=下午 12~18点",
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("respects the AbortSignal", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
@@ -564,6 +540,21 @@ describe("handleHover", () => {
|
||||
});
|
||||
});
|
||||
|
||||
function successfulFillScript(params: unknown) {
|
||||
const script = params as { arguments?: Array<{ value: unknown }>; functionDeclaration: string };
|
||||
const args = script.arguments ?? [];
|
||||
return {
|
||||
result: {
|
||||
value:
|
||||
args.length === 2
|
||||
? { before: "", expected: args[0].value }
|
||||
: script.functionDeclaration.startsWith("function(expected)")
|
||||
? { connected: true, matches: true, valueLength: String(args[0].value).length }
|
||||
: "ready",
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe("handleFill", () => {
|
||||
it("returns not_found for unknown ref", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
@@ -610,7 +601,8 @@ describe("handleFill", () => {
|
||||
"DOM.scrollIntoViewIfNeeded": () => ({}),
|
||||
"DOM.focus": () => ({}),
|
||||
"DOM.resolveNode": () => ({ object: { objectId: "obj-1" } }),
|
||||
"Runtime.callFunctionOn": () => ({ result: { type: "undefined" } }),
|
||||
"Runtime.callFunctionOn": successfulFillScript,
|
||||
"Runtime.releaseObject": () => ({}),
|
||||
"Input.insertText": () => ({}),
|
||||
});
|
||||
const res = await handleFill(
|
||||
@@ -624,13 +616,12 @@ describe("handleFill", () => {
|
||||
expect(res.used_ref).toBe("e1");
|
||||
const insert = fake.sent.find((c) => c.method === "Input.insertText");
|
||||
expect(insert?.params).toEqual({ text: "hello" });
|
||||
// clear_before defaults to true → callFunctionOn invoked once to
|
||||
// clear, once to fire input/change after typing.
|
||||
// Foreground replacement needs no extra caret-positioning round trip.
|
||||
const callFns = fake.sent.filter((c) => c.method === "Runtime.callFunctionOn");
|
||||
expect(callFns.length).toBeGreaterThanOrEqual(2);
|
||||
expect(callFns).toHaveLength(4);
|
||||
});
|
||||
|
||||
it("clear_before=false skips the wipe call", async () => {
|
||||
it("passes clear_before=false to preparation and verifies the result", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.refStore.set("e1", 1, { tabId: 4 });
|
||||
@@ -642,12 +633,12 @@ describe("handleFill", () => {
|
||||
"DOM.focus": () => ({}),
|
||||
"DOM.resolveNode": () => ({ object: { objectId: "obj-2" } }),
|
||||
"Runtime.callFunctionOn": (p) => {
|
||||
const fn = (p as { functionDeclaration?: string }).functionDeclaration ?? "";
|
||||
// No "this.value = ''" clearing on a no-clear path; only the
|
||||
// post-input dispatchEvent.
|
||||
expect(fn).not.toMatch(/this\.value\s*=\s*''/);
|
||||
return { result: { type: "undefined" } };
|
||||
const args = (p as { arguments?: Array<{ value: unknown }> }).arguments ?? [];
|
||||
if (args.length === 2) expect(args[1].value).toBe(false);
|
||||
return successfulFillScript(p);
|
||||
},
|
||||
"Input.dispatchKeyEvent": () => ({}),
|
||||
"Runtime.releaseObject": () => ({}),
|
||||
"Input.insertText": () => ({}),
|
||||
});
|
||||
const res = await handleFill(
|
||||
@@ -674,7 +665,8 @@ describe("handleFill", () => {
|
||||
"DOM.scrollIntoViewIfNeeded": () => ({}),
|
||||
"DOM.focus": () => ({}),
|
||||
"DOM.resolveNode": () => ({ object: { objectId: "obj-3" } }),
|
||||
"Runtime.callFunctionOn": () => ({ result: { type: "undefined" } }),
|
||||
"Runtime.callFunctionOn": successfulFillScript,
|
||||
"Runtime.releaseObject": () => ({}),
|
||||
"Input.insertText": () => ({}),
|
||||
});
|
||||
const res = await handleFill(
|
||||
|
||||
@@ -0,0 +1,266 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { SessionManager } from "@/session-manager/manager";
|
||||
import type { BrowserNavigationApi } from "../browser-navigation";
|
||||
import { handleNavigate, handleReload } from "../navigation";
|
||||
import type { CdpRunner } from "../shared";
|
||||
|
||||
const denied = "Cannot access a chrome-extension:// URL of different extension";
|
||||
const url = "https://example.test/recovered";
|
||||
|
||||
function event<T extends (...args: never[]) => void>() {
|
||||
const listeners = new Set<T>();
|
||||
return {
|
||||
listeners,
|
||||
addListener: (listener: T) => listeners.add(listener),
|
||||
removeListener: (listener: T) => listeners.delete(listener),
|
||||
fire: (...args: Parameters<T>) => {
|
||||
for (const listener of [...listeners]) listener(...args);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function fixture() {
|
||||
const manager = new SessionManager({
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 100),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 4),
|
||||
},
|
||||
});
|
||||
await manager.start("test");
|
||||
type Listener = Parameters<BrowserNavigationApi["onCommitted"]["addListener"]>[0];
|
||||
const events = {
|
||||
onCommitted: event<Listener>(),
|
||||
onDOMContentLoaded: event<Listener>(),
|
||||
onCompleted: event<Listener>(),
|
||||
onErrorOccurred: event<Listener>(),
|
||||
};
|
||||
const cdpEvents = event<Parameters<NonNullable<CdpRunner["onEvent"]>>[0]>();
|
||||
const state = { blocked: true, destinationBlocked: false, emitIdle: true };
|
||||
const send = vi.fn(async (_tabId: number, method: string) => {
|
||||
if (state.blocked) throw new Error(denied);
|
||||
if (method === "Page.getFrameTree") {
|
||||
return { frameTree: { frame: { id: "main", loaderId: "new-loader" } } };
|
||||
}
|
||||
if (method === "Runtime.evaluate") return { result: { value: "complete" } };
|
||||
if (method === "Page.setLifecycleEventsEnabled" && state.emitIdle) {
|
||||
cdpEvents.fire({ tabId: 4 }, "Page.lifecycleEvent", {
|
||||
frameId: "main",
|
||||
loaderId: "new-loader",
|
||||
name: "networkIdle",
|
||||
});
|
||||
}
|
||||
if (method === "Page.navigate") return { frameId: "main", loaderId: "new-loader" };
|
||||
return {};
|
||||
});
|
||||
const cdp: CdpRunner = {
|
||||
send: send as CdpRunner["send"],
|
||||
onEvent: (listener) => {
|
||||
cdpEvents.addListener(listener);
|
||||
return { dispose: () => cdpEvents.removeListener(listener) };
|
||||
},
|
||||
};
|
||||
const main = { tabId: 4, frameId: 0, documentId: "new-document" };
|
||||
const navigate = async () => {
|
||||
state.blocked = state.destinationBlocked;
|
||||
events.onCommitted.fire(main);
|
||||
events.onDOMContentLoaded.fire(main);
|
||||
events.onCompleted.fire(main);
|
||||
};
|
||||
const browserNavigation = { ...events, update: vi.fn(navigate), reload: vi.fn(navigate) };
|
||||
const tab = { id: 4, windowId: 100, active: true, url } as chrome.tabs.Tab;
|
||||
const tabsApi = { get: vi.fn(async () => tab), query: vi.fn(async () => [tab]) };
|
||||
const deps = { cdp, tabsApi, browserNavigation, defaultTimeoutMs: 1000 };
|
||||
const expectCleanedUp = () => {
|
||||
for (const entry of Object.values(events)) expect(entry.listeners.size).toBe(0);
|
||||
expect(cdpEvents.listeners.size).toBe(0);
|
||||
};
|
||||
return { manager, deps, state, send, events, main, cdpEvents, expectCleanedUp };
|
||||
}
|
||||
|
||||
describe("navigation after Chrome denies extension-frame access", () => {
|
||||
it.each([
|
||||
"load",
|
||||
"commit",
|
||||
"domcontentloaded",
|
||||
"networkidle",
|
||||
] as const)("recovers the same tab with the requested %s checkpoint", async (waitUntil) => {
|
||||
const f = await fixture();
|
||||
const result = await handleNavigate(
|
||||
f.manager,
|
||||
{
|
||||
session_id: "test",
|
||||
url,
|
||||
wait_until: waitUntil,
|
||||
},
|
||||
f.deps,
|
||||
);
|
||||
|
||||
expect(result).toMatchObject({ tab_id: 4, final_url: url, reached: waitUntil });
|
||||
expect(f.deps.browserNavigation.update).toHaveBeenCalledExactlyOnceWith(4, { url });
|
||||
expect(f.send.mock.calls.some(([, method]) => method === "Page.navigate")).toBe(false);
|
||||
expect(f.send).toHaveBeenCalledWith(4, "Page.enable", {});
|
||||
f.expectCleanedUp();
|
||||
});
|
||||
|
||||
it("recovers reload through tabs.reload and preserves hard reload", async () => {
|
||||
const f = await fixture();
|
||||
const result = await handleReload(f.manager, { session_id: "test", hard: true }, f.deps);
|
||||
expect(result).toMatchObject({ tab_id: 4, reached: "load" });
|
||||
expect(f.deps.browserNavigation.reload).toHaveBeenCalledExactlyOnceWith(4, {
|
||||
bypassCache: true,
|
||||
});
|
||||
expect(f.deps.browserNavigation.update).not.toHaveBeenCalled();
|
||||
f.expectCleanedUp();
|
||||
});
|
||||
|
||||
it("reports the restriction when reload leaves the conflicting frame present", async () => {
|
||||
const f = await fixture();
|
||||
f.state.destinationBlocked = true;
|
||||
const result = await handleReload(f.manager, { session_id: "test" }, f.deps);
|
||||
expect(result).toMatchObject({
|
||||
code: "cdp_failed",
|
||||
data: { reason: "cdp_extension_access_denied" },
|
||||
});
|
||||
expect(f.deps.browserNavigation.reload).toHaveBeenCalledOnce();
|
||||
f.expectCleanedUp();
|
||||
});
|
||||
|
||||
it("does not fall back for unrelated debugger failures", async () => {
|
||||
const f = await fixture();
|
||||
f.send.mockRejectedValue(new Error("Another debugger is already attached"));
|
||||
const result = await handleNavigate(f.manager, { session_id: "test", url }, f.deps);
|
||||
expect(result).toMatchObject({
|
||||
code: "cdp_failed",
|
||||
message: "Another debugger is already attached",
|
||||
});
|
||||
expect(f.deps.browserNavigation.update).not.toHaveBeenCalled();
|
||||
f.expectCleanedUp();
|
||||
});
|
||||
|
||||
it("does not replay navigation when a dispatched CDP command fails", async () => {
|
||||
const f = await fixture();
|
||||
f.state.blocked = false;
|
||||
const send = f.send.getMockImplementation()!;
|
||||
f.send.mockImplementation(async (tabId, method) => {
|
||||
if (method === "Page.navigate") throw new Error(denied);
|
||||
return send(tabId, method);
|
||||
});
|
||||
const result = await handleNavigate(f.manager, { session_id: "test", url }, f.deps);
|
||||
expect(result).toMatchObject({
|
||||
code: "cdp_failed",
|
||||
data: { reason: "cdp_extension_access_denied" },
|
||||
});
|
||||
expect(f.deps.browserNavigation.update).not.toHaveBeenCalled();
|
||||
f.expectCleanedUp();
|
||||
});
|
||||
|
||||
it("keeps the Agent Window guard in front of the recovery", async () => {
|
||||
const f = await fixture();
|
||||
f.deps.tabsApi.get.mockResolvedValue({
|
||||
id: 4,
|
||||
windowId: 200,
|
||||
active: true,
|
||||
url,
|
||||
} as chrome.tabs.Tab);
|
||||
const result = await handleNavigate(f.manager, { session_id: "test", tab_id: 4, url }, f.deps);
|
||||
expect(result).toMatchObject({ code: "permission_denied" });
|
||||
expect(f.send).not.toHaveBeenCalled();
|
||||
expect(f.deps.browserNavigation.update).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("ignores old-document, subframe, and other-tab completion events", async () => {
|
||||
const f = await fixture();
|
||||
f.deps.browserNavigation.update.mockImplementation(async () => {});
|
||||
let settled = false;
|
||||
const work = handleNavigate(f.manager, { session_id: "test", url }, f.deps).then((result) => {
|
||||
settled = true;
|
||||
return result;
|
||||
});
|
||||
await vi.waitFor(() => expect(f.deps.browserNavigation.update).toHaveBeenCalledOnce());
|
||||
f.events.onCompleted.fire(f.main);
|
||||
f.events.onCommitted.fire({ ...f.main, tabId: 9 });
|
||||
f.events.onCommitted.fire({ ...f.main, frameId: 1 });
|
||||
f.events.onCompleted.fire(f.main);
|
||||
await Promise.resolve();
|
||||
expect(settled).toBe(false);
|
||||
f.events.onCommitted.fire(f.main);
|
||||
f.events.onCompleted.fire({ ...f.main, documentId: "old-document" });
|
||||
await Promise.resolve();
|
||||
expect(settled).toBe(false);
|
||||
f.state.blocked = false;
|
||||
f.events.onCompleted.fire(f.main);
|
||||
expect(await work).toMatchObject({ reached: "load" });
|
||||
f.expectCleanedUp();
|
||||
});
|
||||
|
||||
it("does not infer network idle from a browser load event", async () => {
|
||||
const f = await fixture();
|
||||
f.state.emitIdle = false;
|
||||
const result = await handleNavigate(
|
||||
f.manager,
|
||||
{
|
||||
session_id: "test",
|
||||
url,
|
||||
wait_until: "networkidle",
|
||||
timeout_ms: 10,
|
||||
},
|
||||
f.deps,
|
||||
);
|
||||
expect(result).toMatchObject({ reached: "timeout" });
|
||||
f.expectCleanedUp();
|
||||
});
|
||||
|
||||
it("cleans up browser listeners on timeout", async () => {
|
||||
const f = await fixture();
|
||||
f.deps.browserNavigation.update.mockImplementation(async () => {});
|
||||
const result = await handleNavigate(
|
||||
f.manager,
|
||||
{ session_id: "test", url, timeout_ms: 10 },
|
||||
f.deps,
|
||||
);
|
||||
expect(result).toMatchObject({ reached: "timeout" });
|
||||
f.expectCleanedUp();
|
||||
});
|
||||
|
||||
it("does not navigate an already-cancelled request", async () => {
|
||||
const f = await fixture();
|
||||
const result = await handleNavigate(
|
||||
f.manager,
|
||||
{ session_id: "test", url },
|
||||
{
|
||||
...f.deps,
|
||||
signal: AbortSignal.abort(),
|
||||
},
|
||||
);
|
||||
expect(result).toMatchObject({ code: "cancelled" });
|
||||
expect(f.deps.browserNavigation.update).not.toHaveBeenCalled();
|
||||
f.expectCleanedUp();
|
||||
});
|
||||
|
||||
it("cleans up browser listeners when cancelled after dispatch", async () => {
|
||||
const f = await fixture();
|
||||
const ac = new AbortController();
|
||||
f.deps.browserNavigation.update.mockImplementation(async () => {
|
||||
ac.abort();
|
||||
});
|
||||
const result = await handleNavigate(
|
||||
f.manager,
|
||||
{ session_id: "test", url },
|
||||
{ ...f.deps, signal: ac.signal },
|
||||
);
|
||||
expect(result).toMatchObject({ code: "cancelled" });
|
||||
expect(f.deps.browserNavigation.update).toHaveBeenCalledOnce();
|
||||
f.expectCleanedUp();
|
||||
});
|
||||
|
||||
it("reports browser navigation errors and releases listeners", async () => {
|
||||
const f = await fixture();
|
||||
f.deps.browserNavigation.update.mockImplementation(async () => {
|
||||
f.events.onErrorOccurred.fire({ ...f.main, error: "net::ERR_NAME_NOT_RESOLVED" });
|
||||
});
|
||||
const result = await handleNavigate(f.manager, { session_id: "test", url }, f.deps);
|
||||
expect(result).toMatchObject({ code: "cdp_failed", message: "net::ERR_NAME_NOT_RESOLVED" });
|
||||
f.expectCleanedUp();
|
||||
});
|
||||
});
|
||||
@@ -21,7 +21,7 @@ function fakeAgentWindow(ids: number[]) {
|
||||
return id;
|
||||
}),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -12,7 +12,7 @@ function fakeAgentWindow(ids: number[]) {
|
||||
return id;
|
||||
}),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -28,7 +28,7 @@ function fakeAgentWindow(ids: number[]) {
|
||||
return id;
|
||||
}),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1904,36 +1904,41 @@ describe("buildVomScene", () => {
|
||||
backendDOMNodeId: 20,
|
||||
},
|
||||
];
|
||||
const scene = buildVomScene(axNodes, {
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
surfaceProbes: [
|
||||
{ triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["Shoes", "Bags"] },
|
||||
],
|
||||
nodes: [
|
||||
{
|
||||
backendNodeId: 10,
|
||||
parentBackendNodeId: null,
|
||||
tag: "body",
|
||||
attrs: {},
|
||||
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 20,
|
||||
parentBackendNodeId: 10,
|
||||
tag: "button",
|
||||
attrs: {},
|
||||
rect: { x: 20, y: 20, w: 120, h: 40 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
],
|
||||
});
|
||||
const scene = buildVomScene(
|
||||
axNodes,
|
||||
{
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
nodes: [
|
||||
{
|
||||
backendNodeId: 10,
|
||||
parentBackendNodeId: null,
|
||||
tag: "body",
|
||||
attrs: {},
|
||||
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 20,
|
||||
parentBackendNodeId: 10,
|
||||
tag: "button",
|
||||
attrs: {},
|
||||
rect: { x: 20, y: 20, w: 120, h: 40 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
surfaceProbes: [
|
||||
{ triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["Shoes", "Bags"] },
|
||||
],
|
||||
},
|
||||
);
|
||||
|
||||
expect(scene.surfaces).toEqual([
|
||||
{ triggerId: 20, triggerAction: "hover", subItems: ["Shoes", "Bags"] },
|
||||
@@ -1957,46 +1962,51 @@ describe("buildVomScene", () => {
|
||||
backendDOMNodeId: 21,
|
||||
},
|
||||
];
|
||||
const scene = buildVomScene(axNodes, {
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
surfaceProbes: [
|
||||
{ triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["My profile"] },
|
||||
],
|
||||
nodes: [
|
||||
{
|
||||
backendNodeId: 10,
|
||||
parentBackendNodeId: null,
|
||||
tag: "body",
|
||||
attrs: {},
|
||||
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 20,
|
||||
parentBackendNodeId: 10,
|
||||
tag: "div",
|
||||
attrs: { class: "tg-avatar" },
|
||||
rect: { x: 900, y: 10, w: 30, h: 30 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 21,
|
||||
parentBackendNodeId: 20,
|
||||
tag: "div",
|
||||
attrs: { class: "tg-avatar__inner" },
|
||||
rect: { x: 902, y: 12, w: 26, h: 26 },
|
||||
paintOrder: 2,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
],
|
||||
});
|
||||
const scene = buildVomScene(
|
||||
axNodes,
|
||||
{
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
nodes: [
|
||||
{
|
||||
backendNodeId: 10,
|
||||
parentBackendNodeId: null,
|
||||
tag: "body",
|
||||
attrs: {},
|
||||
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 20,
|
||||
parentBackendNodeId: 10,
|
||||
tag: "div",
|
||||
attrs: { class: "tg-avatar" },
|
||||
rect: { x: 900, y: 10, w: 30, h: 30 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 21,
|
||||
parentBackendNodeId: 20,
|
||||
tag: "div",
|
||||
attrs: { class: "tg-avatar__inner" },
|
||||
rect: { x: 902, y: 12, w: 26, h: 26 },
|
||||
paintOrder: 2,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
surfaceProbes: [
|
||||
{ triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["My profile"] },
|
||||
],
|
||||
},
|
||||
);
|
||||
|
||||
expect(renderVom(scene).text).toContain('@e1 button "image" [hover first: My profile]');
|
||||
});
|
||||
@@ -2017,48 +2027,53 @@ describe("buildVomScene", () => {
|
||||
backendDOMNodeId: 21,
|
||||
},
|
||||
];
|
||||
const scene = buildVomScene(axNodes, {
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
surfaceProbes: [
|
||||
{ triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["My profile"] },
|
||||
{ triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["Sign out"] },
|
||||
{ triggerBackendNodeId: 21, triggerAction: "hover", subItems: ["Settings"] },
|
||||
],
|
||||
nodes: [
|
||||
{
|
||||
backendNodeId: 10,
|
||||
parentBackendNodeId: null,
|
||||
tag: "body",
|
||||
attrs: {},
|
||||
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 20,
|
||||
parentBackendNodeId: 10,
|
||||
tag: "div",
|
||||
attrs: { class: "tg-avatar" },
|
||||
rect: { x: 900, y: 10, w: 30, h: 30 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 21,
|
||||
parentBackendNodeId: 20,
|
||||
tag: "div",
|
||||
attrs: { class: "tg-avatar__inner" },
|
||||
rect: { x: 902, y: 12, w: 26, h: 26 },
|
||||
paintOrder: 2,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
],
|
||||
});
|
||||
const scene = buildVomScene(
|
||||
axNodes,
|
||||
{
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
nodes: [
|
||||
{
|
||||
backendNodeId: 10,
|
||||
parentBackendNodeId: null,
|
||||
tag: "body",
|
||||
attrs: {},
|
||||
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 20,
|
||||
parentBackendNodeId: 10,
|
||||
tag: "div",
|
||||
attrs: { class: "tg-avatar" },
|
||||
rect: { x: 900, y: 10, w: 30, h: 30 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 21,
|
||||
parentBackendNodeId: 20,
|
||||
tag: "div",
|
||||
attrs: { class: "tg-avatar__inner" },
|
||||
rect: { x: 902, y: 12, w: 26, h: 26 },
|
||||
paintOrder: 2,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
surfaceProbes: [
|
||||
{ triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["My profile"] },
|
||||
{ triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["Sign out"] },
|
||||
{ triggerBackendNodeId: 21, triggerAction: "hover", subItems: ["Settings"] },
|
||||
],
|
||||
},
|
||||
);
|
||||
|
||||
expect(scene.surfaces).toEqual([
|
||||
{ triggerId: 21, triggerAction: "hover", subItems: ["My profile"] },
|
||||
@@ -2081,54 +2096,59 @@ describe("buildVomScene", () => {
|
||||
backendDOMNodeId: 21,
|
||||
},
|
||||
];
|
||||
const scene = buildVomScene(axNodes, {
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
surfaceProbes: [
|
||||
{
|
||||
triggerBackendNodeId: 20,
|
||||
triggerPoint: { x: 915, y: 25 },
|
||||
triggerAction: "hover",
|
||||
subItems: ["My profile", "Sign out"],
|
||||
},
|
||||
],
|
||||
nodes: [
|
||||
{
|
||||
backendNodeId: 10,
|
||||
parentBackendNodeId: null,
|
||||
tag: "body",
|
||||
attrs: {},
|
||||
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
cursor: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 20,
|
||||
parentBackendNodeId: 10,
|
||||
tag: "div",
|
||||
attrs: { class: "tg-avatar" },
|
||||
rect: { x: 900, y: 10, w: 30, h: 30 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
cursor: "pointer",
|
||||
},
|
||||
{
|
||||
backendNodeId: 21,
|
||||
parentBackendNodeId: 20,
|
||||
tag: "div",
|
||||
attrs: { class: "tg-avatar__inner" },
|
||||
rect: { x: 902, y: 12, w: 26, h: 26 },
|
||||
paintOrder: 2,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
cursor: "pointer",
|
||||
},
|
||||
],
|
||||
});
|
||||
const scene = buildVomScene(
|
||||
axNodes,
|
||||
{
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
nodes: [
|
||||
{
|
||||
backendNodeId: 10,
|
||||
parentBackendNodeId: null,
|
||||
tag: "body",
|
||||
attrs: {},
|
||||
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
cursor: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 20,
|
||||
parentBackendNodeId: 10,
|
||||
tag: "div",
|
||||
attrs: { class: "tg-avatar" },
|
||||
rect: { x: 900, y: 10, w: 30, h: 30 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
cursor: "pointer",
|
||||
},
|
||||
{
|
||||
backendNodeId: 21,
|
||||
parentBackendNodeId: 20,
|
||||
tag: "div",
|
||||
attrs: { class: "tg-avatar__inner" },
|
||||
rect: { x: 902, y: 12, w: 26, h: 26 },
|
||||
paintOrder: 2,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
cursor: "pointer",
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
surfaceProbes: [
|
||||
{
|
||||
triggerBackendNodeId: 20,
|
||||
triggerPoint: { x: 915, y: 25 },
|
||||
triggerAction: "hover",
|
||||
subItems: ["My profile", "Sign out"],
|
||||
},
|
||||
],
|
||||
},
|
||||
);
|
||||
|
||||
expect(scene.surfaces).toEqual([
|
||||
{ triggerId: 20, triggerAction: "hover", subItems: ["My profile", "Sign out"] },
|
||||
@@ -2154,41 +2174,46 @@ describe("buildVomScene", () => {
|
||||
backendDOMNodeId: 42,
|
||||
},
|
||||
];
|
||||
const scene = buildVomScene(axNodes, {
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
surfaceProbes: [
|
||||
{
|
||||
triggerBackendNodeId: 999,
|
||||
triggerPoint: { x: 900, y: 20 },
|
||||
triggerAction: "hover",
|
||||
subItems: ["My profile"],
|
||||
},
|
||||
],
|
||||
nodes: [
|
||||
{
|
||||
backendNodeId: 10,
|
||||
parentBackendNodeId: null,
|
||||
tag: "body",
|
||||
attrs: {},
|
||||
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 42,
|
||||
parentBackendNodeId: 10,
|
||||
tag: "a",
|
||||
attrs: {},
|
||||
rect: { x: 860, y: 120, w: 40, h: 30 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
],
|
||||
});
|
||||
const scene = buildVomScene(
|
||||
axNodes,
|
||||
{
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
nodes: [
|
||||
{
|
||||
backendNodeId: 10,
|
||||
parentBackendNodeId: null,
|
||||
tag: "body",
|
||||
attrs: {},
|
||||
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 42,
|
||||
parentBackendNodeId: 10,
|
||||
tag: "a",
|
||||
attrs: {},
|
||||
rect: { x: 860, y: 120, w: 40, h: 30 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
surfaceProbes: [
|
||||
{
|
||||
triggerBackendNodeId: 999,
|
||||
triggerPoint: { x: 900, y: 20 },
|
||||
triggerAction: "hover",
|
||||
subItems: ["My profile"],
|
||||
},
|
||||
],
|
||||
},
|
||||
);
|
||||
|
||||
expect(scene.surfaces).toBeUndefined();
|
||||
expect(renderVom(scene).text).not.toContain("[hover first:");
|
||||
@@ -2606,7 +2631,106 @@ describe("handleSnapshot", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("enables conditional surface probing for semantic observe", async () => {
|
||||
it("hovers the page only after the accessibility tree has been captured", async () => {
|
||||
// Hovering can open menus and reflow the page. Probing between the DOM and
|
||||
// AX captures would leave the two halves of one observation describing the
|
||||
// page on either side of that change.
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
await sm.start("aa11");
|
||||
const root: CdpAxNode = {
|
||||
nodeId: "1",
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
name: { type: "computedString", value: "Example" },
|
||||
backendDOMNodeId: 100,
|
||||
childIds: ["2"],
|
||||
};
|
||||
const button: CdpAxNode = {
|
||||
nodeId: "2",
|
||||
parentId: "1",
|
||||
role: { type: "role", value: "button" },
|
||||
name: { type: "computedString", value: "Products" },
|
||||
backendDOMNodeId: 200,
|
||||
};
|
||||
const strings = ["body", "button", "position", "static", "pointer-events", "auto", "cursor"];
|
||||
const i = (s: string) => strings.indexOf(s);
|
||||
const methodOrder: string[] = [];
|
||||
const send = vi.fn(async (_tabId: number, method: string, params?: object) => {
|
||||
methodOrder.push(method);
|
||||
if (method === "Accessibility.enable") return {};
|
||||
if (method === "Accessibility.getFullAXTree") return { nodes: [root, button] };
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 } };
|
||||
}
|
||||
if (method === "DOMSnapshot.enable") return {};
|
||||
if (method === "DOMSnapshot.captureSnapshot") {
|
||||
return {
|
||||
strings,
|
||||
documents: [
|
||||
{
|
||||
nodes: {
|
||||
parentIndex: [-1, 0],
|
||||
nodeName: [i("body"), i("button")],
|
||||
backendNodeId: [100, 200],
|
||||
attributes: [[], []],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [0, 1],
|
||||
styles: [
|
||||
[i("static"), i("auto"), i("auto")],
|
||||
[i("static"), i("auto"), i("auto")],
|
||||
],
|
||||
bounds: [
|
||||
[0, 0, 1000, 800],
|
||||
[20, 20, 120, 40],
|
||||
],
|
||||
paintOrders: [0, 1],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
if (method === "Input.dispatchMouseEvent") return {};
|
||||
if (method === "Runtime.evaluate") {
|
||||
const expression = (params as { expression?: string } | undefined)?.expression ?? "";
|
||||
// Report the button as a `:hover` rule target so a real hover fires.
|
||||
if (expression.includes("document.styleSheets")) {
|
||||
return { result: { value: [{ x: 80, y: 40 }] } };
|
||||
}
|
||||
return { result: { value: [] } };
|
||||
}
|
||||
throw new Error(`unexpected CDP method ${method}`);
|
||||
});
|
||||
|
||||
const res = await handleObserve(
|
||||
sm,
|
||||
{ session_id: "aa11", probe_hover: true },
|
||||
{
|
||||
cdp: {
|
||||
send: send as unknown as <T = unknown>(
|
||||
tabId: number,
|
||||
method: string,
|
||||
params?: object,
|
||||
) => Promise<T>,
|
||||
trackSessionTab: vi.fn(),
|
||||
},
|
||||
tabsApi: {
|
||||
get: vi.fn(
|
||||
async (tabId: number) =>
|
||||
({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab,
|
||||
),
|
||||
query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
const firstHover = methodOrder.indexOf("Input.dispatchMouseEvent");
|
||||
expect(firstHover).toBeGreaterThan(-1);
|
||||
expect(methodOrder.lastIndexOf("DOMSnapshot.captureSnapshot")).toBeLessThan(firstHover);
|
||||
expect(methodOrder.lastIndexOf("Accessibility.getFullAXTree")).toBeLessThan(firstHover);
|
||||
});
|
||||
|
||||
it("runs conditional surface probing only when observe opts in", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
await sm.start("aa11");
|
||||
const root: CdpAxNode = {
|
||||
@@ -2662,30 +2786,46 @@ describe("handleSnapshot", () => {
|
||||
if (method === "Runtime.evaluate") return { result: { value: [] } };
|
||||
throw new Error(`unexpected CDP method ${method}`);
|
||||
});
|
||||
const res = await handleObserve(
|
||||
const deps = {
|
||||
cdp: {
|
||||
send: send as unknown as <T = unknown>(
|
||||
tabId: number,
|
||||
method: string,
|
||||
params?: object,
|
||||
) => Promise<T>,
|
||||
trackSessionTab: vi.fn(),
|
||||
},
|
||||
tabsApi: {
|
||||
get: vi.fn(
|
||||
async (tabId: number) => ({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab,
|
||||
),
|
||||
query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
|
||||
},
|
||||
};
|
||||
|
||||
const withoutProbe = await handleObserve(
|
||||
sm,
|
||||
{ session_id: "aa11", debug_surfaces: true },
|
||||
{
|
||||
cdp: {
|
||||
send: send as unknown as <T = unknown>(
|
||||
tabId: number,
|
||||
method: string,
|
||||
params?: object,
|
||||
) => Promise<T>,
|
||||
trackSessionTab: vi.fn(),
|
||||
},
|
||||
tabsApi: {
|
||||
get: vi.fn(
|
||||
async (tabId: number) =>
|
||||
({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab,
|
||||
),
|
||||
query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
|
||||
},
|
||||
},
|
||||
deps,
|
||||
);
|
||||
if ("code" in withoutProbe) {
|
||||
throw new Error(`unexpected error: ${JSON.stringify(withoutProbe)}`);
|
||||
}
|
||||
expect(withoutProbe.hover_probe).toBeUndefined();
|
||||
expect(send).not.toHaveBeenCalledWith(
|
||||
4,
|
||||
"Runtime.evaluate",
|
||||
expect.objectContaining({ returnByValue: true }),
|
||||
);
|
||||
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
expect(res.debug).toEqual({ surface_probes: [] });
|
||||
const withProbe = await handleObserve(
|
||||
sm,
|
||||
{ session_id: "aa11", debug_surfaces: true, probe_hover: true },
|
||||
deps,
|
||||
);
|
||||
if ("code" in withProbe) throw new Error(`unexpected error: ${JSON.stringify(withProbe)}`);
|
||||
expect(withProbe.debug).toEqual({ surface_probes: [] });
|
||||
expect(withProbe.hover_probe).toEqual({ performed: true, revealed_content: false });
|
||||
expect(send).toHaveBeenCalledWith(
|
||||
4,
|
||||
"Runtime.evaluate",
|
||||
|
||||
@@ -1,8 +1,26 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { SessionManager } from "@/session-manager/manager";
|
||||
import { handleSessionStop } from "../session";
|
||||
import type { ChromeTabsApi } from "../shared";
|
||||
import { type AgentOverlayResetApi, type ChromeWindowsApi, type TabMutationApi } from "../tabs";
|
||||
|
||||
/** Build a read-only query api over a FakeState. */
|
||||
function makeQuery(state: FakeState): ChromeTabsApi {
|
||||
return {
|
||||
get: vi.fn(async (id) => {
|
||||
const t = state.tabs.get(id);
|
||||
if (!t) throw new Error(`tab ${id} not found`);
|
||||
return t;
|
||||
}),
|
||||
query: vi.fn(async (q: chrome.tabs.QueryInfo) => {
|
||||
const w = q.windowId;
|
||||
return Array.from(state.tabs.values()).filter(
|
||||
(t) => typeof w !== "number" || t.windowId === w,
|
||||
);
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
function fakeAgentWindow(ids: number[]) {
|
||||
let i = 0;
|
||||
const create = vi.fn(async () => {
|
||||
@@ -11,7 +29,7 @@ function fakeAgentWindow(ids: number[]) {
|
||||
return id;
|
||||
});
|
||||
const remove = vi.fn(async () => {});
|
||||
const ensureActiveTab = vi.fn(async () => {});
|
||||
const ensureActiveTab = vi.fn(async () => 0);
|
||||
return { create, remove, ensureActiveTab };
|
||||
}
|
||||
|
||||
@@ -30,7 +48,9 @@ function makeApis(
|
||||
} {
|
||||
const tabs: TabMutationApi = {
|
||||
create: vi.fn(),
|
||||
remove: vi.fn(async () => {}),
|
||||
remove: vi.fn(async (id: number) => {
|
||||
state.tabs.delete(id);
|
||||
}),
|
||||
update: vi.fn(async (_id, _p) => undefined),
|
||||
get: vi.fn(async (id) => {
|
||||
const t = state.tabs.get(id);
|
||||
@@ -237,11 +257,15 @@ describe("handleSessionStop with auto-return", () => {
|
||||
moves: [],
|
||||
};
|
||||
const { tabs, windows } = makeApis(state, { moveThrowsFor: new Set([1]) });
|
||||
const cdp = {
|
||||
detachSession: vi.fn(async () => {}),
|
||||
releaseSessionTab: vi.fn(async () => {}),
|
||||
};
|
||||
|
||||
const res = await handleSessionStop(
|
||||
sm,
|
||||
{ session_id: "aa11" },
|
||||
{ tabManagement: { tabs, windows } },
|
||||
{ cdp, tabManagement: { tabs, windows } },
|
||||
);
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
expect(res.return_failures?.map((f) => f.tab_id)).toEqual([1]);
|
||||
@@ -249,6 +273,8 @@ describe("handleSessionStop with auto-return", () => {
|
||||
expect(sm.has("aa11")).toBe(true);
|
||||
expect(ctx.borrowedTabs.has(1)).toBe(true);
|
||||
expect(ctx.borrowedTabs.has(2)).toBe(false);
|
||||
expect(cdp.releaseSessionTab).toHaveBeenCalledExactlyOnceWith("aa11", 2);
|
||||
expect(cdp.detachSession).not.toHaveBeenCalled();
|
||||
expect(aw.remove).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
@@ -267,4 +293,275 @@ describe("handleSessionStop with auto-return", () => {
|
||||
await handleSessionStop(sm, { session_id: "aa11" }, { tabManagement: { tabs, windows } });
|
||||
expect(ctx.refStore.isEmpty()).toBe(true);
|
||||
});
|
||||
|
||||
it.each([
|
||||
"before teardown",
|
||||
"during its return",
|
||||
"after its return failed",
|
||||
"before its turn in the return loop",
|
||||
])("finishes stopping when a borrowed tab closes %s", async (timing) => {
|
||||
const aw = fakeAgentWindow([100]);
|
||||
const sm = new SessionManager({ agentWindow: aw });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.borrowedTabs.set(1, { tabId: 1, originalWindowId: 200, originalIndex: 0 });
|
||||
ctx.borrowedTabs.set(2, { tabId: 2, originalWindowId: 200, originalIndex: 1 });
|
||||
const state: FakeState = {
|
||||
tabs: new Map([
|
||||
[1, { id: 1, windowId: 100 } as chrome.tabs.Tab],
|
||||
[2, { id: 2, windowId: 100 } as chrome.tabs.Tab],
|
||||
]),
|
||||
windowsClosed: new Set(),
|
||||
moves: [],
|
||||
};
|
||||
const { tabs, windows } = makeApis(state);
|
||||
const cdp = { detachSession: vi.fn(async () => {}) };
|
||||
const closeTab = (tabId: number) => {
|
||||
state.tabs.delete(tabId);
|
||||
sm.forgetClosedTab(tabId);
|
||||
};
|
||||
const baseMove = tabs.move;
|
||||
tabs.move = vi.fn(async (tabId, props) => {
|
||||
if (tabId === 1 && timing === "during its return") closeTab(1);
|
||||
if (tabId === 1 && timing === "after its return failed") {
|
||||
throw new Error("simulated move failure");
|
||||
}
|
||||
if (tabId === 2 && timing === "after its return failed") closeTab(1);
|
||||
if (tabId === 1 && timing === "before its turn in the return loop") closeTab(2);
|
||||
if (!state.tabs.has(tabId)) throw new Error(`tab ${tabId} not found`);
|
||||
return baseMove(tabId, props);
|
||||
});
|
||||
if (timing === "before teardown") closeTab(1);
|
||||
|
||||
const res = await handleSessionStop(
|
||||
sm,
|
||||
{ session_id: "aa11" },
|
||||
{
|
||||
cdp,
|
||||
tabManagement: {
|
||||
tabs,
|
||||
windows,
|
||||
agentOverlayReset: { resetAgentOverlays: vi.fn(async () => {}) },
|
||||
},
|
||||
tabsQuery: makeQuery(state),
|
||||
},
|
||||
);
|
||||
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
const closedTabId = timing === "before its turn in the return loop" ? 2 : 1;
|
||||
expect(res.returned_tab_ids).toEqual([closedTabId === 1 ? 2 : 1]);
|
||||
expect(res.return_failures).toBeUndefined();
|
||||
expect(ctx.borrowedTabs.size).toBe(0);
|
||||
expect(sm.has("aa11")).toBe(false);
|
||||
expect(cdp.detachSession).toHaveBeenCalledWith("aa11");
|
||||
expect(aw.remove).toHaveBeenCalledWith(100);
|
||||
if (timing === "before teardown" || timing === "before its turn in the return loop") {
|
||||
expect(tabs.move).not.toHaveBeenCalledWith(closedTabId, expect.anything());
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("handleSessionStop window release (issue #57)", () => {
|
||||
const agentWindowId = 100;
|
||||
|
||||
it("releases the window when user-created tabs remain", async () => {
|
||||
const aw = fakeAgentWindow([agentWindowId]);
|
||||
// ensureActiveTab returns the home tab id (10). We override the fake to
|
||||
// return a known id.
|
||||
aw.ensureActiveTab = vi.fn(async () => 10);
|
||||
const sm = new SessionManager({ agentWindow: aw });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.agentCreatedTabs.add(11);
|
||||
ctx.agentCreatedTabs.add(12);
|
||||
// user tab 99 (created via Chrome UI, NOT in agentCreatedTabs)
|
||||
const state: FakeState = {
|
||||
tabs: new Map([
|
||||
[10, { id: 10, windowId: agentWindowId } as chrome.tabs.Tab],
|
||||
[11, { id: 11, windowId: agentWindowId } as chrome.tabs.Tab],
|
||||
[12, { id: 12, windowId: agentWindowId } as chrome.tabs.Tab],
|
||||
[99, { id: 99, windowId: agentWindowId } as chrome.tabs.Tab],
|
||||
]),
|
||||
windowsClosed: new Set(),
|
||||
moves: [],
|
||||
};
|
||||
const { tabs, windows } = makeApis(state);
|
||||
const query = makeQuery(state);
|
||||
|
||||
const res = await handleSessionStop(
|
||||
sm,
|
||||
{ session_id: "aa11" },
|
||||
{ tabManagement: { tabs, windows }, tabsQuery: query },
|
||||
);
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
expect(res.window_released).toBe(true);
|
||||
// agent tabs + home removed, user tab 99 kept
|
||||
expect((tabs.remove as ReturnType<typeof vi.fn>).mock.calls.map((c) => c[0]).sort()).toEqual([
|
||||
10, 11, 12,
|
||||
]);
|
||||
expect(state.tabs.has(99)).toBe(true);
|
||||
// window released (dropOnly), not closed
|
||||
expect(aw.remove).not.toHaveBeenCalled();
|
||||
expect(sm.has("aa11")).toBe(false);
|
||||
});
|
||||
|
||||
it("closes the window (not release) when no user tabs remain", async () => {
|
||||
const aw = fakeAgentWindow([agentWindowId]);
|
||||
aw.ensureActiveTab = vi.fn(async () => 10);
|
||||
const sm = new SessionManager({ agentWindow: aw });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.agentCreatedTabs.add(11);
|
||||
const state: FakeState = {
|
||||
tabs: new Map([
|
||||
[10, { id: 10, windowId: agentWindowId } as chrome.tabs.Tab],
|
||||
[11, { id: 11, windowId: agentWindowId } as chrome.tabs.Tab],
|
||||
]),
|
||||
windowsClosed: new Set(),
|
||||
moves: [],
|
||||
};
|
||||
const { tabs, windows } = makeApis(state);
|
||||
const query = makeQuery(state);
|
||||
|
||||
const res = await handleSessionStop(
|
||||
sm,
|
||||
{ session_id: "aa11" },
|
||||
{ tabManagement: { tabs, windows }, tabsQuery: query },
|
||||
);
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
expect(res.window_released).toBeFalsy();
|
||||
expect(aw.remove).toHaveBeenCalledWith(agentWindowId);
|
||||
expect(sm.has("aa11")).toBe(false);
|
||||
});
|
||||
|
||||
it("is non-fatal when an agent tab is already gone", async () => {
|
||||
const aw = fakeAgentWindow([agentWindowId]);
|
||||
aw.ensureActiveTab = vi.fn(async () => 10);
|
||||
const sm = new SessionManager({ agentWindow: aw });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.agentCreatedTabs.add(11); // no longer in state → remove throws
|
||||
const state: FakeState = {
|
||||
tabs: new Map([[10, { id: 10, windowId: agentWindowId } as chrome.tabs.Tab]]),
|
||||
windowsClosed: new Set(),
|
||||
moves: [],
|
||||
};
|
||||
const { tabs, windows } = makeApis(state);
|
||||
(tabs.remove as ReturnType<typeof vi.fn>).mockImplementation(async (id: number) => {
|
||||
if (id === 11) throw new Error("tab 11 already closed");
|
||||
state.tabs.delete(id); // home (10) removed normally
|
||||
});
|
||||
const query = makeQuery(state);
|
||||
|
||||
const res = await handleSessionStop(
|
||||
sm,
|
||||
{ session_id: "aa11" },
|
||||
{ tabManagement: { tabs, windows }, tabsQuery: query },
|
||||
);
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
// user tab 10? no — 10 is home, removed. window would be empty → close.
|
||||
expect(res.window_released).toBeFalsy();
|
||||
expect(aw.remove).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("degrades to close-window when tabsQuery is missing", async () => {
|
||||
const aw = fakeAgentWindow([agentWindowId]);
|
||||
aw.ensureActiveTab = vi.fn(async () => 10);
|
||||
const sm = new SessionManager({ agentWindow: aw });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.agentCreatedTabs.add(11);
|
||||
const state: FakeState = {
|
||||
tabs: new Map([
|
||||
[10, { id: 10, windowId: agentWindowId } as chrome.tabs.Tab],
|
||||
[11, { id: 11, windowId: agentWindowId } as chrome.tabs.Tab],
|
||||
]),
|
||||
windowsClosed: new Set(),
|
||||
moves: [],
|
||||
};
|
||||
const { tabs, windows } = makeApis(state);
|
||||
|
||||
const res = await handleSessionStop(
|
||||
sm,
|
||||
{ session_id: "aa11" },
|
||||
{ tabManagement: { tabs, windows } }, // no tabsQuery
|
||||
);
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
// No query available → conservatively close the window.
|
||||
expect(res.window_released).toBeFalsy();
|
||||
expect(aw.remove).toHaveBeenCalledWith(agentWindowId);
|
||||
});
|
||||
|
||||
it("degrades to close-window when tabsQuery.query throws", async () => {
|
||||
const aw = fakeAgentWindow([agentWindowId]);
|
||||
aw.ensureActiveTab = vi.fn(async () => 10);
|
||||
const sm = new SessionManager({ agentWindow: aw });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.agentCreatedTabs.add(11);
|
||||
const state: FakeState = {
|
||||
tabs: new Map([[11, { id: 11, windowId: agentWindowId } as chrome.tabs.Tab]]),
|
||||
windowsClosed: new Set(),
|
||||
moves: [],
|
||||
};
|
||||
const { tabs, windows } = makeApis(state);
|
||||
const query = makeQuery(state);
|
||||
(query.query as ReturnType<typeof vi.fn>).mockImplementation(async () => {
|
||||
throw new Error("query failed");
|
||||
});
|
||||
|
||||
const res = await handleSessionStop(
|
||||
sm,
|
||||
{ session_id: "aa11" },
|
||||
{ tabManagement: { tabs, windows }, tabsQuery: query },
|
||||
);
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
expect(res.window_released).toBeFalsy();
|
||||
expect(aw.remove).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("tracks agentCreatedTabs across tab_create / tab_close", async () => {
|
||||
const aw = fakeAgentWindow([agentWindowId]);
|
||||
aw.ensureActiveTab = vi.fn(async () => 10);
|
||||
const sm = new SessionManager({ agentWindow: aw });
|
||||
const ctx = await sm.start("aa11");
|
||||
// Simulate handleTabCreate adding, then handleTabClose removing.
|
||||
ctx.agentCreatedTabs.add(11);
|
||||
expect(ctx.agentCreatedTabs.has(11)).toBe(true);
|
||||
ctx.agentCreatedTabs.delete(11); // as handleTabClose does on success
|
||||
expect(ctx.agentCreatedTabs.has(11)).toBe(false);
|
||||
// Unknown id delete is a safe no-op (user-closed tab).
|
||||
expect(() => ctx.agentCreatedTabs.delete(999)).not.toThrow();
|
||||
});
|
||||
|
||||
it("closes the window (not release) when an agent tab fails to close", async () => {
|
||||
// Regression: a leaked agent tab (still in agentCreatedTabs, but
|
||||
// still present because Step 4 remove() threw) must NOT be mistaken
|
||||
// for a user tab. Otherwise the window would be released (dropOnly)
|
||||
// and the agent tab would leak (issue #57 regression).
|
||||
const aw = fakeAgentWindow([agentWindowId]);
|
||||
aw.ensureActiveTab = vi.fn(async () => 10);
|
||||
const sm = new SessionManager({ agentWindow: aw });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.agentCreatedTabs.add(11); // Step 4 remove() will throw for this tab
|
||||
const state: FakeState = {
|
||||
tabs: new Map([
|
||||
[10, { id: 10, windowId: agentWindowId } as chrome.tabs.Tab],
|
||||
[11, { id: 11, windowId: agentWindowId } as chrome.tabs.Tab],
|
||||
]),
|
||||
windowsClosed: new Set(),
|
||||
moves: [],
|
||||
};
|
||||
const { tabs, windows } = makeApis(state);
|
||||
(tabs.remove as ReturnType<typeof vi.fn>).mockImplementation(async (id: number) => {
|
||||
if (id === 11) throw new Error("tab 11 failed to close");
|
||||
state.tabs.delete(id); // home (10) removed normally; 11 lingers
|
||||
});
|
||||
const query = makeQuery(state);
|
||||
|
||||
const res = await handleSessionStop(
|
||||
sm,
|
||||
{ session_id: "aa11" },
|
||||
{ tabManagement: { tabs, windows }, tabsQuery: query },
|
||||
);
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
// The leaked agent tab must NOT keep the window open.
|
||||
expect(res.window_released).toBeFalsy();
|
||||
expect(aw.remove).toHaveBeenCalledWith(agentWindowId);
|
||||
expect(sm.has("aa11")).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -12,7 +12,7 @@ function fakeAgentWindow() {
|
||||
return {
|
||||
create: vi.fn(async () => 100),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -11,7 +11,7 @@ function fakeAgentWindow(ids: number[]) {
|
||||
return id;
|
||||
},
|
||||
remove: async () => {},
|
||||
ensureActiveTab: async () => {},
|
||||
ensureActiveTab: async () => 1,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { type CdpDebuggerApi, ChromiumCdp } from "@/browser-driver/chromium-cdp";
|
||||
import { SessionManager } from "@/session-manager/manager";
|
||||
import { handleHover } from "../interaction";
|
||||
import {
|
||||
type AgentOverlayResetApi,
|
||||
type ChromeWindowsApi,
|
||||
@@ -23,7 +25,7 @@ function fakeAgentWindow(ids: number[]) {
|
||||
return id;
|
||||
}),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -221,6 +223,7 @@ describe("handleTabCreate", () => {
|
||||
active: true,
|
||||
});
|
||||
expect(res).toMatchObject({ tab_id: 10, window_id: 100 });
|
||||
expect(sm.get("aa11")?.agentCreatedTabs.has(10)).toBe(true);
|
||||
});
|
||||
|
||||
it("forwards explicit url + index + active=false", async () => {
|
||||
@@ -256,6 +259,41 @@ describe("handleTabCreate", () => {
|
||||
const res = await handleTabCreate(sm, { session_id: "aa11", index: -1 }, { tabs: api });
|
||||
expect(res).toMatchObject({ code: "invalid_params" });
|
||||
});
|
||||
|
||||
it("does not claim any tab when chrome.tabs.create fails", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
await sm.start("aa11");
|
||||
const state: FakeTabState = { tabs: new Map(), nextTabId: 10, windowsClosed: new Set() };
|
||||
const { api, spies } = makeTabMutationApi(state);
|
||||
spies.create.mockRejectedValueOnce(new Error("chrome.tabs.create failed"));
|
||||
|
||||
const res = await handleTabCreate(sm, { session_id: "aa11" }, { tabs: api });
|
||||
expect(res).toMatchObject({ code: "protocol_error" });
|
||||
|
||||
expect(sm.get("aa11")?.agentCreatedTabs).toEqual(new Set([1]));
|
||||
});
|
||||
|
||||
it("claims the returned id even when a user tab appears during tab creation", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
const state: FakeTabState = { tabs: new Map(), nextTabId: 10, windowsClosed: new Set() };
|
||||
const { api, spies } = makeTabMutationApi(state);
|
||||
let resolveCreate!: (tab: chrome.tabs.Tab) => void;
|
||||
const createResult = new Promise<chrome.tabs.Tab>((resolve) => {
|
||||
resolveCreate = resolve;
|
||||
});
|
||||
spies.create.mockImplementationOnce(() => createResult);
|
||||
|
||||
const pending = handleTabCreate(sm, { session_id: "aa11" }, { tabs: api });
|
||||
// A user can create an unrelated same-window tab while Chrome is still
|
||||
// resolving the agent request. There is no pending slot for it to steal.
|
||||
expect(ctx.agentCreatedTabs.has(99)).toBe(false);
|
||||
resolveCreate({ id: 10, windowId: 100, active: true } as chrome.tabs.Tab);
|
||||
|
||||
await expect(pending).resolves.toMatchObject({ tab_id: 10 });
|
||||
expect(ctx.agentCreatedTabs.has(10)).toBe(true);
|
||||
expect(ctx.agentCreatedTabs.has(99)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("handleTabClose", () => {
|
||||
@@ -511,6 +549,148 @@ describe("handleTabBorrow", () => {
|
||||
});
|
||||
|
||||
describe("handleTabReturn", () => {
|
||||
it.each([
|
||||
"not_found",
|
||||
"cdp_failed",
|
||||
])("detaches after selector resolution fails with %s", async (code) => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.borrowedTabs.set(7, { tabId: 7, originalWindowId: 200, originalIndex: 4 });
|
||||
const state: FakeTabState = {
|
||||
tabs: new Map([[7, { id: 7, windowId: 100 } as chrome.tabs.Tab]]),
|
||||
nextTabId: 50,
|
||||
windowsClosed: new Set(),
|
||||
};
|
||||
const { api } = makeTabMutationApi(state);
|
||||
const { api: windows } = makeWindowsApi(state);
|
||||
const debuggerApi = {
|
||||
attach: vi.fn(async () => {}),
|
||||
detach: vi.fn(async () => {}),
|
||||
sendCommand: vi.fn(async (_target: unknown, method: string) => {
|
||||
if (method === "DOM.getDocument") return { root: { nodeId: 1 } };
|
||||
if (method === "DOM.querySelector") {
|
||||
if (code === "cdp_failed") throw new Error("selector query failed");
|
||||
return { nodeId: 0 };
|
||||
}
|
||||
return {};
|
||||
}),
|
||||
onEvent: { addListener: vi.fn(), removeListener: vi.fn() },
|
||||
onDetach: { addListener: vi.fn(), removeListener: vi.fn() },
|
||||
};
|
||||
const cdp = new ChromiumCdp(debuggerApi as unknown as CdpDebuggerApi);
|
||||
|
||||
const hover = await handleHover(
|
||||
sm,
|
||||
{ session_id: "aa11", tab_id: 7, selector: "#missing" },
|
||||
{
|
||||
cdp,
|
||||
tabsApi: { get: api.get, query: vi.fn(async () => []) },
|
||||
},
|
||||
);
|
||||
expect(hover).toMatchObject({ code });
|
||||
expect(cdp.isAttached(7)).toBe(true);
|
||||
|
||||
const returned = await handleTabReturn(
|
||||
sm,
|
||||
{ session_id: "aa11", tab_id: 7 },
|
||||
{
|
||||
tabs: api,
|
||||
windows,
|
||||
cdp,
|
||||
agentOverlayReset: { resetAgentOverlays: vi.fn(async () => {}) },
|
||||
},
|
||||
);
|
||||
expect(returned).toMatchObject({ returned_to_window_id: 200 });
|
||||
expect(ctx.borrowedTabs.has(7)).toBe(false);
|
||||
expect(cdp.isAttached(7)).toBe(false);
|
||||
expect(debuggerApi.detach).toHaveBeenCalledExactlyOnceWith({ tabId: 7 });
|
||||
cdp.dispose();
|
||||
});
|
||||
|
||||
it("honours cancellation while preparing a validated return", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.borrowedTabs.set(7, { tabId: 7, originalWindowId: 200, originalIndex: 4 });
|
||||
const state: FakeTabState = {
|
||||
tabs: new Map([[7, { id: 7, windowId: 100 } as chrome.tabs.Tab]]),
|
||||
nextTabId: 50,
|
||||
windowsClosed: new Set(),
|
||||
};
|
||||
const { api, spies } = makeTabMutationApi(state);
|
||||
const { api: windows } = makeWindowsApi(state);
|
||||
const controller = new AbortController();
|
||||
const beforeReturn = vi.fn(async () => {
|
||||
controller.abort();
|
||||
});
|
||||
const result = await handleTabReturn(
|
||||
sm,
|
||||
{ session_id: "aa11", tab_id: 7 },
|
||||
{
|
||||
tabs: api,
|
||||
windows,
|
||||
signal: controller.signal,
|
||||
beforeReturn,
|
||||
},
|
||||
);
|
||||
|
||||
expect(result).toMatchObject({ code: "cancelled" });
|
||||
expect(beforeReturn).toHaveBeenCalledExactlyOnceWith("aa11", 7);
|
||||
expect(spies.move).not.toHaveBeenCalled();
|
||||
expect(ctx.borrowedTabs.has(7)).toBe(true);
|
||||
});
|
||||
|
||||
it.each([
|
||||
"original",
|
||||
"fallback",
|
||||
"cleanup_failed",
|
||||
"failed",
|
||||
"cancelled",
|
||||
])("releases CDP only after a successful return (%s)", async (mode) => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.borrowedTabs.set(7, { tabId: 7, originalWindowId: 200, originalIndex: 4 });
|
||||
const state: FakeTabState = {
|
||||
tabs: new Map([[7, { id: 7, windowId: 100 } as chrome.tabs.Tab]]),
|
||||
nextTabId: 50,
|
||||
windowsClosed: new Set(),
|
||||
};
|
||||
const { api, spies } = makeTabMutationApi(state);
|
||||
const { api: windowsApi } = makeWindowsApi(state, { lastFocused: 500 });
|
||||
const success = mode !== "failed" && mode !== "cancelled";
|
||||
if (mode === "fallback") spies.move.mockRejectedValueOnce(new Error("original move failed"));
|
||||
if (mode === "failed") spies.move.mockRejectedValue(new Error("move failed"));
|
||||
let windowAtRelease: number | undefined;
|
||||
const cdp = {
|
||||
releaseSessionTab: vi.fn(async () => {
|
||||
windowAtRelease = state.tabs.get(7)?.windowId;
|
||||
if (mode === "cleanup_failed") throw new Error("debugger release failed");
|
||||
}),
|
||||
};
|
||||
const controller = new AbortController();
|
||||
if (mode === "cancelled") controller.abort();
|
||||
|
||||
const result = await handleTabReturn(
|
||||
sm,
|
||||
{ session_id: "aa11", tab_id: 7 },
|
||||
{
|
||||
tabs: api,
|
||||
windows: windowsApi,
|
||||
cdp,
|
||||
signal: controller.signal,
|
||||
agentOverlayReset: { resetAgentOverlays: vi.fn(async () => {}) },
|
||||
},
|
||||
);
|
||||
|
||||
expect("code" in result).toBe(!success);
|
||||
expect(ctx.borrowedTabs.has(7)).toBe(!success);
|
||||
expect(cdp.releaseSessionTab).toHaveBeenCalledTimes(success ? 1 : 0);
|
||||
if (success) {
|
||||
expect(cdp.releaseSessionTab).toHaveBeenCalledWith("aa11", 7);
|
||||
expect(windowAtRelease).toBe(mode === "fallback" ? 500 : 200);
|
||||
expect(spies.move).toHaveBeenCalledTimes(mode === "fallback" ? 2 : 1);
|
||||
}
|
||||
});
|
||||
|
||||
it("returns the tab to its original window/index and clears borrowedTabs", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
|
||||
@@ -12,7 +12,7 @@ function fakeAgentWindow(ids: number[]) {
|
||||
return id;
|
||||
}),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -11,7 +11,7 @@ function fakeAgentWindow(ids: number[]) {
|
||||
return id;
|
||||
});
|
||||
const remove = vi.fn(async () => {});
|
||||
const ensureActiveTab = vi.fn(async () => {});
|
||||
const ensureActiveTab = vi.fn(async () => 1);
|
||||
return { create, remove, ensureActiveTab };
|
||||
}
|
||||
|
||||
|
||||
@@ -24,7 +24,7 @@ export interface BorrowResponseMessage {
|
||||
allowed: boolean;
|
||||
}
|
||||
|
||||
export const CONFIRMATION_TIMEOUT_MS = 5000;
|
||||
export const CONFIRMATION_TIMEOUT_MS = 60_000;
|
||||
const EXIT_ANIMATION_MS = 150;
|
||||
/** Matches BorrowConfirmationOverlay progress ring/bar transition (duration-1000). */
|
||||
export const PROGRESS_TRANSITION_MS = 1000;
|
||||
@@ -480,6 +480,13 @@ export async function requestBorrowConfirmation(
|
||||
settle(false);
|
||||
}, BACKGROUND_TIMEOUT_MS);
|
||||
|
||||
// Bring the user window to the front so the overlay is visible even
|
||||
// when the Agent Window has stolen focus. This is fire-and-forget: if
|
||||
// it fails the overlay / OS notification still give the user a path.
|
||||
void windowsApi.update(notificationAnchor.windowId, { focused: true }).catch((err) => {
|
||||
console.debug("[bsk borrow] proactive focus of user window failed", err);
|
||||
});
|
||||
|
||||
// Surface the OS notification *before* messaging any candidate so the
|
||||
// user has a parallel signal even if every content script is missing.
|
||||
// The Allow / Deny buttons let the user authorize explicitly when the
|
||||
|
||||
@@ -0,0 +1,110 @@
|
||||
import type { RpcError, WaitUntil } from "@/transport/types";
|
||||
import { cdpError } from "./errors";
|
||||
|
||||
interface NavigationEvent {
|
||||
tabId: number;
|
||||
frameId: number;
|
||||
documentId?: string;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
interface NavigationEvents {
|
||||
addListener(listener: (details: NavigationEvent) => void): void;
|
||||
removeListener(listener: (details: NavigationEvent) => void): void;
|
||||
}
|
||||
|
||||
/** Normal browser navigation remains available when chrome.debugger is denied. */
|
||||
export interface BrowserNavigationApi {
|
||||
update(tabId: number, props: { url: string }): Promise<unknown>;
|
||||
reload(tabId: number, props: { bypassCache: boolean }): Promise<void>;
|
||||
onCommitted: NavigationEvents;
|
||||
onDOMContentLoaded: NavigationEvents;
|
||||
onCompleted: NavigationEvents;
|
||||
onErrorOccurred: NavigationEvents;
|
||||
}
|
||||
|
||||
export const chromeBrowserNavigationApi: BrowserNavigationApi = {
|
||||
update: (tabId, props) => chrome.tabs.update(tabId, props),
|
||||
reload: (tabId, props) => chrome.tabs.reload(tabId, props),
|
||||
get onCommitted() {
|
||||
return chrome.webNavigation.onCommitted;
|
||||
},
|
||||
get onDOMContentLoaded() {
|
||||
return chrome.webNavigation.onDOMContentLoaded;
|
||||
},
|
||||
get onCompleted() {
|
||||
return chrome.webNavigation.onCompleted;
|
||||
},
|
||||
get onErrorOccurred() {
|
||||
return chrome.webNavigation.onErrorOccurred;
|
||||
},
|
||||
};
|
||||
|
||||
export type NavigationOutcome =
|
||||
| { reached: "match" | "timeout" | "cancelled"; lastLifecycle?: string }
|
||||
| { reached: "failed"; error: RpcError };
|
||||
|
||||
/** Subscribe before dispatch and ignore the departing document's load events. */
|
||||
export async function navigateWithBrowserApi(
|
||||
api: BrowserNavigationApi,
|
||||
tabId: number,
|
||||
action: () => Promise<unknown>,
|
||||
waitUntil: Exclude<WaitUntil, "networkidle">,
|
||||
timeoutMs: number,
|
||||
signal?: AbortSignal,
|
||||
): Promise<NavigationOutcome> {
|
||||
if (signal?.aborted) return { reached: "cancelled" };
|
||||
if (timeoutMs <= 0) return { reached: "timeout" };
|
||||
let committed = false;
|
||||
let documentId: string | undefined;
|
||||
let lastLifecycle: string | undefined;
|
||||
let finish!: (outcome: NavigationOutcome) => void;
|
||||
const outcome = new Promise<NavigationOutcome>((resolve) => {
|
||||
finish = resolve;
|
||||
});
|
||||
const isMainFrame = (details: NavigationEvent) =>
|
||||
details.tabId === tabId && details.frameId === 0;
|
||||
const onCommitted = (details: NavigationEvent) => {
|
||||
if (!isMainFrame(details)) return;
|
||||
committed = true;
|
||||
documentId = details.documentId;
|
||||
lastLifecycle = "commit";
|
||||
if (waitUntil === "commit") finish({ reached: "match", lastLifecycle });
|
||||
};
|
||||
const onDOMContentLoaded = (details: NavigationEvent) => {
|
||||
if (!isMainFrame(details) || !committed || (documentId && details.documentId !== documentId))
|
||||
return;
|
||||
lastLifecycle = "DOMContentLoaded";
|
||||
if (waitUntil === "domcontentloaded") finish({ reached: "match", lastLifecycle });
|
||||
};
|
||||
const onCompleted = (details: NavigationEvent) => {
|
||||
if (!isMainFrame(details) || !committed || (documentId && details.documentId !== documentId))
|
||||
return;
|
||||
finish({ reached: "match", lastLifecycle: "load" });
|
||||
};
|
||||
const onError = (details: NavigationEvent) => {
|
||||
if (!isMainFrame(details)) return;
|
||||
finish({ reached: "failed", error: cdpError(details.error ?? "browser navigation failed") });
|
||||
};
|
||||
const onAbort = () => finish({ reached: "cancelled", lastLifecycle });
|
||||
const subscriptions = [
|
||||
[api.onCommitted, onCommitted],
|
||||
[api.onDOMContentLoaded, onDOMContentLoaded],
|
||||
[api.onCompleted, onCompleted],
|
||||
[api.onErrorOccurred, onError],
|
||||
] as const;
|
||||
for (const [event, listener] of subscriptions) event.addListener(listener);
|
||||
signal?.addEventListener("abort", onAbort, { once: true });
|
||||
const timer = setTimeout(() => finish({ reached: "timeout", lastLifecycle }), timeoutMs);
|
||||
try {
|
||||
if (signal?.aborted) return { reached: "cancelled" };
|
||||
await action();
|
||||
return await outcome;
|
||||
} catch (error) {
|
||||
return { reached: "failed", error: cdpError(error) };
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
signal?.removeEventListener("abort", onAbort);
|
||||
for (const [event, listener] of subscriptions) event.removeListener(listener);
|
||||
}
|
||||
}
|
||||
@@ -1,10 +1,10 @@
|
||||
import type { DiagnosticSink } from "@/lib/diagnostics";
|
||||
import { OVERLAY_AUTOMATION_BYPASS } from "@/lib/overlay-bridge";
|
||||
import type { SessionManager } from "@/session-manager/manager";
|
||||
import type { Transport } from "@/transport/transport";
|
||||
import type {
|
||||
ClickParams,
|
||||
ConsoleParams,
|
||||
DownloadParams,
|
||||
EmulateParams,
|
||||
EvaluateParams,
|
||||
FillParams,
|
||||
@@ -29,11 +29,14 @@ import type {
|
||||
ScreenshotParams,
|
||||
SelectParams,
|
||||
SnapshotParams,
|
||||
UploadParams,
|
||||
WaitForNavigationParams,
|
||||
} from "@/transport/types";
|
||||
import { isRequestFrame } from "@/transport/types";
|
||||
import { handleConsole } from "./console";
|
||||
import { handleDownload } from "./download";
|
||||
import { type EmulateCdpRunner, handleEmulate } from "./emulate";
|
||||
import { classifyCdpError } from "./errors";
|
||||
import { handleEvaluate } from "./evaluate";
|
||||
import { handleRequestHelp } from "./human-loop";
|
||||
import { handleClick, handleFill, handleHover, handlePress, handleSelect } from "./interaction";
|
||||
@@ -67,6 +70,7 @@ import {
|
||||
import { chromeTabsApi, lookupSession, resolveTargetTab } from "./shared";
|
||||
import {
|
||||
type BorrowConfirmationApprover,
|
||||
chromeTabMutationApi,
|
||||
handleTabBorrow,
|
||||
handleTabClose,
|
||||
handleTabCreate,
|
||||
@@ -80,6 +84,7 @@ import {
|
||||
type TabReturnParams,
|
||||
type TabSelectParams,
|
||||
} from "./tabs";
|
||||
import { handleUpload } from "./upload";
|
||||
import { handleWaitForNavigation } from "./waits";
|
||||
import { handleWindowResize, type WindowResizeParams } from "./window";
|
||||
|
||||
@@ -115,11 +120,12 @@ export interface DispatcherDeps {
|
||||
onSessionsChanged?: () => void;
|
||||
/** Invoked before a tool that dispatches page input or mutates browser state is forwarded. */
|
||||
onBrowserControlResumed?: (sessionId: string) => void;
|
||||
/** Invoked after a tab is explicitly claimed so its overlay can be refreshed immediately. */
|
||||
onAgentTabClaimed?: (tabId: number, windowId: number) => void;
|
||||
/** User approval for `tool.tab_borrow` (overlay in content script). */
|
||||
approveBorrow?: BorrowConfirmationApprover;
|
||||
/** i18n notification copy for `tool.request_help` (resolved per-call). */
|
||||
helpNotificationCopy?: () => { title: string; body: string };
|
||||
diagnostics?: DiagnosticSink;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -145,9 +151,9 @@ export class ToolDispatcher {
|
||||
private readonly recording?: RecordRuntimeDeps;
|
||||
private readonly onSessionsChanged?: () => void;
|
||||
private readonly onBrowserControlResumed?: (sessionId: string) => void;
|
||||
private readonly onAgentTabClaimed?: (tabId: number, windowId: number) => void;
|
||||
private readonly approveBorrow?: BorrowConfirmationApprover;
|
||||
private readonly helpNotificationCopy?: () => { title: string; body: string };
|
||||
private readonly diagnostics?: DiagnosticSink;
|
||||
private subscription: { dispose(): void } | null = null;
|
||||
private readonly hoverBypassTabs = new Map<number, string>();
|
||||
private readonly hoverLatches = new Map<number, HoverLatch>();
|
||||
@@ -166,9 +172,9 @@ export class ToolDispatcher {
|
||||
this.recording = deps.recording;
|
||||
this.onSessionsChanged = deps.onSessionsChanged;
|
||||
this.onBrowserControlResumed = deps.onBrowserControlResumed;
|
||||
this.onAgentTabClaimed = deps.onAgentTabClaimed;
|
||||
this.approveBorrow = deps.approveBorrow;
|
||||
this.helpNotificationCopy = deps.helpNotificationCopy;
|
||||
this.diagnostics = deps.diagnostics;
|
||||
}
|
||||
|
||||
start(): void {
|
||||
@@ -196,16 +202,6 @@ export class ToolDispatcher {
|
||||
private async dispatch(msg: ProtocolFrame): Promise<void> {
|
||||
if (!isRequestFrame(msg)) return;
|
||||
const req = msg as RequestFrame;
|
||||
const requestSessionId = (req.params as { session_id?: unknown } | undefined)?.session_id;
|
||||
const startedAt = performance.now();
|
||||
this.diagnostics?.("dispatcher.rpc.received", {
|
||||
rpc_id: req.id,
|
||||
method: req.method,
|
||||
session_id: typeof requestSessionId === "string" ? requestSessionId : null,
|
||||
transport_state: this.transport.state,
|
||||
live_session_ids: this.sessions.list().map((ctx) => ctx.sessionId),
|
||||
inflight_rpc_ids: Array.from(this.inflightAbortControllers.keys()),
|
||||
});
|
||||
|
||||
// Cancel frames take a fast path: trip the matching controller
|
||||
// (if any), reply with `{cancelled}` so the daemon can answer
|
||||
@@ -214,12 +210,6 @@ export class ToolDispatcher {
|
||||
const params = (req.params as { rpc_id?: string } | undefined) ?? {};
|
||||
const target = typeof params.rpc_id === "string" ? params.rpc_id : "";
|
||||
const ac = target ? this.inflightAbortControllers.get(target) : undefined;
|
||||
this.diagnostics?.("dispatcher.cancel.received", {
|
||||
rpc_id: req.id,
|
||||
target_rpc_id: target,
|
||||
target_found: ac !== undefined,
|
||||
inflight_rpc_ids: Array.from(this.inflightAbortControllers.keys()),
|
||||
});
|
||||
if (ac) {
|
||||
try {
|
||||
ac.abort();
|
||||
@@ -233,17 +223,7 @@ export class ToolDispatcher {
|
||||
};
|
||||
try {
|
||||
this.transport.send(reply);
|
||||
this.diagnostics?.("dispatcher.cancel.ack_sent", {
|
||||
rpc_id: req.id,
|
||||
target_rpc_id: target,
|
||||
cancelled: ac !== undefined,
|
||||
});
|
||||
} catch (sendErr) {
|
||||
this.diagnostics?.("dispatcher.cancel.ack_failed", {
|
||||
rpc_id: req.id,
|
||||
target_rpc_id: target,
|
||||
error: sendErr,
|
||||
});
|
||||
console.warn("[bsk dispatcher] failed to ack cancel", sendErr);
|
||||
}
|
||||
return;
|
||||
@@ -260,7 +240,7 @@ export class ToolDispatcher {
|
||||
if (sessionId) this.onBrowserControlResumed?.(sessionId);
|
||||
const result = await this.invoke(req, ac.signal);
|
||||
if (isRpcError(result)) {
|
||||
body = { id: req.id, error: result };
|
||||
body = { id: req.id, error: classifyCdpError(result) };
|
||||
} else {
|
||||
body = { id: req.id, result };
|
||||
if (req.method === "tool.session_start") {
|
||||
@@ -268,13 +248,6 @@ export class ToolDispatcher {
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
this.diagnostics?.("dispatcher.rpc.invoke_failed", {
|
||||
rpc_id: req.id,
|
||||
method: req.method,
|
||||
session_id: typeof requestSessionId === "string" ? requestSessionId : null,
|
||||
abort_signal: ac.signal.aborted,
|
||||
error: err,
|
||||
});
|
||||
if (isAbortLikeError(err)) {
|
||||
body = {
|
||||
id: req.id,
|
||||
@@ -295,27 +268,8 @@ export class ToolDispatcher {
|
||||
let sent = true;
|
||||
try {
|
||||
this.transport.send(body);
|
||||
this.diagnostics?.("dispatcher.rpc.response_sent", {
|
||||
rpc_id: req.id,
|
||||
method: req.method,
|
||||
session_id: typeof requestSessionId === "string" ? requestSessionId : null,
|
||||
response_kind: "error" in body ? "error" : "ok",
|
||||
error_code: "error" in body ? body.error.code : null,
|
||||
elapsed_ms: Math.round(performance.now() - startedAt),
|
||||
transport_state: this.transport.state,
|
||||
live_session_ids: this.sessions.list().map((ctx) => ctx.sessionId),
|
||||
});
|
||||
} catch (sendErr) {
|
||||
sent = false;
|
||||
this.diagnostics?.("dispatcher.rpc.response_send_failed", {
|
||||
rpc_id: req.id,
|
||||
method: req.method,
|
||||
session_id: typeof requestSessionId === "string" ? requestSessionId : null,
|
||||
started_session_id: startedSession,
|
||||
elapsed_ms: Math.round(performance.now() - startedAt),
|
||||
transport_state: this.transport.state,
|
||||
error: sendErr,
|
||||
});
|
||||
// Transport is dead by the time we want to reply. Drop the link
|
||||
// proactively so the alarm-driven keepalive reconnects sooner
|
||||
// and the daemon's pending RPC times out cleanly instead of
|
||||
@@ -334,23 +288,12 @@ export class ToolDispatcher {
|
||||
try {
|
||||
const ctx = await this.sessions.stop(startedSession);
|
||||
if (ctx) {
|
||||
this.diagnostics?.("dispatcher.session_start.rolled_back", {
|
||||
rpc_id: req.id,
|
||||
session_id: startedSession,
|
||||
agent_window_id: ctx.agentWindowId,
|
||||
reason: "response_send_failed",
|
||||
});
|
||||
console.warn(
|
||||
"[bsk dispatcher] rolled back orphan session after send failure",
|
||||
startedSession,
|
||||
);
|
||||
}
|
||||
} catch (rollbackErr) {
|
||||
this.diagnostics?.("dispatcher.session_start.rollback_failed", {
|
||||
rpc_id: req.id,
|
||||
session_id: startedSession,
|
||||
error: rollbackErr,
|
||||
});
|
||||
console.warn("[bsk dispatcher] session rollback after send failure failed", rollbackErr);
|
||||
}
|
||||
}
|
||||
@@ -365,13 +308,25 @@ export class ToolDispatcher {
|
||||
await this.releaseHoverLatch((req.params as SessionStopParams).session_id);
|
||||
return handleSessionStop(this.sessions, req.params as SessionStopParams, {
|
||||
cdp: this.cdp,
|
||||
// Must be wired in production: the agent-tab cleanup and the
|
||||
// window-release decision (issue #57) read these deps directly
|
||||
// and silently no-op when they are absent.
|
||||
tabManagement: { tabs: chromeTabMutationApi },
|
||||
tabsQuery: chromeTabsApi,
|
||||
signal,
|
||||
});
|
||||
}
|
||||
case "tool.tab_list":
|
||||
return handleTabList(this.sessions, req.params as TabListParams, chromeTabsApi, signal);
|
||||
case "tool.tab_create":
|
||||
return handleTabCreate(this.sessions, req.params as TabCreateParams, { signal });
|
||||
case "tool.tab_create": {
|
||||
const result = await handleTabCreate(this.sessions, req.params as TabCreateParams, {
|
||||
signal,
|
||||
});
|
||||
if (!isRpcError(result)) {
|
||||
this.onAgentTabClaimed?.(result.tab_id, result.window_id);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
case "tool.tab_close":
|
||||
return this.withHoverReleaseForRequest(
|
||||
req.params as TabCloseParams,
|
||||
@@ -380,13 +335,22 @@ export class ToolDispatcher {
|
||||
);
|
||||
case "tool.tab_select":
|
||||
return handleTabSelect(this.sessions, req.params as TabSelectParams, { signal });
|
||||
case "tool.tab_borrow":
|
||||
return handleTabBorrow(this.sessions, req.params as TabBorrowParams, {
|
||||
case "tool.tab_borrow": {
|
||||
const result = await handleTabBorrow(this.sessions, req.params as TabBorrowParams, {
|
||||
signal,
|
||||
approveBorrow: this.approveBorrow,
|
||||
});
|
||||
if (!isRpcError(result)) {
|
||||
this.onAgentTabClaimed?.(result.tab_id, result.agent_window_id);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
case "tool.tab_return":
|
||||
return handleTabReturn(this.sessions, req.params as TabReturnParams, { signal });
|
||||
return handleTabReturn(this.sessions, req.params as TabReturnParams, {
|
||||
signal,
|
||||
cdp: this.cdp,
|
||||
beforeReturn: (sessionId, tabId) => this.releaseHoverLatch(sessionId, tabId),
|
||||
});
|
||||
case "tool.window_resize":
|
||||
return handleWindowResize(
|
||||
this.sessions,
|
||||
@@ -450,7 +414,11 @@ export class ToolDispatcher {
|
||||
? {
|
||||
cdp: this.cdp,
|
||||
tabsApi: chromeTabsCaptureApi,
|
||||
conditionalSurfaceProbe: !this.hasHoverLatchForScope(hoverScope),
|
||||
// Active hover probing is opt-in. A held hover latch still
|
||||
// suppresses it, because probing would move the cursor off
|
||||
// the element the caller is deliberately holding.
|
||||
conditionalSurfaceProbe:
|
||||
params.probe_hover === true && !this.hasHoverLatchForScope(hoverScope),
|
||||
hoverProbeBypassOverlay: bypassOverlay,
|
||||
}
|
||||
: undefined,
|
||||
@@ -580,6 +548,40 @@ export class ToolDispatcher {
|
||||
),
|
||||
signal,
|
||||
);
|
||||
case "tool.upload":
|
||||
return this.withHoverReleaseForRequest(
|
||||
req.params as UploadParams,
|
||||
() =>
|
||||
this.cdp
|
||||
? handleUpload(this.sessions, req.params as UploadParams, {
|
||||
cdp: this.cdp,
|
||||
tabsApi: chromeTabsApi,
|
||||
signal,
|
||||
bypassOverlay,
|
||||
})
|
||||
: Promise.resolve({
|
||||
code: "unsupported",
|
||||
message: "upload requires CDP",
|
||||
} satisfies RpcError),
|
||||
signal,
|
||||
);
|
||||
case "tool.download":
|
||||
return this.withHoverReleaseForRequest(
|
||||
req.params as DownloadParams,
|
||||
() =>
|
||||
this.cdp
|
||||
? handleDownload(this.sessions, req.params as DownloadParams, {
|
||||
cdp: this.cdp,
|
||||
tabsApi: chromeTabsApi,
|
||||
signal,
|
||||
bypassOverlay,
|
||||
})
|
||||
: Promise.resolve({
|
||||
code: "unsupported",
|
||||
message: "download requires CDP",
|
||||
} satisfies RpcError),
|
||||
signal,
|
||||
);
|
||||
case "tool.evaluate":
|
||||
return handleEvaluate(
|
||||
this.sessions,
|
||||
@@ -786,6 +788,8 @@ function sessionIdForBrowserControlMethod(req: RequestFrame): string | null {
|
||||
case "tool.fill":
|
||||
case "tool.press":
|
||||
case "tool.select":
|
||||
case "tool.upload":
|
||||
case "tool.download":
|
||||
case "tool.evaluate":
|
||||
case "tool.observe":
|
||||
case "tool.request_help":
|
||||
|
||||
@@ -0,0 +1,364 @@
|
||||
// Order-independent coordinator for one browser download. CDP supplies the
|
||||
// exact target/frame intent while chrome.downloads supplies the download id
|
||||
// and filename routing hook; neither event is assumed to arrive first.
|
||||
|
||||
import type { CdpTarget } from "@/browser-driver/frame-graph";
|
||||
import type { ClickResult, RpcError, TransferEffectState } from "@/transport/types";
|
||||
import { transferError } from "./errors";
|
||||
import { type CdpRunner, isRpcError } from "./shared";
|
||||
|
||||
const CORRELATION_GRACE_MS = 750;
|
||||
const UNIQUE_SETTLE_MS = 50;
|
||||
const SIZE_POLL_MS = 250;
|
||||
|
||||
type DeterminingFilenameListener = (
|
||||
item: chrome.downloads.DownloadItem,
|
||||
suggest: (suggestion?: chrome.downloads.DownloadFilenameSuggestion) => void,
|
||||
) => void | true;
|
||||
|
||||
interface ListenerEvent<T> {
|
||||
addListener(listener: T): void;
|
||||
removeListener(listener: T): void;
|
||||
}
|
||||
|
||||
export interface DownloadsApi {
|
||||
onCreated: ListenerEvent<(item: chrome.downloads.DownloadItem) => void>;
|
||||
onChanged: ListenerEvent<(delta: chrome.downloads.DownloadDelta) => void>;
|
||||
onDeterminingFilename: ListenerEvent<DeterminingFilenameListener>;
|
||||
search(query: chrome.downloads.DownloadQuery): Promise<chrome.downloads.DownloadItem[]>;
|
||||
cancel(downloadId: number): Promise<void>;
|
||||
removeFile(downloadId: number): Promise<void>;
|
||||
}
|
||||
|
||||
export const chromeDownloadsApi: DownloadsApi = {
|
||||
get onCreated() {
|
||||
return chrome.downloads.onCreated;
|
||||
},
|
||||
get onChanged() {
|
||||
return chrome.downloads.onChanged;
|
||||
},
|
||||
get onDeterminingFilename() {
|
||||
return chrome.downloads.onDeterminingFilename;
|
||||
},
|
||||
search: (query) => chrome.downloads.search(query),
|
||||
cancel: (id) => chrome.downloads.cancel(id),
|
||||
removeFile: (id) => chrome.downloads.removeFile(id),
|
||||
};
|
||||
|
||||
export interface DownloadCaptureOptions {
|
||||
cdp: CdpRunner;
|
||||
target: CdpTarget;
|
||||
expectedFrameId?: string;
|
||||
downloads: DownloadsApi;
|
||||
browserRelativeDir: string;
|
||||
maxByteSize?: number;
|
||||
timeoutMs: number;
|
||||
signal?: AbortSignal;
|
||||
trigger(): Promise<ClickResult | RpcError>;
|
||||
}
|
||||
|
||||
export interface DownloadCaptureResult {
|
||||
click: ClickResult;
|
||||
item: chrome.downloads.DownloadItem;
|
||||
}
|
||||
|
||||
interface DownloadIntent {
|
||||
url: string;
|
||||
suggestedFilename: string;
|
||||
frameId?: string;
|
||||
}
|
||||
|
||||
interface DownloadCandidate {
|
||||
item: chrome.downloads.DownloadItem;
|
||||
suggest: (suggestion?: chrome.downloads.DownloadFilenameSuggestion) => void;
|
||||
suggested: boolean;
|
||||
graceTimer: ReturnType<typeof setTimeout>;
|
||||
}
|
||||
|
||||
function safeBasename(filename: string): string {
|
||||
const basename = filename.split(/[\\/]/).pop()?.trim();
|
||||
return basename && basename !== "." && basename !== ".." ? basename : "download";
|
||||
}
|
||||
|
||||
function sameTarget(source: { tabId?: number; sessionId?: string }, target: CdpTarget): boolean {
|
||||
return source.tabId === target.tabId && source.sessionId === target.sessionId;
|
||||
}
|
||||
|
||||
function matchesIntent(item: chrome.downloads.DownloadItem, intent: DownloadIntent): boolean {
|
||||
const urlMatches = item.url === intent.url || item.finalUrl === intent.url;
|
||||
return urlMatches && safeBasename(item.filename) === safeBasename(intent.suggestedFilename);
|
||||
}
|
||||
|
||||
function knownSize(item: chrome.downloads.DownloadItem): number | undefined {
|
||||
if (item.fileSize >= 0) return item.fileSize;
|
||||
if (item.totalBytes >= 0) return item.totalBytes;
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function captureError(
|
||||
message: string,
|
||||
effectState: TransferEffectState,
|
||||
phase: string,
|
||||
cleanupFailed = false,
|
||||
): RpcError {
|
||||
return transferError("cdp_failed", "download_capture_failed", message, {
|
||||
effectState,
|
||||
phase,
|
||||
...(cleanupFailed ? { cleanupState: "failed" } : {}),
|
||||
});
|
||||
}
|
||||
|
||||
async function cleanupClaimedDownload(downloads: DownloadsApi, downloadId: number): Promise<void> {
|
||||
const lookup = async () => (await downloads.search({ id: downloadId }))[0];
|
||||
const item = await lookup();
|
||||
if (!item || item.state === "interrupted") return;
|
||||
if (item.state === "complete") {
|
||||
await downloads.removeFile(downloadId);
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
await downloads.cancel(downloadId);
|
||||
} catch (cancelError) {
|
||||
// Completion can win the race after the lookup but before cancellation.
|
||||
// Reconcile against Chrome's authoritative state before declaring cleanup
|
||||
// failed so every terminal state has one explicit cleanup path.
|
||||
const reconciled = await lookup();
|
||||
if (!reconciled || reconciled.state === "interrupted") return;
|
||||
if (reconciled.state === "complete") {
|
||||
await downloads.removeFile(downloadId);
|
||||
return;
|
||||
}
|
||||
throw cancelError;
|
||||
}
|
||||
}
|
||||
|
||||
export async function captureBrowserDownload(
|
||||
options: DownloadCaptureOptions,
|
||||
): Promise<DownloadCaptureResult | RpcError> {
|
||||
let click: ClickResult | undefined;
|
||||
let intent: DownloadIntent | undefined;
|
||||
let capturedId: number | undefined;
|
||||
let settled = false;
|
||||
let succeeded = false;
|
||||
let failureResult: RpcError | undefined;
|
||||
let uniquenessTimer: ReturnType<typeof setTimeout> | undefined;
|
||||
let operationTimer: ReturnType<typeof setTimeout> | undefined;
|
||||
let sizePoll: ReturnType<typeof setInterval> | undefined;
|
||||
const candidates = new Map<number, DownloadCandidate>();
|
||||
const createdItems = new Map<number, chrome.downloads.DownloadItem>();
|
||||
|
||||
let resolveCompletion!: (item: chrome.downloads.DownloadItem) => void;
|
||||
let rejectCompletion!: (error: Error) => void;
|
||||
const completion = new Promise<chrome.downloads.DownloadItem>((resolve, reject) => {
|
||||
resolveCompletion = resolve;
|
||||
rejectCompletion = reject;
|
||||
});
|
||||
const fail = (error: Error) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
rejectCompletion(error);
|
||||
};
|
||||
const complete = (item: chrome.downloads.DownloadItem) => {
|
||||
if (settled) return;
|
||||
const size = knownSize(item);
|
||||
if (size !== undefined && options.maxByteSize !== undefined && size > options.maxByteSize) {
|
||||
fail(new Error(`download exceeds transfer limit ${options.maxByteSize}`));
|
||||
return;
|
||||
}
|
||||
settled = true;
|
||||
resolveCompletion(item);
|
||||
};
|
||||
const suggestDefault = (candidate: DownloadCandidate) => {
|
||||
if (candidate.suggested) return;
|
||||
candidate.suggested = true;
|
||||
candidate.suggest();
|
||||
};
|
||||
const matchingCandidates = (): DownloadCandidate[] => {
|
||||
const currentIntent = intent;
|
||||
return currentIntent
|
||||
? [...candidates.values()].filter(
|
||||
(candidate) => !candidate.suggested && matchesIntent(candidate.item, currentIntent),
|
||||
)
|
||||
: [];
|
||||
};
|
||||
|
||||
const claimUnique = () => {
|
||||
uniquenessTimer = undefined;
|
||||
if (settled || capturedId !== undefined || !intent) return;
|
||||
const matches = matchingCandidates();
|
||||
if (matches.length !== 1) {
|
||||
if (matches.length > 1) {
|
||||
for (const candidate of matches) suggestDefault(candidate);
|
||||
fail(new Error("download attribution is ambiguous"));
|
||||
}
|
||||
return;
|
||||
}
|
||||
const candidate = matches[0];
|
||||
candidate.suggested = true;
|
||||
clearTimeout(candidate.graceTimer);
|
||||
capturedId = candidate.item.id;
|
||||
candidate.suggest({
|
||||
filename: `${options.browserRelativeDir}/${safeBasename(intent.suggestedFilename)}`,
|
||||
conflictAction: "overwrite",
|
||||
});
|
||||
const size = knownSize(candidate.item);
|
||||
if (size !== undefined && options.maxByteSize !== undefined && size > options.maxByteSize) {
|
||||
fail(new Error(`download exceeds transfer limit ${options.maxByteSize}`));
|
||||
return;
|
||||
}
|
||||
const created = createdItems.get(candidate.item.id);
|
||||
if (created?.state === "interrupted") {
|
||||
fail(new Error(created.error ?? "download interrupted"));
|
||||
} else if (created?.state === "complete") {
|
||||
complete(created);
|
||||
}
|
||||
};
|
||||
const reconcile = () => {
|
||||
if (settled || capturedId !== undefined || !intent) return;
|
||||
const matches = matchingCandidates();
|
||||
if (matches.length > 1) {
|
||||
for (const candidate of matches) suggestDefault(candidate);
|
||||
fail(new Error("download attribution is ambiguous"));
|
||||
return;
|
||||
}
|
||||
if (matches.length === 1 && !uniquenessTimer) {
|
||||
uniquenessTimer = setTimeout(claimUnique, UNIQUE_SETTLE_MS);
|
||||
}
|
||||
};
|
||||
|
||||
const determiningListener: DeterminingFilenameListener = (item, suggest) => {
|
||||
const candidate: DownloadCandidate = {
|
||||
item,
|
||||
suggest,
|
||||
suggested: false,
|
||||
graceTimer: setTimeout(() => {
|
||||
suggestDefault(candidate);
|
||||
candidates.delete(item.id);
|
||||
if (intent && matchesIntent(item, intent) && capturedId === undefined) {
|
||||
fail(new Error("download correlation grace elapsed before unique attribution"));
|
||||
}
|
||||
}, CORRELATION_GRACE_MS),
|
||||
};
|
||||
candidates.set(item.id, candidate);
|
||||
reconcile();
|
||||
return true;
|
||||
};
|
||||
const createdListener = (item: chrome.downloads.DownloadItem) => {
|
||||
createdItems.set(item.id, item);
|
||||
if (capturedId !== item.id) return;
|
||||
if (item.state === "interrupted") {
|
||||
fail(new Error(item.error ?? "download interrupted"));
|
||||
} else if (item.state === "complete") {
|
||||
complete(item);
|
||||
}
|
||||
};
|
||||
const changedListener = async (delta: chrome.downloads.DownloadDelta) => {
|
||||
if (capturedId === undefined || delta.id !== capturedId || settled) return;
|
||||
if (delta.state?.current === "interrupted" || delta.error?.current) {
|
||||
fail(new Error(delta.error?.current ?? "download interrupted"));
|
||||
return;
|
||||
}
|
||||
if (delta.state?.current === "complete") {
|
||||
try {
|
||||
const [item] = await options.downloads.search({ id: delta.id });
|
||||
if (item) complete(item);
|
||||
else fail(new Error("completed download disappeared"));
|
||||
} catch (err) {
|
||||
fail(err instanceof Error ? err : new Error(String(err)));
|
||||
}
|
||||
}
|
||||
};
|
||||
const onAbort = () => fail(new DOMException("aborted", "AbortError"));
|
||||
const cdpSubscription = options.cdp.onEvent?.((source, method, raw) => {
|
||||
if (method !== "Page.downloadWillBegin" || !sameTarget(source, options.target)) return;
|
||||
const event = raw as { url?: unknown; suggestedFilename?: unknown; frameId?: unknown };
|
||||
if (typeof event.url !== "string" || typeof event.suggestedFilename !== "string") return;
|
||||
if (options.expectedFrameId && event.frameId !== options.expectedFrameId) {
|
||||
fail(new Error("download originated from a different frame"));
|
||||
return;
|
||||
}
|
||||
if (intent) {
|
||||
fail(new Error("download trigger produced more than one browser download intent"));
|
||||
return;
|
||||
}
|
||||
intent = {
|
||||
url: event.url,
|
||||
suggestedFilename: event.suggestedFilename,
|
||||
...(typeof event.frameId === "string" ? { frameId: event.frameId } : {}),
|
||||
};
|
||||
reconcile();
|
||||
});
|
||||
if (!cdpSubscription) {
|
||||
return captureError("CDP download intent subscription unavailable", "none", "arm");
|
||||
}
|
||||
|
||||
options.downloads.onDeterminingFilename.addListener(determiningListener);
|
||||
options.downloads.onCreated.addListener(createdListener);
|
||||
options.downloads.onChanged.addListener(changedListener);
|
||||
options.signal?.addEventListener("abort", onAbort, { once: true });
|
||||
operationTimer = setTimeout(
|
||||
() => fail(new Error("download did not complete before timeout")),
|
||||
options.timeoutMs,
|
||||
);
|
||||
sizePoll = setInterval(() => {
|
||||
if (capturedId === undefined || settled || options.maxByteSize === undefined) return;
|
||||
void options.downloads
|
||||
.search({ id: capturedId })
|
||||
.then(([item]) => {
|
||||
if (!item || settled) return;
|
||||
if (item.bytesReceived > (options.maxByteSize as number)) {
|
||||
fail(new Error(`download exceeds transfer limit ${options.maxByteSize}`));
|
||||
}
|
||||
})
|
||||
.catch((err) => fail(err instanceof Error ? err : new Error(String(err))));
|
||||
}, SIZE_POLL_MS);
|
||||
|
||||
try {
|
||||
const triggered = await options.trigger();
|
||||
if (isRpcError(triggered)) {
|
||||
void completion.catch(() => undefined);
|
||||
const effect: TransferEffectState =
|
||||
capturedId !== undefined ? "committed" : intent ? "unknown" : "none";
|
||||
failureResult = {
|
||||
...triggered,
|
||||
data: { ...triggered.data, effect_state: effect, phase: "trigger" },
|
||||
};
|
||||
return failureResult;
|
||||
}
|
||||
click = triggered;
|
||||
const item = await completion;
|
||||
succeeded = true;
|
||||
return { click, item };
|
||||
} catch (err) {
|
||||
const effect: TransferEffectState =
|
||||
capturedId !== undefined ? "committed" : click ? "unknown" : "none";
|
||||
failureResult = captureError(
|
||||
err instanceof Error ? err.message : String(err),
|
||||
effect,
|
||||
capturedId !== undefined ? "download" : "attribution",
|
||||
);
|
||||
return failureResult;
|
||||
} finally {
|
||||
settled = true;
|
||||
if (operationTimer) clearTimeout(operationTimer);
|
||||
if (uniquenessTimer) clearTimeout(uniquenessTimer);
|
||||
if (sizePoll) clearInterval(sizePoll);
|
||||
options.signal?.removeEventListener("abort", onAbort);
|
||||
options.downloads.onDeterminingFilename.removeListener(determiningListener);
|
||||
options.downloads.onCreated.removeListener(createdListener);
|
||||
options.downloads.onChanged.removeListener(changedListener);
|
||||
cdpSubscription.dispose();
|
||||
for (const candidate of candidates.values()) {
|
||||
clearTimeout(candidate.graceTimer);
|
||||
if (candidate.item.id !== capturedId) suggestDefault(candidate);
|
||||
}
|
||||
if (!succeeded && capturedId !== undefined) {
|
||||
try {
|
||||
await cleanupClaimedDownload(options.downloads, capturedId);
|
||||
} catch {
|
||||
if (failureResult?.data) failureResult.data.cleanup_state = "failed";
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
// Download orchestration: validate session/tab ownership, then delegate one
|
||||
// browser-global chrome.downloads transaction to download-capture.ts.
|
||||
|
||||
import type { SessionManager } from "@/session-manager/manager";
|
||||
import type { DownloadParams, DownloadResult, RpcError } from "@/transport/types";
|
||||
import { captureBrowserDownload, chromeDownloadsApi, type DownloadsApi } from "./download-capture";
|
||||
import { clickResolvedTarget, type InteractionDeps, resolveActionTarget } from "./interaction";
|
||||
import { enforceAgentWindow, isRpcError, lookupSession, resolveTargetTab } from "./shared";
|
||||
|
||||
let downloadActive = false;
|
||||
|
||||
export type { DownloadsApi } from "./download-capture";
|
||||
|
||||
export interface DownloadDeps extends InteractionDeps {
|
||||
downloads?: DownloadsApi;
|
||||
}
|
||||
|
||||
export async function handleDownload(
|
||||
manager: SessionManager,
|
||||
params: DownloadParams,
|
||||
deps: DownloadDeps,
|
||||
): Promise<DownloadResult | RpcError> {
|
||||
if (downloadActive) return { code: "invalid_params", message: "another bsk download is active" };
|
||||
downloadActive = true;
|
||||
try {
|
||||
const ctx = lookupSession(manager, params, "download");
|
||||
if (isRpcError(ctx)) return ctx;
|
||||
const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi);
|
||||
if (isRpcError(target)) return target;
|
||||
const denied = enforceAgentWindow(ctx, target, "download");
|
||||
if (denied) return denied;
|
||||
if (!params.browser_relative_dir) {
|
||||
return { code: "invalid_params", message: "download requires a daemon capability directory" };
|
||||
}
|
||||
const address = await resolveActionTarget(deps.cdp, ctx, target, params, "download");
|
||||
if (isRpcError(address)) return address;
|
||||
|
||||
const capture = await captureBrowserDownload({
|
||||
cdp: deps.cdp,
|
||||
target: address.cdpTarget,
|
||||
downloads: deps.downloads ?? chromeDownloadsApi,
|
||||
browserRelativeDir: params.browser_relative_dir,
|
||||
maxByteSize: params.max_byte_size,
|
||||
timeoutMs: params.timeout_ms ?? 120_000,
|
||||
signal: deps.signal,
|
||||
expectedFrameId: address.frameId,
|
||||
trigger: () => clickResolvedTarget(ctx, address, {}, deps),
|
||||
});
|
||||
if (isRpcError(capture)) return capture;
|
||||
const { click, item } = capture;
|
||||
return {
|
||||
tab_id: target.tabId,
|
||||
used_ref: click.used_ref,
|
||||
used_selector: click.used_selector,
|
||||
suggested_filename: item.filename.split(/[\\/]/).pop() ?? "download",
|
||||
byte_size: item.fileSize >= 0 ? item.fileSize : item.totalBytes,
|
||||
mime: item.mime || undefined,
|
||||
danger: item.danger,
|
||||
browser_path: item.filename,
|
||||
};
|
||||
} finally {
|
||||
downloadActive = false;
|
||||
}
|
||||
}
|
||||
@@ -2,10 +2,51 @@
|
||||
// rendering. Extension handlers attach reasons here; human-facing copy
|
||||
// lives in bsk-cli `render_error.rs`.
|
||||
|
||||
import type { ErrorCode, RpcError, RpcErrorData, RpcErrorReason } from "@/transport/types";
|
||||
import type {
|
||||
ErrorCode,
|
||||
RpcError,
|
||||
RpcErrorData,
|
||||
RpcErrorReason,
|
||||
TransferCleanupState,
|
||||
TransferEffectState,
|
||||
} from "@/transport/types";
|
||||
|
||||
export type { RpcErrorData, RpcErrorReason };
|
||||
|
||||
/** Chrome checks the whole tab's frame tree, including other extensions' frames. */
|
||||
export function isCdpExtensionAccessDenied(error: unknown): boolean {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
return message.includes("Cannot access a chrome-extension:// URL of different extension");
|
||||
}
|
||||
|
||||
/** Preserve more specific reasons, especially file-transfer outcome/cleanup details. */
|
||||
export function classifyCdpError(error: RpcError): RpcError {
|
||||
if (
|
||||
error.code !== "cdp_failed" ||
|
||||
error.data?.reason ||
|
||||
!isCdpExtensionAccessDenied(error.message)
|
||||
) {
|
||||
return error;
|
||||
}
|
||||
return {
|
||||
...error,
|
||||
data: { ...error.data, reason: "cdp_extension_access_denied" },
|
||||
};
|
||||
}
|
||||
|
||||
export function cdpError(error: unknown): RpcError {
|
||||
return classifyCdpError({
|
||||
code: "cdp_failed",
|
||||
message: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
}
|
||||
|
||||
export interface TransferErrorOptions {
|
||||
effectState: TransferEffectState;
|
||||
phase: string;
|
||||
cleanupState?: TransferCleanupState;
|
||||
}
|
||||
|
||||
export function rpcError(
|
||||
code: ErrorCode,
|
||||
reason: RpcErrorReason,
|
||||
@@ -15,3 +56,16 @@ export function rpcError(
|
||||
const data: RpcErrorData = { reason, ...extra };
|
||||
return { code, message, data };
|
||||
}
|
||||
|
||||
export function transferError(
|
||||
code: ErrorCode,
|
||||
reason: RpcErrorReason,
|
||||
message: string,
|
||||
options: TransferErrorOptions,
|
||||
): RpcError {
|
||||
return rpcError(code, reason, message, {
|
||||
effect_state: options.effectState,
|
||||
phase: options.phase,
|
||||
...(options.cleanupState ? { cleanup_state: options.cleanupState } : {}),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -0,0 +1,319 @@
|
||||
// One native file-drop transaction bound to an explicit page target. CDP drag
|
||||
// events are sent to the target that owns the node: root targets use top-level
|
||||
// coordinates, while OOPIF targets require coordinates local to their session.
|
||||
|
||||
import type { RpcError, TransferEffectState } from "@/transport/types";
|
||||
import { transferError } from "./errors";
|
||||
import { resolveNodeGeometry } from "./frame-geometry";
|
||||
import type { ResolvedActionTarget } from "./interaction";
|
||||
import { type CdpRunner, isRpcError, sendToCdpTarget } from "./shared";
|
||||
import { BoundedWaitError, waitBounded } from "./transfer-transaction";
|
||||
|
||||
const CLEANUP_TIMEOUT_MS = 1_000;
|
||||
|
||||
type FileDropPhase = "resolve_target" | "drag_enter" | "drag_over" | "drop" | "cleanup";
|
||||
|
||||
interface RuntimeReply {
|
||||
result?: { value?: unknown; objectId?: string };
|
||||
exceptionDetails?: { text?: string; exception?: { description?: string } };
|
||||
}
|
||||
|
||||
interface DragData {
|
||||
items: [];
|
||||
files: string[];
|
||||
dragOperationsMask: number;
|
||||
}
|
||||
|
||||
export interface FileDropTransactionOptions {
|
||||
cdp: CdpRunner;
|
||||
actionTarget: ResolvedActionTarget;
|
||||
files: string[];
|
||||
timeoutMs: number;
|
||||
signal?: AbortSignal;
|
||||
withOverlayHidden: <T>(operation: () => Promise<T>) => Promise<T>;
|
||||
}
|
||||
|
||||
export interface FileDropTransactionResult {
|
||||
x: number;
|
||||
y: number;
|
||||
}
|
||||
|
||||
function failure(
|
||||
code: RpcError["code"],
|
||||
reason: "upload_mechanism_unsupported" | "file_drop_target_unavailable" | "file_drop_failed",
|
||||
message: string,
|
||||
effectState: TransferEffectState,
|
||||
phase: FileDropPhase,
|
||||
): RpcError {
|
||||
const error = transferError(code, reason, message, { effectState, phase });
|
||||
error.data = { ...error.data, mechanism: "drop" };
|
||||
return error;
|
||||
}
|
||||
|
||||
function enrichFailure(
|
||||
error: RpcError,
|
||||
effectState: TransferEffectState,
|
||||
phase: FileDropPhase,
|
||||
): RpcError {
|
||||
return {
|
||||
...error,
|
||||
data: {
|
||||
...error.data,
|
||||
mechanism: "drop",
|
||||
effect_state: effectState,
|
||||
phase,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function messageOf(error: unknown): string {
|
||||
return error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
|
||||
function methodUnavailable(error: unknown): boolean {
|
||||
return /(?:method|command).*(?:not found|wasn't found|unsupported)|wasn't found|-32601/i.test(
|
||||
messageOf(error),
|
||||
);
|
||||
}
|
||||
|
||||
function runtimeError(reply: RuntimeReply, fallback: string): Error | null {
|
||||
if (!reply.exceptionDetails) return null;
|
||||
return new Error(
|
||||
reply.exceptionDetails.exception?.description ?? reply.exceptionDetails.text ?? fallback,
|
||||
);
|
||||
}
|
||||
|
||||
async function pointBelongsToTarget(
|
||||
options: FileDropTransactionOptions,
|
||||
x: number,
|
||||
y: number,
|
||||
objectGroup: string,
|
||||
): Promise<boolean> {
|
||||
const target = options.actionTarget.cdpTarget;
|
||||
const hit = await sendToCdpTarget<{ backendNodeId?: number; frameId?: string }>(
|
||||
options.cdp,
|
||||
target,
|
||||
"DOM.getNodeForLocation",
|
||||
{ x: Math.round(x), y: Math.round(y), includeUserAgentShadowDOM: true },
|
||||
);
|
||||
if (typeof hit.backendNodeId !== "number") return false;
|
||||
if (options.actionTarget.frameId && hit.frameId !== options.actionTarget.frameId) return false;
|
||||
|
||||
const [resolvedTarget, resolvedHit] = await Promise.all([
|
||||
sendToCdpTarget<{ object?: { objectId?: string } }>(options.cdp, target, "DOM.resolveNode", {
|
||||
backendNodeId: options.actionTarget.backendNodeId,
|
||||
objectGroup,
|
||||
}),
|
||||
sendToCdpTarget<{ object?: { objectId?: string } }>(options.cdp, target, "DOM.resolveNode", {
|
||||
backendNodeId: hit.backendNodeId,
|
||||
objectGroup,
|
||||
}),
|
||||
]);
|
||||
const targetObjectId = resolvedTarget.object?.objectId;
|
||||
const hitObjectId = resolvedHit.object?.objectId;
|
||||
if (!targetObjectId || !hitObjectId) return false;
|
||||
|
||||
const reply = await sendToCdpTarget<RuntimeReply>(options.cdp, target, "Runtime.callFunctionOn", {
|
||||
objectId: targetObjectId,
|
||||
functionDeclaration: `function(hit) {
|
||||
for (let node = hit; node; ) {
|
||||
if (node === this) return true;
|
||||
if (this.contains?.(node)) return true;
|
||||
const root = node.getRootNode?.();
|
||||
node = node.parentNode || (root && root !== node ? root.host : null);
|
||||
}
|
||||
return false;
|
||||
}`,
|
||||
arguments: [{ objectId: hitObjectId }],
|
||||
returnByValue: true,
|
||||
});
|
||||
const error = runtimeError(reply, "failed to verify file-drop target");
|
||||
if (error) throw error;
|
||||
return reply.result?.value === true;
|
||||
}
|
||||
|
||||
export async function uploadThroughFileDrop(
|
||||
options: FileDropTransactionOptions,
|
||||
): Promise<FileDropTransactionResult | RpcError> {
|
||||
const deadline = Date.now() + options.timeoutMs;
|
||||
const target = options.actionTarget.cdpTarget;
|
||||
const objectGroup = `bsk-file-drop-${crypto.randomUUID()}`;
|
||||
const dragData: DragData = {
|
||||
items: [],
|
||||
files: options.files,
|
||||
dragOperationsMask: 1,
|
||||
};
|
||||
let dragStarted = false;
|
||||
let dropSent = false;
|
||||
let dragPoint = { x: 0, y: 0 };
|
||||
let cleanupFailed = false;
|
||||
let outcome: FileDropTransactionResult | RpcError = failure(
|
||||
"protocol_error",
|
||||
"file_drop_failed",
|
||||
"file-drop transaction ended without an outcome",
|
||||
"none",
|
||||
"cleanup",
|
||||
);
|
||||
|
||||
try {
|
||||
const geometry = await waitBounded(
|
||||
resolveNodeGeometry(
|
||||
options.cdp,
|
||||
options.actionTarget.tab.tabId,
|
||||
{
|
||||
target,
|
||||
backendNodeId: options.actionTarget.backendNodeId,
|
||||
...(options.actionTarget.frameId ? { frameId: options.actionTarget.frameId } : {}),
|
||||
},
|
||||
{ scrollIntoView: true },
|
||||
),
|
||||
deadline,
|
||||
options.signal,
|
||||
"resolving file-drop target timed out",
|
||||
);
|
||||
if (isRpcError(geometry)) {
|
||||
outcome = enrichFailure(geometry, "none", "resolve_target");
|
||||
return outcome;
|
||||
}
|
||||
const point = geometry.targetActionPoint;
|
||||
dragPoint = point;
|
||||
|
||||
outcome = await options.withOverlayHidden(async () => {
|
||||
try {
|
||||
try {
|
||||
const belongs = await waitBounded(
|
||||
pointBelongsToTarget(options, point.x, point.y, objectGroup),
|
||||
deadline,
|
||||
options.signal,
|
||||
"verifying file-drop target timed out",
|
||||
);
|
||||
if (!belongs) {
|
||||
return failure(
|
||||
"permission_denied",
|
||||
"file_drop_target_unavailable",
|
||||
"file-drop target is not the topmost element at its action point",
|
||||
"none",
|
||||
"resolve_target",
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
return failure(
|
||||
error instanceof BoundedWaitError ? "timeout" : "cdp_failed",
|
||||
"file_drop_target_unavailable",
|
||||
messageOf(error),
|
||||
"none",
|
||||
"resolve_target",
|
||||
);
|
||||
}
|
||||
|
||||
for (const [type, phase] of [
|
||||
["dragEnter", "drag_enter"],
|
||||
["dragOver", "drag_over"],
|
||||
] as const) {
|
||||
try {
|
||||
if (type === "dragEnter") dragStarted = true;
|
||||
await waitBounded(
|
||||
sendToCdpTarget(options.cdp, target, "Input.dispatchDragEvent", {
|
||||
type,
|
||||
x: point.x,
|
||||
y: point.y,
|
||||
data: dragData,
|
||||
}),
|
||||
deadline,
|
||||
options.signal,
|
||||
`${type} timed out`,
|
||||
);
|
||||
} catch (error) {
|
||||
const unsupported = methodUnavailable(error);
|
||||
if (type === "dragEnter" && unsupported) dragStarted = false;
|
||||
return failure(
|
||||
unsupported
|
||||
? "unsupported"
|
||||
: error instanceof BoundedWaitError
|
||||
? "timeout"
|
||||
: "cdp_failed",
|
||||
unsupported ? "upload_mechanism_unsupported" : "file_drop_failed",
|
||||
messageOf(error),
|
||||
"none",
|
||||
phase,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
dropSent = true;
|
||||
await waitBounded(
|
||||
sendToCdpTarget(options.cdp, target, "Input.dispatchDragEvent", {
|
||||
type: "drop",
|
||||
x: point.x,
|
||||
y: point.y,
|
||||
data: dragData,
|
||||
}),
|
||||
deadline,
|
||||
options.signal,
|
||||
"file drop timed out",
|
||||
);
|
||||
return { x: point.x, y: point.y };
|
||||
} catch (error) {
|
||||
outcome = failure(
|
||||
error instanceof BoundedWaitError ? "timeout" : "cdp_failed",
|
||||
"file_drop_failed",
|
||||
messageOf(error),
|
||||
"unknown",
|
||||
"drop",
|
||||
);
|
||||
return outcome;
|
||||
}
|
||||
} finally {
|
||||
if (dragStarted && !dropSent) {
|
||||
try {
|
||||
await waitBounded(
|
||||
sendToCdpTarget(options.cdp, target, "Input.dispatchDragEvent", {
|
||||
type: "dragCancel",
|
||||
x: dragPoint.x,
|
||||
y: dragPoint.y,
|
||||
data: dragData,
|
||||
}),
|
||||
Date.now() + CLEANUP_TIMEOUT_MS,
|
||||
undefined,
|
||||
"cancelling file drag timed out",
|
||||
);
|
||||
} catch {
|
||||
cleanupFailed = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
return outcome;
|
||||
} catch (error) {
|
||||
outcome = failure(
|
||||
error instanceof BoundedWaitError ? "timeout" : "cdp_failed",
|
||||
"file_drop_target_unavailable",
|
||||
messageOf(error),
|
||||
"none",
|
||||
"resolve_target",
|
||||
);
|
||||
return outcome;
|
||||
} finally {
|
||||
try {
|
||||
await waitBounded(
|
||||
sendToCdpTarget(options.cdp, target, "Runtime.releaseObjectGroup", { objectGroup }),
|
||||
Date.now() + CLEANUP_TIMEOUT_MS,
|
||||
undefined,
|
||||
"releasing file-drop object group timed out",
|
||||
);
|
||||
} catch {
|
||||
cleanupFailed = true;
|
||||
}
|
||||
|
||||
const effect = isRpcError(outcome)
|
||||
? (outcome.data?.effect_state as TransferEffectState | undefined)
|
||||
: "committed";
|
||||
if (effect === "unknown" || cleanupFailed) {
|
||||
await options.cdp.detach?.(target.tabId);
|
||||
}
|
||||
if (cleanupFailed && isRpcError(outcome)) {
|
||||
outcome.data = { ...outcome.data, cleanup_state: "failed" };
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,445 @@
|
||||
// One upload transaction with an explicit browser-side commit boundary.
|
||||
// Chrome interception prevents a native chooser from escaping automation;
|
||||
// chooser events and a frame-scoped DOM probe are independent input-location
|
||||
// signals, so event delivery is not required for standard file inputs.
|
||||
|
||||
import type { ClickResult, RpcError, TransferEffectState } from "@/transport/types";
|
||||
import { transferError } from "./errors";
|
||||
import type { ResolvedActionTarget } from "./interaction";
|
||||
import { type CdpRunner, isRpcError, sendToCdpTarget } from "./shared";
|
||||
import { BoundedWaitError, remainingMs, waitBounded } from "./transfer-transaction";
|
||||
|
||||
const CLEANUP_TIMEOUT_MS = 1_000;
|
||||
const ACTIVATION_GRACE_MS = 1_000;
|
||||
const PROBE_INTERVAL_MS = 20;
|
||||
|
||||
type UploadPhase =
|
||||
| "arm_interception"
|
||||
| "arm_input_probe"
|
||||
| "trigger"
|
||||
| "resolve_input"
|
||||
| "set_files"
|
||||
| "cleanup";
|
||||
|
||||
interface RuntimeReply {
|
||||
result?: { value?: unknown; objectId?: string };
|
||||
exceptionDetails?: { text?: string; exception?: { description?: string } };
|
||||
}
|
||||
|
||||
interface ChooserEvent {
|
||||
frameId?: string;
|
||||
backendNodeId?: number;
|
||||
mode?: "selectSingle" | "selectMultiple";
|
||||
}
|
||||
|
||||
export interface FileInputTransactionOptions {
|
||||
cdp: CdpRunner;
|
||||
actionTarget: ResolvedActionTarget;
|
||||
files: string[];
|
||||
timeoutMs: number;
|
||||
signal?: AbortSignal;
|
||||
trigger(): Promise<ClickResult | RpcError>;
|
||||
}
|
||||
|
||||
export interface FileInputTransactionResult {
|
||||
click: ClickResult;
|
||||
multiple: boolean;
|
||||
}
|
||||
|
||||
function runtimeError(reply: RuntimeReply, fallback: string): Error | null {
|
||||
if (!reply.exceptionDetails) return null;
|
||||
return new Error(
|
||||
reply.exceptionDetails.exception?.description ?? reply.exceptionDetails.text ?? fallback,
|
||||
);
|
||||
}
|
||||
|
||||
function transferFailure(
|
||||
code: RpcError["code"],
|
||||
reason: "file_input_probe_failed" | "file_input_not_activated" | "set_file_input_failed",
|
||||
message: string,
|
||||
effectState: TransferEffectState,
|
||||
phase: UploadPhase,
|
||||
): RpcError {
|
||||
return transferError(code, reason, message, { effectState, phase });
|
||||
}
|
||||
|
||||
function enrichFailure(
|
||||
error: RpcError,
|
||||
effectState: TransferEffectState,
|
||||
phase: UploadPhase,
|
||||
): RpcError {
|
||||
return {
|
||||
...error,
|
||||
data: { ...error.data, effect_state: effectState, phase },
|
||||
};
|
||||
}
|
||||
|
||||
function sameTarget(
|
||||
source: { tabId?: number; sessionId?: string },
|
||||
target: { tabId: number; sessionId?: string },
|
||||
): boolean {
|
||||
return source.tabId === target.tabId && source.sessionId === target.sessionId;
|
||||
}
|
||||
|
||||
async function delay(ms: number, signal?: AbortSignal): Promise<void> {
|
||||
await waitBounded(
|
||||
new Promise((resolve) => setTimeout(resolve, ms)),
|
||||
Date.now() + ms + 1,
|
||||
signal,
|
||||
"upload activation probe timed out",
|
||||
);
|
||||
}
|
||||
|
||||
async function callOnTrigger<T = RuntimeReply>(
|
||||
options: FileInputTransactionOptions,
|
||||
objectId: string,
|
||||
functionDeclaration: string,
|
||||
args: unknown[],
|
||||
returnByValue: boolean,
|
||||
): Promise<T> {
|
||||
return sendToCdpTarget<T>(options.cdp, options.actionTarget.cdpTarget, "Runtime.callFunctionOn", {
|
||||
objectId,
|
||||
functionDeclaration,
|
||||
arguments: args.map((value) => ({ value })),
|
||||
returnByValue,
|
||||
awaitPromise: false,
|
||||
});
|
||||
}
|
||||
|
||||
export async function uploadThroughActivatedFileInput(
|
||||
options: FileInputTransactionOptions,
|
||||
): Promise<FileInputTransactionResult | RpcError> {
|
||||
const deadline = Date.now() + options.timeoutMs;
|
||||
const target = options.actionTarget.cdpTarget;
|
||||
const objectGroup = `bsk-upload-${crypto.randomUUID()}`;
|
||||
const stateKey = `__bskUpload_${crypto.randomUUID().replaceAll("-", "")}`;
|
||||
const chooserEvents: ChooserEvent[] = [];
|
||||
let interceptionArmed = false;
|
||||
let probeArmed = false;
|
||||
let triggerObjectId: string | undefined;
|
||||
let outcome: FileInputTransactionResult | RpcError = transferFailure(
|
||||
"protocol_error",
|
||||
"file_input_probe_failed",
|
||||
"upload transaction ended without an outcome",
|
||||
"none",
|
||||
"cleanup",
|
||||
);
|
||||
|
||||
const chooserSubscription = options.cdp.onEvent?.((source, method, raw) => {
|
||||
if (method !== "Page.fileChooserOpened" || !sameTarget(source, target)) return;
|
||||
const event = raw as ChooserEvent;
|
||||
chooserEvents.push(event);
|
||||
});
|
||||
|
||||
try {
|
||||
try {
|
||||
await waitBounded(
|
||||
sendToCdpTarget(options.cdp, target, "Page.setInterceptFileChooserDialog", {
|
||||
enabled: true,
|
||||
}),
|
||||
deadline,
|
||||
options.signal,
|
||||
"arming native file chooser interception timed out",
|
||||
);
|
||||
interceptionArmed = true;
|
||||
} catch (err) {
|
||||
outcome = transferFailure(
|
||||
err instanceof BoundedWaitError ? "timeout" : "cdp_failed",
|
||||
"file_input_probe_failed",
|
||||
err instanceof Error ? err.message : String(err),
|
||||
"none",
|
||||
"arm_interception",
|
||||
);
|
||||
return outcome;
|
||||
}
|
||||
|
||||
try {
|
||||
const resolved = await waitBounded(
|
||||
sendToCdpTarget<{ object?: { objectId?: string } }>(
|
||||
options.cdp,
|
||||
target,
|
||||
"DOM.resolveNode",
|
||||
{
|
||||
backendNodeId: options.actionTarget.backendNodeId,
|
||||
objectGroup,
|
||||
},
|
||||
),
|
||||
deadline,
|
||||
options.signal,
|
||||
"resolving upload trigger timed out",
|
||||
);
|
||||
triggerObjectId = resolved.object?.objectId;
|
||||
if (!triggerObjectId) throw new Error("DOM.resolveNode returned no trigger objectId");
|
||||
|
||||
const armed = await waitBounded(
|
||||
callOnTrigger<RuntimeReply>(
|
||||
options,
|
||||
triggerObjectId,
|
||||
`function(key) {
|
||||
const doc = this.ownerDocument;
|
||||
const owner = doc.defaultView;
|
||||
if (!owner) return false;
|
||||
const state = { inputs: [], listener: null };
|
||||
Object.defineProperty(owner, key, { value: state, configurable: true });
|
||||
state.listener = event => {
|
||||
const path = typeof event.composedPath === "function" ? event.composedPath() : [];
|
||||
const candidate = path[0] || event.target;
|
||||
if (candidate && candidate.nodeType === 1 &&
|
||||
candidate.localName === "input" && candidate.type === "file") {
|
||||
if (!state.inputs.includes(candidate)) state.inputs.push(candidate);
|
||||
event.preventDefault();
|
||||
}
|
||||
};
|
||||
doc.addEventListener("click", state.listener, true);
|
||||
return true;
|
||||
}`,
|
||||
[stateKey],
|
||||
true,
|
||||
),
|
||||
deadline,
|
||||
options.signal,
|
||||
"arming frame-scoped file input probe timed out",
|
||||
);
|
||||
const armError = runtimeError(armed, "failed to arm file input probe");
|
||||
if (armError || armed.result?.value !== true) {
|
||||
throw armError ?? new Error("file input probe did not arm");
|
||||
}
|
||||
probeArmed = true;
|
||||
} catch (err) {
|
||||
outcome = transferFailure(
|
||||
err instanceof BoundedWaitError ? "timeout" : "cdp_failed",
|
||||
"file_input_probe_failed",
|
||||
err instanceof Error ? err.message : String(err),
|
||||
"none",
|
||||
"arm_input_probe",
|
||||
);
|
||||
return outcome;
|
||||
}
|
||||
|
||||
let click: ClickResult | RpcError;
|
||||
try {
|
||||
click = await waitBounded(
|
||||
options.trigger(),
|
||||
deadline,
|
||||
options.signal,
|
||||
"upload trigger timed out",
|
||||
);
|
||||
} catch (err) {
|
||||
outcome = transferFailure(
|
||||
err instanceof BoundedWaitError ? "timeout" : "cdp_failed",
|
||||
"file_input_probe_failed",
|
||||
err instanceof Error ? err.message : String(err),
|
||||
"none",
|
||||
"trigger",
|
||||
);
|
||||
return outcome;
|
||||
}
|
||||
if (isRpcError(click)) {
|
||||
outcome = enrichFailure(click, "none", "trigger");
|
||||
return outcome;
|
||||
}
|
||||
|
||||
try {
|
||||
const activationDeadline = Math.min(deadline, Date.now() + ACTIVATION_GRACE_MS);
|
||||
let summary: { count: number; multiple: boolean } = { count: 0, multiple: false };
|
||||
while (Date.now() < activationDeadline) {
|
||||
if (chooserEvents.length > 0) break;
|
||||
const reply = await callOnTrigger<RuntimeReply>(
|
||||
options,
|
||||
triggerObjectId,
|
||||
`function(key) {
|
||||
const state = this.ownerDocument.defaultView?.[key];
|
||||
return state
|
||||
? { count: state.inputs.length, multiple: state.inputs[0]?.multiple === true }
|
||||
: { count: 0, multiple: false };
|
||||
}`,
|
||||
[stateKey],
|
||||
true,
|
||||
);
|
||||
const summaryError = runtimeError(reply, "failed to inspect activated file input");
|
||||
if (summaryError) throw summaryError;
|
||||
const value = reply.result?.value as { count?: unknown; multiple?: unknown } | undefined;
|
||||
summary = {
|
||||
count: typeof value?.count === "number" ? value.count : 0,
|
||||
multiple: value?.multiple === true,
|
||||
};
|
||||
if (summary.count > 0) break;
|
||||
await delay(Math.min(PROBE_INTERVAL_MS, remainingMs(activationDeadline)), options.signal);
|
||||
}
|
||||
|
||||
if (chooserEvents.length > 1) {
|
||||
outcome = transferFailure(
|
||||
"unsupported",
|
||||
"file_input_not_activated",
|
||||
"upload trigger activated more than one file chooser",
|
||||
"none",
|
||||
"resolve_input",
|
||||
);
|
||||
return outcome;
|
||||
}
|
||||
|
||||
const chooser = chooserEvents[0];
|
||||
let backendNodeId: number | undefined;
|
||||
let multiple = summary.multiple;
|
||||
if (chooser) {
|
||||
if (options.actionTarget.frameId && chooser.frameId !== options.actionTarget.frameId) {
|
||||
outcome = transferFailure(
|
||||
"unsupported",
|
||||
"file_input_not_activated",
|
||||
"upload trigger activated a file chooser in a different frame",
|
||||
"none",
|
||||
"resolve_input",
|
||||
);
|
||||
return outcome;
|
||||
}
|
||||
if (typeof chooser.backendNodeId !== "number") {
|
||||
outcome = transferFailure(
|
||||
"unsupported",
|
||||
"file_input_not_activated",
|
||||
"upload trigger invoked a non-input file picker",
|
||||
"none",
|
||||
"resolve_input",
|
||||
);
|
||||
return outcome;
|
||||
}
|
||||
backendNodeId = chooser.backendNodeId;
|
||||
multiple = chooser.mode === "selectMultiple";
|
||||
} else {
|
||||
if (summary.count !== 1) {
|
||||
outcome = transferFailure(
|
||||
"unsupported",
|
||||
"file_input_not_activated",
|
||||
summary.count === 0
|
||||
? "upload trigger did not activate an input[type=file]"
|
||||
: "upload trigger activated more than one input[type=file]",
|
||||
"none",
|
||||
"resolve_input",
|
||||
);
|
||||
return outcome;
|
||||
}
|
||||
const input = await callOnTrigger<RuntimeReply>(
|
||||
options,
|
||||
triggerObjectId,
|
||||
`function(key) { return this.ownerDocument.defaultView?.[key]?.inputs[0]; }`,
|
||||
[stateKey],
|
||||
false,
|
||||
);
|
||||
const inputError = runtimeError(input, "failed to resolve activated file input object");
|
||||
if (inputError) throw inputError;
|
||||
if (!input.result?.objectId) throw new Error("activated file input returned no objectId");
|
||||
const described = await sendToCdpTarget<{ node?: { backendNodeId?: number } }>(
|
||||
options.cdp,
|
||||
target,
|
||||
"DOM.describeNode",
|
||||
{ objectId: input.result.objectId },
|
||||
);
|
||||
backendNodeId = described.node?.backendNodeId;
|
||||
if (typeof backendNodeId !== "number") {
|
||||
throw new Error("DOM.describeNode returned no file input backendNodeId");
|
||||
}
|
||||
}
|
||||
|
||||
if (!multiple && options.files.length !== 1) {
|
||||
outcome = enrichFailure(
|
||||
{ code: "invalid_params", message: "file input accepts exactly one file" },
|
||||
"none",
|
||||
"resolve_input",
|
||||
);
|
||||
return outcome;
|
||||
}
|
||||
|
||||
try {
|
||||
await waitBounded(
|
||||
sendToCdpTarget(options.cdp, target, "DOM.setFileInputFiles", {
|
||||
files: options.files,
|
||||
backendNodeId,
|
||||
}),
|
||||
deadline,
|
||||
options.signal,
|
||||
"setting file input files timed out",
|
||||
);
|
||||
} catch (err) {
|
||||
outcome = transferFailure(
|
||||
err instanceof BoundedWaitError ? "timeout" : "cdp_failed",
|
||||
"set_file_input_failed",
|
||||
err instanceof Error ? err.message : String(err),
|
||||
"unknown",
|
||||
"set_files",
|
||||
);
|
||||
return outcome;
|
||||
}
|
||||
outcome = { click, multiple };
|
||||
return outcome;
|
||||
} catch (err) {
|
||||
outcome = transferFailure(
|
||||
err instanceof BoundedWaitError ? "timeout" : "cdp_failed",
|
||||
"file_input_probe_failed",
|
||||
err instanceof Error ? err.message : String(err),
|
||||
"none",
|
||||
"resolve_input",
|
||||
);
|
||||
return outcome;
|
||||
}
|
||||
} finally {
|
||||
chooserSubscription?.dispose();
|
||||
let cleanupFailed = false;
|
||||
if (probeArmed && triggerObjectId) {
|
||||
try {
|
||||
await waitBounded(
|
||||
callOnTrigger(
|
||||
options,
|
||||
triggerObjectId,
|
||||
`function(key) {
|
||||
const owner = this.ownerDocument.defaultView;
|
||||
const state = owner?.[key];
|
||||
if (state?.listener) this.ownerDocument.removeEventListener("click", state.listener, true);
|
||||
if (owner) delete owner[key];
|
||||
}`,
|
||||
[stateKey],
|
||||
true,
|
||||
),
|
||||
Date.now() + CLEANUP_TIMEOUT_MS,
|
||||
undefined,
|
||||
"cleaning file input probe timed out",
|
||||
);
|
||||
} catch {
|
||||
cleanupFailed = true;
|
||||
}
|
||||
}
|
||||
if (interceptionArmed) {
|
||||
try {
|
||||
await waitBounded(
|
||||
sendToCdpTarget(options.cdp, target, "Page.setInterceptFileChooserDialog", {
|
||||
enabled: false,
|
||||
cancel: true,
|
||||
}),
|
||||
Date.now() + CLEANUP_TIMEOUT_MS,
|
||||
undefined,
|
||||
"disabling file chooser interception timed out",
|
||||
);
|
||||
} catch {
|
||||
cleanupFailed = true;
|
||||
}
|
||||
}
|
||||
try {
|
||||
await waitBounded(
|
||||
sendToCdpTarget(options.cdp, target, "Runtime.releaseObjectGroup", { objectGroup }),
|
||||
Date.now() + CLEANUP_TIMEOUT_MS,
|
||||
undefined,
|
||||
"releasing upload object group timed out",
|
||||
);
|
||||
} catch {
|
||||
cleanupFailed = true;
|
||||
}
|
||||
|
||||
const effect = isRpcError(outcome)
|
||||
? (outcome.data?.effect_state as TransferEffectState | undefined)
|
||||
: "committed";
|
||||
if (effect === "unknown" || cleanupFailed) {
|
||||
await options.cdp.detach?.(target.tabId);
|
||||
}
|
||||
if (cleanupFailed && isRpcError(outcome)) {
|
||||
outcome.data = { ...outcome.data, cleanup_state: "failed" };
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -29,7 +29,10 @@ export interface NodeAddress {
|
||||
export interface ResolvedNodeGeometry {
|
||||
topVisibleRegions: Region;
|
||||
topBounds: ViewportRect;
|
||||
/** Point in the top-level tab viewport, used by root-target input events. */
|
||||
actionPoint: Point;
|
||||
/** Point in the addressed CDP target's viewport, used by OOPIF-local input events. */
|
||||
targetActionPoint: Point;
|
||||
}
|
||||
|
||||
function geometryError(message: string): RpcError {
|
||||
@@ -262,11 +265,30 @@ export async function resolveNodeGeometry(
|
||||
if (isRpcError(localRegion)) return localRegion;
|
||||
|
||||
let topVisibleRegions: Region;
|
||||
let targetActionPoint: Point | null = null;
|
||||
if (address.frameId && graph) {
|
||||
const projection = await resolveFrameProjection(cdp, graph, address.frameId);
|
||||
if (!projection)
|
||||
return geometryError(`could not resolve frame geometry for ${address.frameId}`);
|
||||
topVisibleRegions = projectRegionToViewport(localRegion, projection);
|
||||
if (address.target.sessionId) {
|
||||
const localViewport = projection.edges[0]?.sourceViewport ?? projection.topViewport;
|
||||
const localVisibleRegions = localRegion
|
||||
.map((polygon) =>
|
||||
clipPolygon(
|
||||
polygon,
|
||||
rectPolygon({
|
||||
x: 0,
|
||||
y: 0,
|
||||
w: localViewport.width,
|
||||
h: localViewport.height,
|
||||
}),
|
||||
),
|
||||
)
|
||||
.filter((polygon) => polygon.length >= 3);
|
||||
const localActionRegion = largestRegion(localVisibleRegions);
|
||||
targetActionPoint = localActionRegion ? polygonCentroid(localActionRegion) : null;
|
||||
}
|
||||
} else {
|
||||
const viewport = await targetViewport(cdp, address.target);
|
||||
if (!viewport) return geometryError("could not resolve top viewport geometry");
|
||||
@@ -283,7 +305,11 @@ export async function resolveNodeGeometry(
|
||||
if (!topBounds || !actionPoint) {
|
||||
return { code: "permission_denied", message: "element not visible" };
|
||||
}
|
||||
return { topVisibleRegions, topBounds, actionPoint };
|
||||
if (address.target.sessionId && !targetActionPoint) {
|
||||
return { code: "permission_denied", message: "element not visible in its target" };
|
||||
}
|
||||
targetActionPoint ??= actionPoint;
|
||||
return { topVisibleRegions, topBounds, actionPoint, targetActionPoint };
|
||||
} catch (error) {
|
||||
return geometryError(error instanceof Error ? error.message : String(error));
|
||||
}
|
||||
|
||||
@@ -40,6 +40,7 @@ import {
|
||||
enforceAgentWindow,
|
||||
isRpcError,
|
||||
lookupSession,
|
||||
type ResolvedTargetTab,
|
||||
resolveTargetTab,
|
||||
} from "./shared";
|
||||
import { resolveSnapshotRef } from "./snapshot-ref";
|
||||
@@ -56,6 +57,15 @@ export interface InteractionDeps {
|
||||
keepOverlayBypassAfterHover?: boolean;
|
||||
}
|
||||
|
||||
export interface ResolvedActionTarget {
|
||||
tab: ResolvedTargetTab;
|
||||
backendNodeId: number;
|
||||
cdpTarget: CdpTarget;
|
||||
frameId?: string;
|
||||
usedRef?: string;
|
||||
usedSelector?: string;
|
||||
}
|
||||
|
||||
const DEFAULT_TIMEOUT_MS = 30_000;
|
||||
const DEFAULT_HOVER_SETTLE_MS = 200;
|
||||
|
||||
@@ -106,14 +116,6 @@ function isAbortLikeError(err: unknown): boolean {
|
||||
return err instanceof DOMException && err.name === "AbortError";
|
||||
}
|
||||
|
||||
function isInvalidCssSelectorError(err: unknown): boolean {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
return (
|
||||
message.includes("DOM Error while querying") ||
|
||||
/invalid selector|not a valid selector|querySelector.*failed/i.test(message)
|
||||
);
|
||||
}
|
||||
|
||||
async function wait(ms: number, signal?: AbortSignal): Promise<void> {
|
||||
if (signal?.aborted) throw new DOMException("aborted", "AbortError");
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
@@ -136,7 +138,7 @@ async function wait(ms: number, signal?: AbortSignal): Promise<void> {
|
||||
* `RpcError` if the caller supplied neither (or both), or if neither
|
||||
* lookup matched.
|
||||
*/
|
||||
async function resolveBackendNode(
|
||||
export async function resolveBackendNode(
|
||||
cdp: CdpRunner,
|
||||
ctx: SessionContext,
|
||||
target: { tabId: number },
|
||||
@@ -181,6 +183,8 @@ async function resolveBackendNode(
|
||||
}
|
||||
// selector path
|
||||
try {
|
||||
// Selector lookup itself attaches CDP, even when no element is found.
|
||||
cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
const doc = await cdp.send<{ root?: { nodeId?: number } }>(target.tabId, "DOM.getDocument", {
|
||||
depth: 0,
|
||||
});
|
||||
@@ -216,14 +220,6 @@ async function resolveBackendNode(
|
||||
}
|
||||
return { backendNodeId, cdpTarget: { tabId: target.tabId }, usedSelector: params.selector };
|
||||
} catch (err) {
|
||||
if (isInvalidCssSelectorError(err)) {
|
||||
return rpcError(
|
||||
"invalid_params",
|
||||
"invalid_selector",
|
||||
`selector is not valid CSS: ${params.selector}`,
|
||||
{ selector: params.selector },
|
||||
);
|
||||
}
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
message: err instanceof Error ? err.message : String(err),
|
||||
@@ -231,6 +227,17 @@ async function resolveBackendNode(
|
||||
}
|
||||
}
|
||||
|
||||
export async function resolveActionTarget(
|
||||
cdp: CdpRunner,
|
||||
ctx: SessionContext,
|
||||
target: ResolvedTargetTab,
|
||||
params: { ref?: string; selector?: string },
|
||||
toolName: string,
|
||||
): Promise<ResolvedActionTarget | RpcError> {
|
||||
const node = await resolveBackendNode(cdp, ctx, target, params, toolName);
|
||||
return isRpcError(node) ? node : { tab: target, ...node };
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// tool.click
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -249,10 +256,19 @@ export async function handleClick(
|
||||
if (isRpcError(target)) return target;
|
||||
const denied = enforceAgentWindow(ctx, target, "click");
|
||||
if (denied) return denied;
|
||||
const dialogCursor = markDialogCursor(deps.cdp, target.tabId);
|
||||
const resolved = await resolveActionTarget(deps.cdp, ctx, target, params, "click");
|
||||
if (isRpcError(resolved)) return resolved;
|
||||
return clickResolvedTarget(ctx, resolved, params, deps);
|
||||
}
|
||||
|
||||
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "click");
|
||||
if (isRpcError(node)) return node;
|
||||
export async function clickResolvedTarget(
|
||||
ctx: SessionContext,
|
||||
resolved: ResolvedActionTarget,
|
||||
params: Pick<ClickParams, "button" | "click_count" | "modifiers">,
|
||||
deps: InteractionDeps,
|
||||
): Promise<ClickResult | RpcError> {
|
||||
const { tab: target } = resolved;
|
||||
const dialogCursor = markDialogCursor(deps.cdp, target.tabId);
|
||||
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "click aborted" };
|
||||
@@ -263,9 +279,9 @@ export async function handleClick(
|
||||
deps.cdp,
|
||||
target.tabId,
|
||||
{
|
||||
target: node.cdpTarget,
|
||||
backendNodeId: node.backendNodeId,
|
||||
...(node.frameId ? { frameId: node.frameId } : {}),
|
||||
target: resolved.cdpTarget,
|
||||
backendNodeId: resolved.backendNodeId,
|
||||
...(resolved.frameId ? { frameId: resolved.frameId } : {}),
|
||||
},
|
||||
{ scrollIntoView: true },
|
||||
);
|
||||
@@ -353,8 +369,8 @@ export async function handleClick(
|
||||
|
||||
return attachDialogs(deps.cdp, target.tabId, dialogCursor, {
|
||||
tab_id: target.tabId,
|
||||
used_ref: node.usedRef,
|
||||
used_selector: node.usedSelector,
|
||||
used_ref: resolved.usedRef,
|
||||
used_selector: resolved.usedSelector,
|
||||
x: centre.x,
|
||||
y: centre.y,
|
||||
});
|
||||
@@ -593,6 +609,53 @@ type SelectMutationResult =
|
||||
// tool.fill
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type FillFailureReason =
|
||||
| "target_not_fillable"
|
||||
| "fill_value_invalid"
|
||||
| "fill_target_changed"
|
||||
| "fill_focus_lost"
|
||||
| "fill_value_mismatch"
|
||||
| "fill_failed";
|
||||
|
||||
type FillPreparation =
|
||||
| { before: string; expected: string }
|
||||
| { reason: FillFailureReason; error: string };
|
||||
|
||||
type FillReadiness = "ready" | "background" | "fill_target_changed" | "fill_focus_lost";
|
||||
|
||||
function fillError(reason: FillFailureReason, message: string): RpcError {
|
||||
return rpcError(
|
||||
reason === "target_not_fillable" || reason === "fill_value_invalid"
|
||||
? "invalid_params"
|
||||
: "cdp_failed",
|
||||
reason,
|
||||
message,
|
||||
);
|
||||
}
|
||||
|
||||
interface FillScriptReply<T> {
|
||||
result?: { value?: T };
|
||||
exceptionDetails?: unknown;
|
||||
}
|
||||
|
||||
const FILL_EDITABLE_FUNCTION = `function() {
|
||||
const tag = this.tagName.toLowerCase();
|
||||
const supported = tag === 'input'
|
||||
? ['text', 'search', 'tel', 'url', 'email', 'password', 'number'].includes(this.type)
|
||||
: tag === 'textarea' || this.isContentEditable;
|
||||
return this.isConnected && supported && !this.readOnly && !this.matches(':disabled');
|
||||
}`;
|
||||
|
||||
// Chrome renders a trailing editable newline with an empty <div><br></div>.
|
||||
// innerText includes the padding break; it is not an extra typed character.
|
||||
const FILL_VALUE_FUNCTION = `function() {
|
||||
if (!this.isContentEditable) return this.value;
|
||||
const value = this.innerText;
|
||||
const tail = this.lastChild;
|
||||
const padding = tail && tail.nodeName === 'DIV' && tail.childNodes.length === 1 && tail.firstChild.nodeName === 'BR';
|
||||
return padding && value.endsWith('\\n\\n') ? value.slice(0, -1) : value;
|
||||
}`;
|
||||
|
||||
export async function handleFill(
|
||||
manager: SessionManager,
|
||||
params: FillParams,
|
||||
@@ -644,12 +707,8 @@ export async function handleFill(
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
await nodeCdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
|
||||
} catch (err) {
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
message: err instanceof Error ? err.message : String(err),
|
||||
};
|
||||
return fillError("fill_failed", err instanceof Error ? err.message : String(err));
|
||||
}
|
||||
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
@@ -662,57 +721,252 @@ export async function handleFill(
|
||||
const clearBefore = params.clear_before ?? true;
|
||||
|
||||
try {
|
||||
if (clearBefore) {
|
||||
// Clear input/textarea value or wipe contenteditable innerText,
|
||||
// then fire `input` so frameworks observe the empty state.
|
||||
await nodeCdp.send(target.tabId, "Runtime.callFunctionOn", {
|
||||
objectId,
|
||||
functionDeclaration: `function() {
|
||||
if (this.isContentEditable) { this.textContent = ''; }
|
||||
else {
|
||||
const proto = this instanceof HTMLTextAreaElement
|
||||
? HTMLTextAreaElement.prototype
|
||||
: HTMLInputElement.prototype;
|
||||
const descriptor = Object.getOwnPropertyDescriptor(proto, 'value');
|
||||
if (descriptor && descriptor.set) descriptor.set.call(this, '');
|
||||
else this.value = '';
|
||||
}
|
||||
this.dispatchEvent(new Event('input', { bubbles: true }));
|
||||
}`,
|
||||
returnByValue: true,
|
||||
});
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
try {
|
||||
await nodeCdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
|
||||
} catch (error) {
|
||||
// Chrome rejects DOM.focus for disabled controls, including inherited
|
||||
// fieldset state. Classify the live target before treating this as a
|
||||
// browser failure; the normal path needs no extra round trip.
|
||||
const editable = await nodeCdp.send<FillScriptReply<boolean>>(
|
||||
target.tabId,
|
||||
"Runtime.callFunctionOn",
|
||||
{ objectId, functionDeclaration: FILL_EDITABLE_FUNCTION, returnByValue: true },
|
||||
);
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
if (!editable.exceptionDetails && editable.result?.value === false) {
|
||||
return fillError(
|
||||
"target_not_fillable",
|
||||
"fill target is not editable or its input type is unsupported",
|
||||
);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
// Check editability before clearing. Keep the expected result tied to
|
||||
// this object, including the existing value on the append path.
|
||||
const prepared = await nodeCdp.send<FillScriptReply<FillPreparation>>(
|
||||
target.tabId,
|
||||
"Runtime.callFunctionOn",
|
||||
{
|
||||
objectId,
|
||||
functionDeclaration: `function(value, clearBefore) {
|
||||
const tag = this.tagName.toLowerCase();
|
||||
const native = tag === 'input' || tag === 'textarea';
|
||||
if (!(${FILL_EDITABLE_FUNCTION}).call(this)) return { reason: 'target_not_fillable', error: 'fill target is not editable or its input type is unsupported' };
|
||||
if (this.getRootNode().activeElement !== this) return { reason: 'fill_focus_lost', error: 'fill target does not have focus' };
|
||||
const before = clearBefore ? '' : (${FILL_VALUE_FUNCTION}).call(this);
|
||||
let expected = before + value;
|
||||
if (native) {
|
||||
// Use the browser's own value sanitization (e.g. textarea
|
||||
// line endings) without changing the live control.
|
||||
const normalizer = this.ownerDocument.createElement(tag);
|
||||
if (tag === 'input') {
|
||||
normalizer.type = this.type;
|
||||
normalizer.multiple = this.multiple;
|
||||
}
|
||||
// insertText treats line breaks in a single-line input as spaces.
|
||||
if (tag === 'input') expected = expected.replace(/\\r\\n|\\r|\\n/g, ' ');
|
||||
normalizer.value = expected;
|
||||
if (expected !== '' && normalizer.value === '' && this.type === 'number') {
|
||||
return { reason: 'fill_value_invalid', error: 'fill value is not valid for a number input' };
|
||||
}
|
||||
expected = normalizer.value;
|
||||
if (this.type !== 'number' && this.maxLength >= 0 && expected.length > this.maxLength) {
|
||||
return { reason: 'fill_value_invalid', error: 'fill value exceeds the target maxlength' };
|
||||
}
|
||||
} else {
|
||||
expected = expected.replace(/\\r\\n?/g, '\\n');
|
||||
}
|
||||
if (clearBefore) {
|
||||
if (native) {
|
||||
const proto = tag === 'textarea' ? HTMLTextAreaElement.prototype : HTMLInputElement.prototype;
|
||||
Object.getOwnPropertyDescriptor(proto, 'value').set.call(this, '');
|
||||
} else {
|
||||
this.textContent = '';
|
||||
}
|
||||
this.dispatchEvent(new Event('input', { bubbles: true }));
|
||||
}
|
||||
return { before, expected };
|
||||
}`,
|
||||
arguments: [{ value: params.value }, { value: clearBefore }],
|
||||
returnByValue: true,
|
||||
},
|
||||
);
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
if (prepared.exceptionDetails) {
|
||||
return fillError("fill_failed", "fill preparation script failed");
|
||||
}
|
||||
const preparation = prepared.result?.value;
|
||||
if (preparation && "error" in preparation) {
|
||||
return fillError(preparation.reason, preparation.error);
|
||||
}
|
||||
if (typeof preparation?.before !== "string" || typeof preparation?.expected !== "string") {
|
||||
return fillError("fill_failed", "fill preparation returned an unexpected result");
|
||||
}
|
||||
// A separate call runs after the clearing event's microtasks drain.
|
||||
// DOM focus works in background tabs. Use document.hasFocus() only to
|
||||
// detect deferred focus events, never to reject background input.
|
||||
const checkReady = async (): Promise<FillReadiness | RpcError> => {
|
||||
const reply = await nodeCdp.send<FillScriptReply<FillReadiness>>(
|
||||
target.tabId,
|
||||
"Runtime.callFunctionOn",
|
||||
{
|
||||
objectId,
|
||||
functionDeclaration: `function(before) {
|
||||
if (!(${FILL_EDITABLE_FUNCTION}).call(this) ||
|
||||
(${FILL_VALUE_FUNCTION}).call(this) !== before) return 'fill_target_changed';
|
||||
if (this.getRootNode().activeElement !== this) return 'fill_focus_lost';
|
||||
return this.ownerDocument.hasFocus() ? 'ready' : 'background';
|
||||
}`,
|
||||
arguments: [{ value: preparation.before }],
|
||||
returnByValue: true,
|
||||
},
|
||||
);
|
||||
const state = reply.result?.value;
|
||||
if (
|
||||
reply.exceptionDetails ||
|
||||
(state !== "ready" &&
|
||||
state !== "background" &&
|
||||
state !== "fill_target_changed" &&
|
||||
state !== "fill_focus_lost")
|
||||
) {
|
||||
return fillError(
|
||||
"fill_failed",
|
||||
"fill readiness script failed or returned an unexpected result",
|
||||
);
|
||||
}
|
||||
return state;
|
||||
};
|
||||
let ready = await checkReady();
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
// Restore focus once only if clearing left this same target editable
|
||||
// and unchanged. Recheck after focus handlers have run.
|
||||
if (clearBefore && ready === "fill_focus_lost") {
|
||||
await nodeCdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
ready = await checkReady();
|
||||
}
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
if (typeof ready !== "string") return ready;
|
||||
if (ready !== "ready" && ready !== "background") {
|
||||
return fillError(ready, "fill target changed or lost focus before typing");
|
||||
}
|
||||
if (params.value !== "" && (!clearBefore || ready === "background")) {
|
||||
// Native editing commands also support number/email inputs, whose
|
||||
// selection APIs cannot set a caret, and multiline contenteditables.
|
||||
// Also do this for background replacement: CDP input focuses the
|
||||
// renderer, delivering deferred focus events before it inserts text.
|
||||
// Use a command-only event so this does not invoke an End shortcut.
|
||||
try {
|
||||
await deps.cdp.send(target.tabId, "Input.dispatchKeyEvent", {
|
||||
type: "rawKeyDown",
|
||||
commands: ["moveToEndOfDocument"],
|
||||
});
|
||||
} finally {
|
||||
await deps.cdp.send(target.tabId, "Input.dispatchKeyEvent", { type: "keyUp" });
|
||||
}
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
// Focus and key handlers can change the target or move focus too.
|
||||
ready = await checkReady();
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
if (typeof ready !== "string") return ready;
|
||||
if (ready !== "ready" && ready !== "background") {
|
||||
return fillError(ready, "fill target changed or lost focus while positioning the caret");
|
||||
}
|
||||
}
|
||||
// CDP `Input.insertText` handles IME / multi-byte input out of the
|
||||
// box, much more reliably than per-key `dispatchKeyEvent`.
|
||||
await deps.cdp.send(target.tabId, "Input.insertText", { text: params.value });
|
||||
if (params.value !== "") {
|
||||
await deps.cdp.send(target.tabId, "Input.insertText", { text: params.value });
|
||||
}
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
// Fire `input` + `change` so React / Vue controlled inputs commit.
|
||||
await nodeCdp.send(target.tabId, "Runtime.callFunctionOn", {
|
||||
// Keep the existing notifications, then read in a separate call so
|
||||
// nested microtasks queued by the page cannot race the verification.
|
||||
const notified = await nodeCdp.send<FillScriptReply<unknown>>(
|
||||
target.tabId,
|
||||
"Runtime.callFunctionOn",
|
||||
{
|
||||
objectId,
|
||||
functionDeclaration: `function() {
|
||||
this.dispatchEvent(new Event('input', { bubbles: true }));
|
||||
this.dispatchEvent(new Event('change', { bubbles: true }));
|
||||
}`,
|
||||
returnByValue: true,
|
||||
},
|
||||
);
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
if (notified.exceptionDetails) {
|
||||
return fillError("fill_failed", "fill notification script failed");
|
||||
}
|
||||
const verified = await nodeCdp.send<
|
||||
FillScriptReply<{ connected: boolean; matches: boolean; valueLength: number }>
|
||||
>(target.tabId, "Runtime.callFunctionOn", {
|
||||
objectId,
|
||||
functionDeclaration: `function() {
|
||||
this.dispatchEvent(new Event('input', { bubbles: true }));
|
||||
this.dispatchEvent(new Event('change', { bubbles: true }));
|
||||
}`,
|
||||
functionDeclaration: `function(expected) {
|
||||
const value = (${FILL_VALUE_FUNCTION}).call(this);
|
||||
return { connected: this.isConnected, matches: value === expected, valueLength: value.length };
|
||||
}`,
|
||||
arguments: [{ value: preparation.expected }],
|
||||
returnByValue: true,
|
||||
});
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
const verification = verified.result?.value;
|
||||
if (
|
||||
verified.exceptionDetails ||
|
||||
typeof verification?.connected !== "boolean" ||
|
||||
typeof verification?.matches !== "boolean" ||
|
||||
typeof verification?.valueLength !== "number"
|
||||
) {
|
||||
return fillError(
|
||||
"fill_failed",
|
||||
"fill verification script failed or returned an unexpected result",
|
||||
);
|
||||
}
|
||||
if (!verification.connected) {
|
||||
return fillError("fill_target_changed", "fill target was removed or replaced during input");
|
||||
}
|
||||
if (!verification.matches) {
|
||||
return fillError(
|
||||
"fill_value_mismatch",
|
||||
"fill could not verify the expected value; observe the page before deciding whether to retry",
|
||||
);
|
||||
}
|
||||
return attachDialogs(deps.cdp, target.tabId, dialogCursor, {
|
||||
tab_id: target.tabId,
|
||||
used_ref: node.usedRef,
|
||||
used_selector: node.usedSelector,
|
||||
value_length: verification.valueLength,
|
||||
});
|
||||
} catch (err) {
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
message: err instanceof Error ? err.message : String(err),
|
||||
};
|
||||
return fillError("fill_failed", err instanceof Error ? err.message : String(err));
|
||||
} finally {
|
||||
await nodeCdp.send(target.tabId, "Runtime.releaseObject", { objectId }).catch(() => undefined);
|
||||
}
|
||||
|
||||
return attachDialogs(deps.cdp, target.tabId, dialogCursor, {
|
||||
tab_id: target.tabId,
|
||||
used_ref: node.usedRef,
|
||||
used_selector: node.usedSelector,
|
||||
value_length: params.value.length,
|
||||
});
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
@@ -28,7 +28,13 @@ import type {
|
||||
RpcError,
|
||||
WaitUntil,
|
||||
} from "@/transport/types";
|
||||
import {
|
||||
type BrowserNavigationApi,
|
||||
chromeBrowserNavigationApi,
|
||||
navigateWithBrowserApi,
|
||||
} from "./browser-navigation";
|
||||
import { attachDialogs, markDialogCursor } from "./dialogs";
|
||||
import { cdpError, isCdpExtensionAccessDenied } from "./errors";
|
||||
import {
|
||||
type CdpRunner,
|
||||
type ChromeTabsApi,
|
||||
@@ -42,6 +48,7 @@ import {
|
||||
export interface NavigationDeps {
|
||||
cdp: CdpRunner;
|
||||
tabsApi: ChromeTabsApi;
|
||||
browserNavigation?: BrowserNavigationApi;
|
||||
/** Optional AbortSignal — M7 abort hook (M10.2 will wire the full chain). */
|
||||
signal?: AbortSignal;
|
||||
/** Override default timeout when the caller omits `timeout_ms`. */
|
||||
@@ -479,6 +486,81 @@ async function readTabUrl(api: ChromeTabsApi, tabId: number): Promise<string | u
|
||||
}
|
||||
}
|
||||
|
||||
/** Recover only a failed preflight; never replay an already-dispatched navigation. */
|
||||
async function recoverBrowserNavigation(
|
||||
deps: NavigationDeps,
|
||||
tabId: number,
|
||||
action: (api: BrowserNavigationApi) => Promise<unknown>,
|
||||
waitUntil: WaitUntil,
|
||||
timeoutMs: number,
|
||||
): Promise<ReloadResult | RpcError> {
|
||||
const abort = linkedAbortSignal(deps.signal);
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
const api = deps.browserNavigation ?? chromeBrowserNavigationApi;
|
||||
try {
|
||||
let outcome = await navigateWithBrowserApi(
|
||||
api,
|
||||
tabId,
|
||||
() => action(api),
|
||||
waitUntil === "networkidle" ? "commit" : waitUntil,
|
||||
timeoutMs,
|
||||
abort.signal,
|
||||
);
|
||||
if (outcome.reached === "failed") return outcome.error;
|
||||
if (outcome.reached === "cancelled" || abort.signal.aborted) {
|
||||
return { code: "cancelled", message: "navigation recovery aborted" };
|
||||
}
|
||||
if (outcome.reached === "match") {
|
||||
// A completed reload is not a recovery if the restricted frame remains.
|
||||
await deps.cdp.send(tabId, "Page.enable", {});
|
||||
if (abort.signal.aborted)
|
||||
return { code: "cancelled", message: "navigation recovery aborted" };
|
||||
if (waitUntil === "networkidle") {
|
||||
// webNavigation cannot prove network idle. Subscribe on the new
|
||||
// document before enabling CDP lifecycle events after its commit.
|
||||
const frame = await readMainFrameInfo(deps.cdp, tabId);
|
||||
if (!frame.frameId) return cdpError("no main frame after navigation");
|
||||
const remaining = deadline - Date.now();
|
||||
if (remaining <= 0) {
|
||||
outcome = { reached: "timeout", lastLifecycle: "commit" };
|
||||
} else {
|
||||
const wait = startLifecycleWait(
|
||||
deps.cdp,
|
||||
tabId,
|
||||
frame.frameId,
|
||||
"networkIdle",
|
||||
remaining,
|
||||
abort.signal,
|
||||
{ loaderId: frame.loaderId ?? undefined },
|
||||
);
|
||||
await deps.cdp.send(tabId, "Page.setLifecycleEventsEnabled", { enabled: true });
|
||||
outcome = await wait.promise;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (outcome.reached === "cancelled" || abort.signal.aborted) {
|
||||
return { code: "cancelled", message: "navigation recovery aborted" };
|
||||
}
|
||||
return {
|
||||
tab_id: tabId,
|
||||
final_url: await readTabUrl(deps.tabsApi, tabId),
|
||||
reached: outcome.reached === "match" ? waitUntil : "timeout",
|
||||
...(outcome.reached === "timeout"
|
||||
? {
|
||||
error_text: `timed out waiting for ${waitUntil} during navigation recovery after ${timeoutMs}ms`,
|
||||
}
|
||||
: {}),
|
||||
};
|
||||
} catch (error) {
|
||||
return abort.signal.aborted
|
||||
? { code: "cancelled", message: "navigation recovery aborted" }
|
||||
: cdpError(error);
|
||||
} finally {
|
||||
abort.abort();
|
||||
abort.cleanup();
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// tool.navigate
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -505,7 +587,20 @@ export async function handleNavigate(
|
||||
|
||||
try {
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
await ensureCdpReady(deps.cdp, target.tabId);
|
||||
try {
|
||||
await ensureCdpReady(deps.cdp, target.tabId);
|
||||
} catch (error) {
|
||||
if (!isCdpExtensionAccessDenied(error)) throw error;
|
||||
const recovered = await recoverBrowserNavigation(
|
||||
deps,
|
||||
target.tabId,
|
||||
(api) => api.update(target.tabId, { url: params.url }),
|
||||
waitUntil,
|
||||
timeoutMs,
|
||||
);
|
||||
if (isRpcError(recovered)) return recovered;
|
||||
return attachDialogs(deps.cdp, target.tabId, dialogCursor, { ...recovered, url: params.url });
|
||||
}
|
||||
const expected = cdpLifecycleName(waitUntil);
|
||||
const beforeReadyState = await probeMainFrameReadyState(deps.cdp, target.tabId);
|
||||
const beforeFrame = await readMainFrameInfo(deps.cdp, target.tabId);
|
||||
@@ -574,10 +669,7 @@ export async function handleNavigate(
|
||||
}`,
|
||||
});
|
||||
} catch (err) {
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
message: err instanceof Error ? err.message : String(err),
|
||||
};
|
||||
return cdpError(err);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -727,7 +819,24 @@ export async function handleReload(
|
||||
|
||||
try {
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
await ensureCdpReady(deps.cdp, target.tabId);
|
||||
try {
|
||||
await ensureCdpReady(deps.cdp, target.tabId);
|
||||
} catch (error) {
|
||||
if (!isCdpExtensionAccessDenied(error)) throw error;
|
||||
const previousUrl = await readTabUrl(deps.tabsApi, target.tabId);
|
||||
const recovered = await recoverBrowserNavigation(
|
||||
deps,
|
||||
target.tabId,
|
||||
(api) => api.reload(target.tabId, { bypassCache: ignoreCache }),
|
||||
waitUntil,
|
||||
timeoutMs,
|
||||
);
|
||||
if (isRpcError(recovered)) return recovered;
|
||||
return attachDialogs(deps.cdp, target.tabId, dialogCursor, {
|
||||
...recovered,
|
||||
previous_url: previousUrl,
|
||||
});
|
||||
}
|
||||
const previousUrl = await readTabUrl(deps.tabsApi, target.tabId);
|
||||
const beforeFrame = await readMainFrameInfo(deps.cdp, target.tabId);
|
||||
const expected = cdpLifecycleName(waitUntil);
|
||||
@@ -776,10 +885,7 @@ export async function handleReload(
|
||||
}`,
|
||||
});
|
||||
} catch (err) {
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
message: err instanceof Error ? err.message : String(err),
|
||||
};
|
||||
return cdpError(err);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -17,7 +17,7 @@ import { ChromiumCdp } from "@/browser-driver/chromium-cdp";
|
||||
import type { CdpTarget } from "@/browser-driver/frame-graph";
|
||||
import {
|
||||
type CaptureSuppressSendToTab,
|
||||
withOverlaysHiddenForCapture,
|
||||
withExtensionOverlayHidden,
|
||||
} from "@/lib/capture-suppress-bridge";
|
||||
import type { SessionContext, SessionManager } from "@/session-manager/manager";
|
||||
import type {
|
||||
@@ -49,11 +49,14 @@ import {
|
||||
import { resolveSnapshotRef } from "./snapshot-ref";
|
||||
import {
|
||||
type CapturedNode,
|
||||
type CapturedSurfaceProbe,
|
||||
type CapturedViewModel,
|
||||
captureViewModel,
|
||||
collectOverlayExcludedBackendIds,
|
||||
probeHoverSurfaces,
|
||||
} from "./vom/capture";
|
||||
import { type CapturedFrameDocument, captureFrameData } from "./vom/frame-capture";
|
||||
import { withOverlayBypass } from "./vom/hover-perception";
|
||||
import { probeTooltipNames } from "./vom/name-enrichment";
|
||||
import {
|
||||
type CaptureVomObservationResult,
|
||||
@@ -321,7 +324,7 @@ export async function handleScreenshot(
|
||||
tabId: target.tabId,
|
||||
...(node.cdpSessionId ? { sessionId: node.cdpSessionId } : {}),
|
||||
};
|
||||
const captured = await withOverlaysHiddenForCapture(
|
||||
const captured = await withExtensionOverlayHidden(
|
||||
target.tabId,
|
||||
() =>
|
||||
captureElementScreenshot(
|
||||
@@ -353,7 +356,7 @@ export async function handleScreenshot(
|
||||
);
|
||||
}
|
||||
|
||||
const captured = await withOverlaysHiddenForCapture(
|
||||
const captured = await withExtensionOverlayHidden(
|
||||
target.tabId,
|
||||
() => captureFullTabPng(deps, ctx, target, signal),
|
||||
deps.sendToTab,
|
||||
@@ -525,8 +528,11 @@ function buildActiveScopeBlocks(nodes: VomNode[], signals: VomNodeDomSignals): A
|
||||
return blocks;
|
||||
}
|
||||
|
||||
function buildConditionalSurfaces(nodes: VomNode[], captured: CapturedViewModel): CondSurface[] {
|
||||
const probes = captured.surfaceProbes ?? [];
|
||||
function buildConditionalSurfaces(
|
||||
nodes: VomNode[],
|
||||
captured: CapturedViewModel,
|
||||
probes: CapturedSurfaceProbe[],
|
||||
): CondSurface[] {
|
||||
if (probes.length === 0) return [];
|
||||
|
||||
const surfaces: CondSurface[] = [];
|
||||
@@ -649,6 +655,7 @@ function findSurfaceNodeByPoint(
|
||||
export interface BuildVomSceneOptions {
|
||||
pageUrl?: string;
|
||||
supplementalNames?: ReadonlyMap<string, string>;
|
||||
surfaceProbes?: CapturedSurfaceProbe[];
|
||||
}
|
||||
|
||||
export type VomFrameDocument = CapturedFrameDocument<CdpAxNode>;
|
||||
@@ -817,18 +824,19 @@ export function buildFrameVomScene(
|
||||
excludedBackendNodeIds: captured.excludedBackendNodeIds,
|
||||
supplementalNames: options.supplementalNames,
|
||||
});
|
||||
return attachCapturedSceneAnnotations(scene, documents, captured);
|
||||
return attachCapturedSceneAnnotations(scene, documents, captured, options.surfaceProbes ?? []);
|
||||
}
|
||||
|
||||
function attachCapturedSceneAnnotations(
|
||||
scene: VomScene,
|
||||
documents: VomFrameDocument[],
|
||||
captured: CapturedViewModel,
|
||||
surfaceProbes: CapturedSurfaceProbe[],
|
||||
): VomScene {
|
||||
const rootDocument = documents.find((document) => document.frameId === scene.rootFrameId);
|
||||
const signals = capturedOnlySignals(rootDocument?.domNodes ?? captured.nodes);
|
||||
const activeScopeBlocks = buildActiveScopeBlocks(scene.nodes, signals);
|
||||
const surfaces = buildConditionalSurfaces(scene.nodes, captured);
|
||||
const surfaces = buildConditionalSurfaces(scene.nodes, captured, surfaceProbes);
|
||||
return {
|
||||
...scene,
|
||||
...(surfaces.length > 0 ? { surfaces } : {}),
|
||||
@@ -1010,17 +1018,69 @@ async function captureForVom(
|
||||
options: CaptureVomObservationOptions,
|
||||
): Promise<CapturedViewModel> {
|
||||
try {
|
||||
return await captureViewModel(cdp, tabId, {
|
||||
conditionalSurfaceProbe: options.conditionalSurfaceProbe ?? false,
|
||||
hoverProbeBypassOverlay: options.hoverProbeBypassOverlay,
|
||||
signal: options.signal,
|
||||
});
|
||||
return await captureViewModel(cdp, tabId, { signal: options.signal });
|
||||
} catch (error) {
|
||||
if (isAbortError(error)) throw error;
|
||||
return fallbackCapturedViewModel(cdp, tabId, options.signal);
|
||||
}
|
||||
}
|
||||
|
||||
export interface HoverProbeOutcome {
|
||||
/** Whether any active hover was dispatched during this observation. */
|
||||
performed: boolean;
|
||||
/**
|
||||
* Whether hovering surfaced content absent from the static snapshot. Cached
|
||||
* tooltip names count, so this may over-report on repeat observations — it
|
||||
* errs towards telling the caller the snapshot is less fresh than it looks.
|
||||
*/
|
||||
revealedContent: boolean;
|
||||
surfaceProbes: CapturedSurfaceProbe[];
|
||||
tooltipNames: Map<string, string>;
|
||||
}
|
||||
|
||||
const NO_HOVER_PROBES: HoverProbeOutcome = {
|
||||
performed: false,
|
||||
revealedContent: false,
|
||||
surfaceProbes: [],
|
||||
tooltipNames: new Map(),
|
||||
};
|
||||
|
||||
/**
|
||||
* Runs both active-hover chains back to back.
|
||||
*
|
||||
* Ordering matters: this happens after DOM *and* accessibility capture so a
|
||||
* hover that opens a menu cannot leave one half of the observation describing
|
||||
* the page before the change and the other half after it. Sharing one overlay
|
||||
* bypass span also means the agent overlay toggles once per observation
|
||||
* instead of once per chain.
|
||||
*/
|
||||
async function runHoverProbes(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
captured: CapturedViewModel,
|
||||
documents: VomFrameDocument[],
|
||||
staticSemantics: ReturnType<typeof resolveSemanticGraph>,
|
||||
options: CaptureVomObservationOptions,
|
||||
): Promise<HoverProbeOutcome> {
|
||||
if (!options.conditionalSurfaceProbe) return NO_HOVER_PROBES;
|
||||
|
||||
return withOverlayBypass(options.hoverProbeBypassOverlay, tabId, async () => {
|
||||
const surfaceProbes = await probeHoverSurfaces(cdp, tabId, captured.nodes, {
|
||||
signal: options.signal,
|
||||
});
|
||||
throwIfAborted(options.signal, "observation");
|
||||
const tooltipNames = await probeTooltipNames(cdp, tabId, documents, staticSemantics, {
|
||||
signal: options.signal,
|
||||
});
|
||||
return {
|
||||
performed: true,
|
||||
revealedContent: surfaceProbes.length > 0 || tooltipNames.size > 0,
|
||||
surfaceProbes,
|
||||
tooltipNames,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
export interface CaptureVomObservationOptions extends VomOptions {
|
||||
conditionalSurfaceProbe?: boolean;
|
||||
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
|
||||
@@ -1051,19 +1111,26 @@ export async function captureVomObservation(
|
||||
excludedBackendNodeIds: captured.excludedBackendNodeIds,
|
||||
});
|
||||
const staticSemantics = resolveSemanticGraph(semanticGraph, { identifierFallback: false });
|
||||
const tooltipNames = options.conditionalSurfaceProbe
|
||||
? await probeTooltipNames(cdp, tabId, normalizedDocuments, staticSemantics, {
|
||||
signal: options.signal,
|
||||
hoverProbeBypassOverlay: options.hoverProbeBypassOverlay,
|
||||
})
|
||||
: new Map<string, string>();
|
||||
const hoverProbes = await runHoverProbes(
|
||||
cdp,
|
||||
tabId,
|
||||
captured,
|
||||
normalizedDocuments,
|
||||
staticSemantics,
|
||||
options,
|
||||
);
|
||||
throwIfAborted(options.signal, "observation");
|
||||
const scene = projectSemanticGraph(
|
||||
normalizeSemanticStructure(
|
||||
resolveSemanticGraph(semanticGraph, { supplementalNames: tooltipNames }),
|
||||
resolveSemanticGraph(semanticGraph, { supplementalNames: hoverProbes.tooltipNames }),
|
||||
),
|
||||
);
|
||||
const decoratedScene = attachCapturedSceneAnnotations(scene, normalizedDocuments, captured);
|
||||
const decoratedScene = attachCapturedSceneAnnotations(
|
||||
scene,
|
||||
normalizedDocuments,
|
||||
captured,
|
||||
hoverProbes.surfaceProbes,
|
||||
);
|
||||
const rendered = renderVom(decoratedScene, {
|
||||
maxDepth: options.maxDepth,
|
||||
maxTokens: options.maxTokens,
|
||||
@@ -1075,7 +1142,11 @@ export async function captureVomObservation(
|
||||
rootFrameId: captured.rootFrameId ?? normalizedDocuments[0]?.frameId ?? "root",
|
||||
frameDocuments: normalizedDocuments,
|
||||
rendered,
|
||||
surfaceProbes: captured.surfaceProbes,
|
||||
surfaceProbes: hoverProbes.surfaceProbes,
|
||||
hoverProbe: {
|
||||
performed: hoverProbes.performed,
|
||||
revealedContent: hoverProbes.revealedContent,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1141,6 +1212,14 @@ async function handleVomObservation(
|
||||
ref_count: observation.refs.length,
|
||||
tab_id: target.tabId,
|
||||
truncated: observation.truncated,
|
||||
...(toolName === "observe" && observation.hoverProbe?.performed
|
||||
? {
|
||||
hover_probe: {
|
||||
performed: true,
|
||||
revealed_content: observation.hoverProbe.revealedContent,
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
...(toolName === "observe" && (params as ObserveParams).debug_surfaces
|
||||
? {
|
||||
debug: {
|
||||
@@ -1179,5 +1258,13 @@ export async function handleObserve(
|
||||
deps: SnapshotDeps = getDefaultDeps(),
|
||||
signal?: AbortSignal,
|
||||
): Promise<ObserveResult | RpcError> {
|
||||
return handleVomObservation(manager, params, "observe", "transient_input", true, deps, signal);
|
||||
return handleVomObservation(
|
||||
manager,
|
||||
params,
|
||||
"observe",
|
||||
"transient_input",
|
||||
params.probe_hover === true,
|
||||
deps,
|
||||
signal,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ import { type SessionManager, SessionStartCleanupError } from "@/session-manager
|
||||
import type { RpcError } from "@/transport/types";
|
||||
import { rpcError } from "./errors";
|
||||
import { clearRecordingForSession } from "./record";
|
||||
import type { CdpRunner, ChromeTabsApi } from "./shared";
|
||||
import { isRpcError } from "./shared";
|
||||
import { returnBorrowedTab, type TabManagementDeps } from "./tabs";
|
||||
|
||||
@@ -73,11 +74,17 @@ export interface SessionStopResult {
|
||||
/** Tab ids whose return path failed; those entries remain borrowed so
|
||||
* shutdown can be retried without closing the Agent Window. */
|
||||
return_failures?: Array<{ tab_id: number; code: string; message: string }>;
|
||||
/**
|
||||
* True when the Agent Window was *released* to the user (instead of being
|
||||
* closed) because it still contained user-created tabs after the agent's own
|
||||
* tabs were closed. Single-session scope.
|
||||
*/
|
||||
window_released?: boolean;
|
||||
}
|
||||
|
||||
export interface SessionStopDeps {
|
||||
signal?: AbortSignal;
|
||||
cdp?: {
|
||||
cdp?: Pick<CdpRunner, "releaseSessionTab"> & {
|
||||
detachSession(sessionId: string): Promise<void>;
|
||||
};
|
||||
/**
|
||||
@@ -87,6 +94,13 @@ export interface SessionStopDeps {
|
||||
* without a real browser.
|
||||
*/
|
||||
tabManagement?: TabManagementDeps;
|
||||
/**
|
||||
* Read-only Chrome tabs API used to *query* remaining tabs before deciding
|
||||
* whether to release the window. Kept separate from `tabManagement.tabs`
|
||||
* (a `TabMutationApi` that has no `query`) so the mutation interface stays
|
||||
* pure. Tests can inject a fake `ChromeTabsApi` too.
|
||||
*/
|
||||
tabsQuery?: ChromeTabsApi;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -152,8 +166,10 @@ export async function handleSessionStart(
|
||||
* cleanly (review parity with M6).
|
||||
* 3. Detach CDP sessions the extension still holds for this
|
||||
* session (no-op if M6/M7 didn't attach to any tab).
|
||||
* 4. Close the Agent Window. SessionManager.stop() removes the
|
||||
* Chrome window and forgets the context.
|
||||
* 4. Close the agent-created tabs and the home tab (design §3.2).
|
||||
* 5. If any tab remains — those are user-created tabs — release the
|
||||
* window to the user (dropOnly) instead of closing it, so user
|
||||
* tabs survive. Otherwise close the window as before.
|
||||
*
|
||||
* Failures in step 1 keep the Agent Window open: a failed borrowed tab
|
||||
* may still be there, so closing the window would risk losing user
|
||||
@@ -188,15 +204,19 @@ export async function handleSessionStop(
|
||||
const returnFailures: SessionStopResult["return_failures"] = [];
|
||||
const borrowedIds = Array.from(ctx.borrowedTabs.keys());
|
||||
for (const tabId of borrowedIds) {
|
||||
// A close event may have removed this borrow while another tab returned.
|
||||
if (!ctx.borrowedTabs.has(tabId)) continue;
|
||||
try {
|
||||
const tabManagement = {
|
||||
...(deps.tabManagement ?? {}),
|
||||
cdp: deps.cdp ?? deps.tabManagement?.cdp,
|
||||
signal: deps.signal,
|
||||
isAgentWindowId:
|
||||
deps.tabManagement?.isAgentWindowId ??
|
||||
((windowId: number) => manager.findByWindowId(windowId) !== null),
|
||||
};
|
||||
const outcome = await returnBorrowedTab(ctx, tabId, tabManagement);
|
||||
if (!ctx.borrowedTabs.has(tabId)) continue;
|
||||
if (typeof outcome === "object" && "code" in outcome) {
|
||||
console.warn(`[bsk session_stop] auto-return failed for tab ${tabId}`, outcome);
|
||||
returnFailures?.push({
|
||||
@@ -209,6 +229,7 @@ export async function handleSessionStop(
|
||||
ctx.borrowedTabs.delete(tabId);
|
||||
}
|
||||
} catch (err) {
|
||||
if (!ctx.borrowedTabs.has(tabId)) continue;
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
console.warn(`[bsk session_stop] auto-return threw for tab ${tabId}: ${message}`);
|
||||
returnFailures?.push({
|
||||
@@ -219,8 +240,15 @@ export async function handleSessionStop(
|
||||
}
|
||||
}
|
||||
|
||||
// A previously failed tab may have closed during a later return. Only
|
||||
// confirmed close events remove borrows; ordinary move failures stay retryable.
|
||||
const pendingReturnFailures = returnFailures.filter((failure) =>
|
||||
ctx.borrowedTabs.has(failure.tab_id),
|
||||
);
|
||||
if (deps.signal?.aborted) {
|
||||
const nonCancelFailures = returnFailures?.filter((failure) => failure.code !== "cancelled");
|
||||
const nonCancelFailures = pendingReturnFailures.filter(
|
||||
(failure) => failure.code !== "cancelled",
|
||||
);
|
||||
if (nonCancelFailures && nonCancelFailures.length > 0) {
|
||||
return {
|
||||
...(returnedTabIds.length > 0 ? { returned_tab_ids: returnedTabIds } : {}),
|
||||
@@ -232,8 +260,8 @@ export async function handleSessionStop(
|
||||
|
||||
const result: SessionStopResult = {};
|
||||
if (returnedTabIds.length > 0) result.returned_tab_ids = returnedTabIds;
|
||||
if (returnFailures && returnFailures.length > 0) {
|
||||
result.return_failures = returnFailures;
|
||||
if (pendingReturnFailures.length > 0) {
|
||||
result.return_failures = pendingReturnFailures;
|
||||
// A failed return means at least one borrowed user tab may still be
|
||||
// inside the Agent Window. Keep the session/window alive so the user
|
||||
// can retry `bsk session stop` or explicitly `bsk tab return` after the
|
||||
@@ -253,8 +281,66 @@ export async function handleSessionStop(
|
||||
return { code: "cancelled", message: "session_stop aborted before window close" };
|
||||
}
|
||||
|
||||
// Step 4: close the Agent Window and drop the context.
|
||||
await manager.stop(params.session_id);
|
||||
// Step 4: close every tab explicitly created by the agent, including the
|
||||
// home tab. `tabsApi` is a
|
||||
// TabMutationApi (remove only); `queryApi` is a separate read-only
|
||||
// ChromeTabsApi. If neither is injected, we conservatively fall back to
|
||||
// closing the window (see Step 5).
|
||||
const tabsApi = deps.tabManagement?.tabs;
|
||||
const queryApi = deps.tabsQuery;
|
||||
|
||||
if (tabsApi) {
|
||||
// Close each agent-created tab that still exists.
|
||||
const agentCreatedTabIds = Array.from(ctx.agentCreatedTabs);
|
||||
for (const tabId of agentCreatedTabIds) {
|
||||
try {
|
||||
await tabsApi.remove(tabId);
|
||||
ctx.agentCreatedTabs.delete(tabId);
|
||||
} catch (err) {
|
||||
// Tab may already be gone (closed by the user). Non-fatal.
|
||||
console.warn(`[bsk session_stop] failed to close agent tab ${tabId}`, err);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Step 5: decide whether to release (keep) the window or close it.
|
||||
let shouldRelease = false;
|
||||
if (queryApi) {
|
||||
try {
|
||||
const liveWindowTabs = await queryApi.query({ windowId: ctx.agentWindowId });
|
||||
// Only genuine *user* tabs count toward keeping the window open.
|
||||
// An agent tab that failed to close in Step 4 may still be present
|
||||
// here; if we counted it as a reason to release (dropOnly), the
|
||||
// window would be kept and that agent tab would leak (issue #57
|
||||
// regression). Exclude any id still tracked in agentCreatedTabs.
|
||||
const userTabs = liveWindowTabs.filter((t) => {
|
||||
if (t.id === undefined) return false;
|
||||
return !ctx.agentCreatedTabs.has(t.id);
|
||||
});
|
||||
const leakedAgentTabs = liveWindowTabs.filter(
|
||||
(t) => t.id !== undefined && ctx.agentCreatedTabs.has(t.id),
|
||||
);
|
||||
if (leakedAgentTabs.length > 0) {
|
||||
console.warn(
|
||||
`[bsk session_stop] ${leakedAgentTabs.length} agent tab(s) failed to close; forcing window close instead of release`,
|
||||
leakedAgentTabs.map((t) => t.id),
|
||||
);
|
||||
}
|
||||
shouldRelease = userTabs.length > 0;
|
||||
} catch {
|
||||
// Query failed (e.g. window already gone) — conservatively close it.
|
||||
shouldRelease = false;
|
||||
}
|
||||
}
|
||||
|
||||
if (shouldRelease) {
|
||||
// Keep the window + its user tabs; only drop the session binding.
|
||||
await manager.stop(params.session_id, { dropOnly: true });
|
||||
result.window_released = true;
|
||||
} else {
|
||||
// Window is empty (or we couldn't verify state) — close it.
|
||||
await manager.stop(params.session_id);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
@@ -53,9 +53,11 @@ export type { DialogCursor };
|
||||
export interface CdpRunner {
|
||||
send<T = unknown>(tabId: number, method: string, params?: object): Promise<T>;
|
||||
sendToTarget?<T = unknown>(target: CdpTarget, method: string, params?: object): Promise<T>;
|
||||
detach?(tabId: number): Promise<void>;
|
||||
getFrameGraph?(tabId: number): Promise<CdpFrameGraph>;
|
||||
ensureAttachedToUrl?(tabId: number, expectedUrl: string | undefined): Promise<void>;
|
||||
trackSessionTab?(sessionId: string, tabId: number): void;
|
||||
releaseSessionTab?(sessionId: string, tabId: number): Promise<void>;
|
||||
onEvent?(handler: (source: CdpDebuggee, method: string, params: unknown) => void): {
|
||||
dispose(): void;
|
||||
};
|
||||
|
||||
@@ -10,7 +10,7 @@ import {
|
||||
import type { SessionContext, SessionManager } from "@/session-manager/manager";
|
||||
import type { RpcError } from "@/transport/types";
|
||||
import { rpcError } from "./errors";
|
||||
import { isRpcError, lookupSession } from "./shared";
|
||||
import { type CdpRunner, isRpcError, lookupSession } from "./shared";
|
||||
|
||||
export type TabScope = "user" | "agent" | "all";
|
||||
|
||||
@@ -287,6 +287,10 @@ export interface TabManagementDeps {
|
||||
approveBorrow?: BorrowConfirmationApprover;
|
||||
/** Clears Agent-scoped overlays after a borrowed tab is returned. */
|
||||
agentOverlayReset?: AgentOverlayResetApi;
|
||||
/** Releases this session's CDP claim after a borrowed tab is returned. */
|
||||
cdp?: Pick<CdpRunner, "releaseSessionTab">;
|
||||
/** Runs after tab_return validation, before moving the borrowed tab. */
|
||||
beforeReturn?: (sessionId: string, tabId: number) => Promise<void>;
|
||||
/**
|
||||
* Reports whether `windowId` is any live session's Agent Window.
|
||||
* `tab_return`'s fallback window picker uses this to avoid moving a
|
||||
@@ -369,6 +373,7 @@ function buildCreateProps(
|
||||
* Returns the created tab on success, or an `RpcError` on failure.
|
||||
*/
|
||||
async function createTabAndCleanup(
|
||||
ctx: SessionContext,
|
||||
deps: TabManagementDeps,
|
||||
createProps: chrome.tabs.CreateProperties,
|
||||
): Promise<CreatedChromeTab | RpcError> {
|
||||
@@ -381,25 +386,28 @@ async function createTabAndCleanup(
|
||||
message: err instanceof Error ? err.message : String(err),
|
||||
};
|
||||
}
|
||||
if (aborted(deps.signal, "tab_create")) {
|
||||
// We already opened the tab; close it on abort so we don't leak.
|
||||
if (typeof tab.id === "number") {
|
||||
try {
|
||||
await getTabsApi(deps).remove(tab.id);
|
||||
} catch (cleanupErr) {
|
||||
return rpcError(
|
||||
"protocol_error",
|
||||
"cleanup_failed",
|
||||
`tab_create aborted but cleanup of tab ${tab.id} failed: ${describeError(cleanupErr)}`,
|
||||
{ resource_type: "tab", resource_id: tab.id },
|
||||
);
|
||||
}
|
||||
}
|
||||
return { code: "cancelled", message: "tab_create aborted" };
|
||||
}
|
||||
if (typeof tab.id !== "number") {
|
||||
return { code: "protocol_error", message: "chrome.tabs.create returned no tab id" };
|
||||
}
|
||||
// Claim the concrete id returned by Chrome. Ownership never depends on
|
||||
// matching this request to an asynchronous onCreated event.
|
||||
ctx.agentCreatedTabs.add(tab.id);
|
||||
if (aborted(deps.signal, "tab_create")) {
|
||||
try {
|
||||
await getTabsApi(deps).remove(tab.id);
|
||||
ctx.agentCreatedTabs.delete(tab.id);
|
||||
} catch (cleanupErr) {
|
||||
// Keep the claim when cleanup fails so session_stop can retry instead
|
||||
// of releasing an agent-owned tab to the user.
|
||||
return rpcError(
|
||||
"protocol_error",
|
||||
"cleanup_failed",
|
||||
`tab_create aborted but cleanup of tab ${tab.id} failed: ${describeError(cleanupErr)}`,
|
||||
{ resource_type: "tab", resource_id: tab.id },
|
||||
);
|
||||
}
|
||||
return { code: "cancelled", message: "tab_create aborted" };
|
||||
}
|
||||
return { ...tab, id: tab.id };
|
||||
}
|
||||
|
||||
@@ -426,7 +434,7 @@ export async function handleTabCreate(
|
||||
const paramErr = validateTabCreateParams(params);
|
||||
if (paramErr) return paramErr;
|
||||
|
||||
const tab = await createTabAndCleanup(deps, buildCreateProps(ctx, params));
|
||||
const tab = await createTabAndCleanup(ctx, deps, buildCreateProps(ctx, params));
|
||||
if (isRpcError(tab)) return tab;
|
||||
|
||||
return {
|
||||
@@ -526,6 +534,9 @@ export async function handleTabClose(
|
||||
}
|
||||
try {
|
||||
await getTabsApi(deps).remove(params.tab_id);
|
||||
// Keep the tracking set accurate so session_stop won't try to close a
|
||||
// tab that's already gone (design §3.1).
|
||||
ctx.agentCreatedTabs.delete(params.tab_id);
|
||||
} catch (err) {
|
||||
return {
|
||||
code: "protocol_error",
|
||||
@@ -753,13 +764,6 @@ async function executeBorrowCore(
|
||||
);
|
||||
if (moveErr) return moveErr;
|
||||
|
||||
// Best-effort activation — failure is non-fatal.
|
||||
try {
|
||||
await p.tabsApi.update(p.tabId, { active: true });
|
||||
} catch (err) {
|
||||
console.debug("[bsk tab_borrow] activate after move failed", err);
|
||||
}
|
||||
|
||||
return { originalWindowId, originalIndex };
|
||||
}
|
||||
|
||||
@@ -823,6 +827,12 @@ export async function handleTabBorrow(
|
||||
message: err instanceof Error ? err.message : String(err),
|
||||
};
|
||||
}
|
||||
// Activate after commit so overlay/event observers see the tab as claimed.
|
||||
try {
|
||||
await tabsApi.update(params.tab_id, { active: true });
|
||||
} catch (err) {
|
||||
console.debug("[bsk tab_borrow] activate after move failed", err);
|
||||
}
|
||||
return {
|
||||
tab_id: params.tab_id,
|
||||
original_window_id: coreResult.originalWindowId,
|
||||
@@ -939,16 +949,22 @@ async function cleanupUnusedFallbackWindow(
|
||||
}
|
||||
}
|
||||
|
||||
function resetAgentOverlaysInReturnedTab(
|
||||
async function releaseReturnedTabState(
|
||||
ctx: SessionContext,
|
||||
tabId: number,
|
||||
deps: TabManagementDeps,
|
||||
): void {
|
||||
): Promise<void> {
|
||||
void getAgentOverlayResetApi(deps)
|
||||
.resetAgentOverlays(tabId, ctx.sessionId)
|
||||
.catch((err) => {
|
||||
console.debug("[bsk tab_return] agent overlay reset failed", err);
|
||||
});
|
||||
try {
|
||||
await deps.cdp?.releaseSessionTab?.(ctx.sessionId, tabId);
|
||||
} catch (err) {
|
||||
// A successful move must not be retried because debugger cleanup failed.
|
||||
console.debug("[bsk tab_return] CDP release failed", err);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -1022,7 +1038,7 @@ export async function returnBorrowedTab(
|
||||
});
|
||||
const movedTab = Array.isArray(moved) ? moved[0] : moved;
|
||||
const finalIndex = typeof movedTab?.index === "number" ? movedTab.index : targetIndex;
|
||||
resetAgentOverlaysInReturnedTab(ctx, tabId, deps);
|
||||
await releaseReturnedTabState(ctx, tabId, deps);
|
||||
return {
|
||||
tabId,
|
||||
toWindowId: targetWindowId,
|
||||
@@ -1064,7 +1080,7 @@ export async function returnBorrowedTab(
|
||||
});
|
||||
const movedTab = Array.isArray(moved) ? moved[0] : moved;
|
||||
const finalIndex = typeof movedTab?.index === "number" ? movedTab.index : target.index;
|
||||
resetAgentOverlaysInReturnedTab(ctx, tabId, deps);
|
||||
await releaseReturnedTabState(ctx, tabId, deps);
|
||||
return {
|
||||
tabId,
|
||||
toWindowId: target.windowId,
|
||||
@@ -1110,6 +1126,7 @@ export async function handleTabReturn(
|
||||
message: `tab_return: tab ${params.tab_id} is not borrowed by this session`,
|
||||
};
|
||||
}
|
||||
await deps.beforeReturn?.(ctx.sessionId, params.tab_id);
|
||||
const outcome = await returnBorrowedTab(ctx, params.tab_id, {
|
||||
...deps,
|
||||
isAgentWindowId:
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
// Shared time boundary for browser-side file-transfer transactions. Concrete
|
||||
// upload/download mechanisms own their state machines and effect semantics.
|
||||
|
||||
export class BoundedWaitError extends Error {
|
||||
constructor(
|
||||
readonly kind: "timeout" | "aborted",
|
||||
message: string,
|
||||
) {
|
||||
super(message);
|
||||
}
|
||||
}
|
||||
|
||||
export function remainingMs(deadline: number): number {
|
||||
return Math.max(0, deadline - Date.now());
|
||||
}
|
||||
|
||||
export async function waitBounded<T>(
|
||||
promise: Promise<T>,
|
||||
deadline: number,
|
||||
signal: AbortSignal | undefined,
|
||||
timeoutMessage: string,
|
||||
): Promise<T> {
|
||||
const remaining = remainingMs(deadline);
|
||||
if (signal?.aborted) throw new BoundedWaitError("aborted", "file transfer aborted");
|
||||
if (remaining === 0) throw new BoundedWaitError("timeout", timeoutMessage);
|
||||
|
||||
let timer: ReturnType<typeof setTimeout> | undefined;
|
||||
let onAbort: (() => void) | undefined;
|
||||
const boundary = new Promise<never>((_resolve, reject) => {
|
||||
timer = setTimeout(() => reject(new BoundedWaitError("timeout", timeoutMessage)), remaining);
|
||||
if (signal) {
|
||||
onAbort = () => reject(new BoundedWaitError("aborted", "file transfer aborted"));
|
||||
signal.addEventListener("abort", onAbort, { once: true });
|
||||
}
|
||||
});
|
||||
try {
|
||||
return await Promise.race([promise, boundary]);
|
||||
} finally {
|
||||
if (timer) clearTimeout(timer);
|
||||
if (signal && onAbort) signal.removeEventListener("abort", onAbort);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
// Upload orchestration: validate the session-scoped request, resolve its click
|
||||
// target, then delegate the browser protocol transaction to the file-input
|
||||
// transaction module.
|
||||
|
||||
import {
|
||||
type CaptureSuppressSendToTab,
|
||||
withExtensionOverlayHidden,
|
||||
} from "@/lib/capture-suppress-bridge";
|
||||
import type { SessionManager } from "@/session-manager/manager";
|
||||
import type { RpcError, UploadParams, UploadResult } from "@/transport/types";
|
||||
import { uploadThroughFileDrop } from "./file-drop-transaction";
|
||||
import { uploadThroughActivatedFileInput } from "./file-input-transaction";
|
||||
import { clickResolvedTarget, type InteractionDeps, resolveActionTarget } from "./interaction";
|
||||
import {
|
||||
type CdpRunner,
|
||||
enforceAgentWindow,
|
||||
isRpcError,
|
||||
lookupSession,
|
||||
resolveTargetTab,
|
||||
} from "./shared";
|
||||
|
||||
const DEFAULT_TIMEOUT_MS = 120_000;
|
||||
|
||||
export interface UploadDeps extends InteractionDeps {
|
||||
cdp: CdpRunner;
|
||||
sendToTab?: CaptureSuppressSendToTab;
|
||||
}
|
||||
|
||||
export async function handleUpload(
|
||||
manager: SessionManager,
|
||||
params: UploadParams,
|
||||
deps: UploadDeps,
|
||||
): Promise<UploadResult | RpcError> {
|
||||
const ctx = lookupSession(manager, params, "upload");
|
||||
if (isRpcError(ctx)) return ctx;
|
||||
const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi);
|
||||
if (isRpcError(target)) return target;
|
||||
const denied = enforceAgentWindow(ctx, target, "upload");
|
||||
if (denied) return denied;
|
||||
if (
|
||||
params.files.length === 0 ||
|
||||
params.files.length > 20 ||
|
||||
params.files.some((file) => !file.staged_path)
|
||||
) {
|
||||
return { code: "invalid_params", message: "upload requires daemon-staged files" };
|
||||
}
|
||||
const address = await resolveActionTarget(deps.cdp, ctx, target, params, "upload");
|
||||
if (isRpcError(address)) return address;
|
||||
const timeoutMs = params.timeout_ms ?? DEFAULT_TIMEOUT_MS;
|
||||
const files = params.files.map((file) => file.staged_path as string);
|
||||
const mode = params.mode ?? "input";
|
||||
let usedRef = address.usedRef;
|
||||
let usedSelector = address.usedSelector;
|
||||
if (mode === "drop") {
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
const transaction = await uploadThroughFileDrop({
|
||||
cdp: deps.cdp,
|
||||
actionTarget: address,
|
||||
files,
|
||||
timeoutMs,
|
||||
signal: deps.signal,
|
||||
withOverlayHidden: (operation) =>
|
||||
withExtensionOverlayHidden(target.tabId, operation, deps.sendToTab),
|
||||
});
|
||||
if (isRpcError(transaction)) return transaction;
|
||||
} else {
|
||||
const transaction = await uploadThroughActivatedFileInput({
|
||||
cdp: deps.cdp,
|
||||
actionTarget: address,
|
||||
files,
|
||||
timeoutMs,
|
||||
signal: deps.signal,
|
||||
trigger: () => clickResolvedTarget(ctx, address, {}, deps),
|
||||
});
|
||||
if (isRpcError(transaction)) return transaction;
|
||||
usedRef = transaction.click.used_ref;
|
||||
usedSelector = transaction.click.used_selector;
|
||||
}
|
||||
return {
|
||||
tab_id: target.tabId,
|
||||
used_ref: usedRef,
|
||||
used_selector: usedSelector,
|
||||
file_names: params.files.map((file) => file.name),
|
||||
};
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { OVERLAY_HOST_MARKER_ATTR, OVERLAY_HOST_NAME } from "../../../lib/overlay-bridge";
|
||||
import { captureViewModel, collectOverlayExcludedBackendIds } from "../capture";
|
||||
import { captureViewModel, collectOverlayExcludedBackendIds, probeHoverSurfaces } from "../capture";
|
||||
|
||||
// Minimal but format-accurate captureSnapshot reply: a body with one
|
||||
// fixed full-screen overlay div carrying a password input.
|
||||
@@ -215,21 +215,29 @@ function makeCdp(snapshot: unknown) {
|
||||
if (method === "Runtime.evaluate") {
|
||||
return {
|
||||
result: {
|
||||
value: {
|
||||
controls: [
|
||||
deepSerializedValue: {
|
||||
type: "array",
|
||||
value: [
|
||||
{
|
||||
state: "filled",
|
||||
sensitive: true,
|
||||
defaultValue: "hunter2",
|
||||
value: "hunter2",
|
||||
placeholder: "secret",
|
||||
type: "array",
|
||||
value: [
|
||||
{ type: "node", value: { backendNodeId: 13 } },
|
||||
{
|
||||
type: "string",
|
||||
value: JSON.stringify({
|
||||
state: "filled",
|
||||
sensitive: true,
|
||||
placeholder: "secret",
|
||||
}),
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
childFrames: [],
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
if (method === "Runtime.releaseObjectGroup") return {};
|
||||
throw new Error(`unexpected ${method}`);
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
@@ -251,11 +259,10 @@ describe("captureViewModel", () => {
|
||||
expect(input?.attrs).toEqual({ type: "password" });
|
||||
});
|
||||
|
||||
it("does not run conditional surface probes by default", async () => {
|
||||
it("never hovers the page while capturing", async () => {
|
||||
const cdp = makeCdp(fakeSnapshotReply());
|
||||
const result = await captureViewModel(cdp, 4);
|
||||
await captureViewModel(cdp, 4);
|
||||
|
||||
expect(result.surfaceProbes).toEqual([]);
|
||||
expect(cdp.send).not.toHaveBeenCalledWith(4, "Input.dispatchMouseEvent", expect.anything());
|
||||
});
|
||||
|
||||
@@ -274,8 +281,11 @@ describe("captureViewModel", () => {
|
||||
expect(cdp.send).toHaveBeenCalledWith(
|
||||
4,
|
||||
"Runtime.evaluate",
|
||||
expect.objectContaining({ returnByValue: true }),
|
||||
expect.objectContaining({
|
||||
serializationOptions: expect.objectContaining({ serialization: "deep" }),
|
||||
}),
|
||||
);
|
||||
expect(cdp.send).toHaveBeenCalledWith(4, "Runtime.releaseObjectGroup", expect.anything());
|
||||
expect(cdp.send).not.toHaveBeenCalledWith(4, "DOM.resolveNode", expect.anything());
|
||||
expect(cdp.send).not.toHaveBeenCalledWith(4, "Runtime.callFunctionOn", expect.anything());
|
||||
});
|
||||
@@ -319,9 +329,10 @@ describe("captureViewModel", () => {
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
|
||||
const result = await captureViewModel(cdp, 4, { conditionalSurfaceProbe: true });
|
||||
const captured = await captureViewModel(cdp, 4);
|
||||
const surfaceProbes = await probeHoverSurfaces(cdp, 4, captured.nodes);
|
||||
|
||||
expect(result.surfaceProbes).toEqual([
|
||||
expect(surfaceProbes).toEqual([
|
||||
{
|
||||
triggerBackendNodeId: 12,
|
||||
triggerPoint: { x: 956, y: 32 },
|
||||
@@ -378,9 +389,10 @@ describe("captureViewModel", () => {
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
|
||||
const result = await captureViewModel(cdp, 4, { conditionalSurfaceProbe: true });
|
||||
const captured = await captureViewModel(cdp, 4);
|
||||
const surfaceProbes = await probeHoverSurfaces(cdp, 4, captured.nodes);
|
||||
|
||||
expect(result.surfaceProbes).toEqual([
|
||||
expect(surfaceProbes).toEqual([
|
||||
expect.objectContaining({
|
||||
triggerBackendNodeId: 12,
|
||||
subItems: ["My profile", "Sign out"],
|
||||
@@ -422,7 +434,8 @@ describe("captureViewModel", () => {
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
|
||||
await captureViewModel(cdp, 4, { conditionalSurfaceProbe: true });
|
||||
const captured = await captureViewModel(cdp, 4);
|
||||
await probeHoverSurfaces(cdp, 4, captured.nodes);
|
||||
|
||||
expect(hoverMoves).toBe(1);
|
||||
});
|
||||
|
||||
@@ -0,0 +1,270 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { type CdpRunner, cdpRunnerForTarget } from "../../shared";
|
||||
import { captureViewModel } from "../capture";
|
||||
|
||||
interface Control {
|
||||
id: number;
|
||||
value: string;
|
||||
type?: string;
|
||||
}
|
||||
|
||||
function snapshot(groups: Control[][]) {
|
||||
const strings: string[] = [];
|
||||
const str = (value: string) => {
|
||||
let index = strings.indexOf(value);
|
||||
if (index < 0) index = strings.push(value) - 1;
|
||||
return index;
|
||||
};
|
||||
const documents = groups.map((controls, groupIndex) => ({
|
||||
frameId: `frame-${groupIndex}`,
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, ...controls.map(() => 1)],
|
||||
nodeName: [str("html"), str("body"), ...controls.map(() => str("input"))],
|
||||
backendNodeId: [8000 + groupIndex * 2, 8001 + groupIndex * 2, ...controls.map((c) => c.id)],
|
||||
attributes: [
|
||||
[],
|
||||
[],
|
||||
...controls.map((c) => [
|
||||
str("type"),
|
||||
str(c.type ?? "text"),
|
||||
str("placeholder"),
|
||||
str("snapshot placeholder"),
|
||||
]),
|
||||
],
|
||||
inputValue: {
|
||||
index: controls.map((_, i) => i + 2),
|
||||
value: controls.map((c) => str(c.value)),
|
||||
},
|
||||
contentDocumentIndex: { index: [] as number[], value: [] as number[] },
|
||||
},
|
||||
}));
|
||||
const root = documents[0].nodes;
|
||||
for (let i = 1; i < documents.length; i++) {
|
||||
root.contentDocumentIndex.index.push(root.backendNodeId.length);
|
||||
root.contentDocumentIndex.value.push(i);
|
||||
root.parentIndex.push(1);
|
||||
root.nodeName.push(str("iframe"));
|
||||
root.backendNodeId.push(9000 + i);
|
||||
root.attributes.push([]);
|
||||
}
|
||||
return { strings, documents };
|
||||
}
|
||||
|
||||
function state(value: string) {
|
||||
return {
|
||||
value,
|
||||
defaultValue: "",
|
||||
placeholder: "live placeholder",
|
||||
state: value ? "filled" : "empty",
|
||||
sensitive: false,
|
||||
};
|
||||
}
|
||||
|
||||
function reply(entries: Array<[number, unknown]>) {
|
||||
return {
|
||||
result: {
|
||||
deepSerializedValue: {
|
||||
type: "array",
|
||||
value: entries.map(([backendNodeId, fields]) => ({
|
||||
type: "array",
|
||||
value: [
|
||||
{ type: "node", value: { backendNodeId } },
|
||||
{ type: "string", value: JSON.stringify(fields) },
|
||||
],
|
||||
})),
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function fakeCdp(groups: Control[][], evaluate: (params: Record<string, unknown>) => unknown) {
|
||||
const snap = snapshot(groups);
|
||||
const send = vi.fn(async (_tabId: number, method: string, params?: object) => {
|
||||
if (method === "DOMSnapshot.captureSnapshot") return snap;
|
||||
if (method === "Runtime.evaluate") return evaluate(params as Record<string, unknown>);
|
||||
return {};
|
||||
});
|
||||
return { send, cdp: { send: send as CdpRunner["send"] } };
|
||||
}
|
||||
|
||||
describe("form state identity", () => {
|
||||
it("matches reordered controls by identity and preserves controls missing from the batch", async () => {
|
||||
// The snapshot includes a shadow input. The runtime query skips it,
|
||||
// encounters a new input, and sees the remaining inputs in a new order.
|
||||
const { cdp } = fakeCdp(
|
||||
[
|
||||
[
|
||||
{ id: 10, value: "" },
|
||||
{ id: 11, value: "old email" },
|
||||
{ id: 12, value: "old city" },
|
||||
],
|
||||
],
|
||||
() =>
|
||||
reply([
|
||||
[99, state("new control")],
|
||||
[12, state("上海")],
|
||||
[11, state("alice@example.com")],
|
||||
]),
|
||||
);
|
||||
const captured = await captureViewModel(cdp, 7);
|
||||
expect(captured.nodes.find((n) => n.backendNodeId === 10)).toMatchObject({
|
||||
formValue: "",
|
||||
formState: "empty",
|
||||
formPlaceholder: "snapshot placeholder",
|
||||
});
|
||||
expect(captured.nodes.find((n) => n.backendNodeId === 11)).toMatchObject({
|
||||
formValue: "alice@example.com",
|
||||
});
|
||||
expect(captured.nodes.find((n) => n.backendNodeId === 12)).toMatchObject({ formValue: "上海" });
|
||||
expect(captured.nodes.some((n) => n.backendNodeId === 99)).toBe(false);
|
||||
});
|
||||
|
||||
it("does not shift state between frames when one frame is absent from the runtime batch", async () => {
|
||||
const { cdp } = fakeCdp(
|
||||
[
|
||||
[],
|
||||
[{ id: 11, value: "inaccessible frame" }],
|
||||
[{ id: 21, value: "old second" }],
|
||||
[{ id: 31, value: "old third" }],
|
||||
],
|
||||
() =>
|
||||
reply([
|
||||
[31, state("third frame")],
|
||||
[21, state("second frame")],
|
||||
]),
|
||||
);
|
||||
const captured = await captureViewModel(cdp, 7);
|
||||
const controls = [...captured.iframeNodes.values()].flat();
|
||||
expect(controls.find((n) => n.backendNodeId === 11)?.formValue).toBe("inaccessible frame");
|
||||
expect(controls.find((n) => n.backendNodeId === 21)?.formValue).toBe("second frame");
|
||||
expect(controls.find((n) => n.backendNodeId === 31)?.formValue).toBe("third frame");
|
||||
});
|
||||
|
||||
it("keeps identical backend ids in different OOPIF targets separate", async () => {
|
||||
const sendToTarget = vi.fn(async (target, method) => {
|
||||
if (method === "DOMSnapshot.captureSnapshot") return snapshot([[{ id: 11, value: "old" }]]);
|
||||
if (method === "Runtime.evaluate") return reply([[11, state(target.sessionId)]]);
|
||||
return {};
|
||||
});
|
||||
const send = vi.fn(async () => {
|
||||
throw new Error("unexpected top-level target");
|
||||
});
|
||||
const cdp = { send, sendToTarget } as unknown as CdpRunner;
|
||||
for (const sessionId of ["left-frame", "right-frame"]) {
|
||||
const captured = await captureViewModel(cdpRunnerForTarget(cdp, { tabId: 7, sessionId }), 7);
|
||||
expect(captured.nodes.find((n) => n.backendNodeId === 11)?.formValue).toBe(sessionId);
|
||||
expect(sendToTarget).toHaveBeenCalledWith(
|
||||
{ tabId: 7, sessionId },
|
||||
"Runtime.releaseObjectGroup",
|
||||
expect.anything(),
|
||||
);
|
||||
}
|
||||
expect(send).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each([
|
||||
"unsupported",
|
||||
"legacy",
|
||||
"malformed",
|
||||
"invalid-state",
|
||||
])("preserves snapshot data when enrichment is %s", async (mode) => {
|
||||
const { cdp, send } = fakeCdp([[{ id: 11, value: "snapshot value" }]], () => {
|
||||
if (mode === "unsupported") throw new Error("serialization unsupported");
|
||||
if (mode === "legacy") return { result: { value: { controls: [state("wrong value")] } } };
|
||||
const result = reply([
|
||||
[11, mode === "invalid-state" ? { value: "wrong value" } : state("wrong value")],
|
||||
]);
|
||||
if (mode === "malformed")
|
||||
result.result.deepSerializedValue.value[0].value[1].value = "not json";
|
||||
return result;
|
||||
});
|
||||
const captured = await captureViewModel(cdp, 7);
|
||||
expect(captured.nodes.find((n) => n.backendNodeId === 11)).toMatchObject({
|
||||
formValue: "snapshot value",
|
||||
formPlaceholder: "snapshot placeholder",
|
||||
});
|
||||
expect(send).toHaveBeenCalledWith(7, "Runtime.releaseObjectGroup", expect.anything());
|
||||
});
|
||||
|
||||
it("releases the batch's remote objects when capture is cancelled", async () => {
|
||||
const controller = new AbortController();
|
||||
const { cdp, send } = fakeCdp([[{ id: 11, value: "old" }]], () => {
|
||||
controller.abort();
|
||||
return reply([[11, state("new")]]);
|
||||
});
|
||||
await expect(captureViewModel(cdp, 7, { signal: controller.signal })).rejects.toMatchObject({
|
||||
name: "AbortError",
|
||||
});
|
||||
expect(send).toHaveBeenCalledWith(7, "Runtime.releaseObjectGroup", expect.anything());
|
||||
});
|
||||
|
||||
it("skips entries without node identity while still applying valid entries", async () => {
|
||||
const { cdp } = fakeCdp(
|
||||
[
|
||||
[
|
||||
{ id: 11, value: "snapshot value" },
|
||||
{ id: 12, value: "old value" },
|
||||
],
|
||||
],
|
||||
() => {
|
||||
const result = reply([
|
||||
[11, state("wrong value")],
|
||||
[12, state("live value")],
|
||||
]);
|
||||
const element = result.result.deepSerializedValue.value[0].value[0];
|
||||
delete (element.value as { backendNodeId?: number }).backendNodeId;
|
||||
return result;
|
||||
},
|
||||
);
|
||||
const captured = await captureViewModel(cdp, 7);
|
||||
expect(captured.nodes.find((n) => n.backendNodeId === 11)?.formValue).toBe("snapshot value");
|
||||
expect(captured.nodes.find((n) => n.backendNodeId === 12)?.formValue).toBe("live value");
|
||||
});
|
||||
|
||||
it.each([
|
||||
"password",
|
||||
"text",
|
||||
])("redacts a sensitive %s control after identity matching", async (type) => {
|
||||
const { cdp } = fakeCdp([[{ id: 11, value: "secret", type }]], () =>
|
||||
reply([[11, { ...state("secret"), sensitive: true }]]),
|
||||
);
|
||||
const captured = await captureViewModel(cdp, 7);
|
||||
const control = captured.nodes.find((n) => n.backendNodeId === 11);
|
||||
expect(control?.formState).toBe("filled");
|
||||
expect(control?.formValue).toBeUndefined();
|
||||
expect(control?.formDefaultValue).toBeUndefined();
|
||||
expect(control?.attrs.value).toBeUndefined();
|
||||
});
|
||||
|
||||
it("keeps one bounded batch without per-control CDP queries", async () => {
|
||||
const controls = Array.from({ length: 251 }, (_, i) => ({ id: i + 1, value: "old" }));
|
||||
const elements = controls.map((c) => ({
|
||||
backendNodeId: c.id,
|
||||
tagName: "INPUT",
|
||||
type: "text",
|
||||
value: `live ${c.id}`,
|
||||
defaultValue: "",
|
||||
placeholder: "",
|
||||
}));
|
||||
const { cdp, send } = fakeCdp([controls], (params) => {
|
||||
const doc = {
|
||||
querySelectorAll: (selector: string) =>
|
||||
selector === "input,textarea,select" ? elements : [],
|
||||
};
|
||||
const entries = new Function("document", `return ${params.expression}`)(doc) as Array<
|
||||
[{ backendNodeId: number }, string]
|
||||
>;
|
||||
expect(entries).toHaveLength(250);
|
||||
return reply(entries.map(([element, json]) => [element.backendNodeId, JSON.parse(json)]));
|
||||
});
|
||||
const captured = await captureViewModel(cdp, 7);
|
||||
expect(captured.nodes.find((n) => n.backendNodeId === 250)?.formValue).toBe("live 250");
|
||||
expect(captured.nodes.find((n) => n.backendNodeId === 251)?.formValue).toBe("old");
|
||||
expect(send.mock.calls.filter(([, method]) => method === "Runtime.evaluate")).toHaveLength(1);
|
||||
expect(
|
||||
send.mock.calls.filter(([, method]) => method === "Runtime.releaseObjectGroup"),
|
||||
).toHaveLength(1);
|
||||
expect(send).not.toHaveBeenCalledWith(7, "DOM.resolveNode", expect.anything());
|
||||
expect(send).not.toHaveBeenCalledWith(7, "Runtime.callFunctionOn", expect.anything());
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,84 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { ProbeBudget, resetOverlayBypassDepth, withOverlayBypass } from "../hover-perception";
|
||||
|
||||
afterEach(() => {
|
||||
resetOverlayBypassDepth();
|
||||
});
|
||||
|
||||
describe("ProbeBudget", () => {
|
||||
it("refuses a candidate that cannot finish inside the budget", () => {
|
||||
const budget = new ProbeBudget(500);
|
||||
|
||||
expect(budget.canAfford(400)).toBe(true);
|
||||
expect(budget.canAfford(600)).toBe(false);
|
||||
});
|
||||
|
||||
it("stops admitting candidates once the remaining budget is smaller than one", async () => {
|
||||
const budget = new ProbeBudget(60);
|
||||
await new Promise((resolve) => setTimeout(resolve, 40));
|
||||
|
||||
// A top-of-loop `elapsed > total` check would still admit this candidate
|
||||
// and then overshoot by its full cost.
|
||||
expect(budget.canAfford(50)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("withOverlayBypass", () => {
|
||||
it("toggles the overlay once around nested probe phases", async () => {
|
||||
const bypass = vi.fn(async () => {});
|
||||
|
||||
await withOverlayBypass(bypass, 4, async () => {
|
||||
await withOverlayBypass(bypass, 4, async () => {
|
||||
expect(bypass).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
expect(bypass).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
expect(bypass.mock.calls).toEqual([
|
||||
[4, true],
|
||||
[4, false],
|
||||
]);
|
||||
});
|
||||
|
||||
it("restores the overlay when the probe throws", async () => {
|
||||
const bypass = vi.fn(async () => {});
|
||||
|
||||
await expect(
|
||||
withOverlayBypass(bypass, 4, async () => {
|
||||
throw new Error("probe exploded");
|
||||
}),
|
||||
).rejects.toThrow("probe exploded");
|
||||
|
||||
expect(bypass).toHaveBeenLastCalledWith(4, false);
|
||||
});
|
||||
|
||||
it("reports a failed restore instead of swallowing it", async () => {
|
||||
const onRestoreFailure = vi.fn();
|
||||
const bypass = vi.fn(async (_tabId: number, enabled: boolean) => {
|
||||
if (!enabled) throw new Error("tab is gone");
|
||||
});
|
||||
|
||||
await withOverlayBypass(bypass, 4, async () => undefined, { onRestoreFailure });
|
||||
|
||||
expect(onRestoreFailure).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("does not pin the overlay off for the tab when a restore fails", async () => {
|
||||
const failing = vi.fn(async (_tabId: number, enabled: boolean) => {
|
||||
if (!enabled) throw new Error("tab is gone");
|
||||
});
|
||||
await withOverlayBypass(failing, 4, async () => undefined, { onRestoreFailure: () => {} });
|
||||
|
||||
const bypass = vi.fn(async () => {});
|
||||
await withOverlayBypass(bypass, 4, async () => undefined);
|
||||
|
||||
expect(bypass.mock.calls).toEqual([
|
||||
[4, true],
|
||||
[4, false],
|
||||
]);
|
||||
});
|
||||
|
||||
it("runs the probe untouched when no bypass is wired up", async () => {
|
||||
await expect(withOverlayBypass(undefined, 4, async () => "done")).resolves.toBe("done");
|
||||
});
|
||||
});
|
||||
@@ -132,6 +132,37 @@ describe("tooltip name enrichment", () => {
|
||||
expect(cdp.sendToTarget).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("remembers controls that revealed no tooltip so the miss is paid once", async () => {
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async () => ({})) as CdpRunner["send"],
|
||||
sendToTarget: vi.fn(async (_target, method) => {
|
||||
if (method === "Page.createIsolatedWorld") return { executionContextId: 31 };
|
||||
if (method === "DOM.resolveNode") return { object: { objectId: "button-801" } };
|
||||
if (method === "Runtime.callFunctionOn") return { result: { value: [] } };
|
||||
return {};
|
||||
}) as CdpRunner["sendToTarget"],
|
||||
};
|
||||
|
||||
const documents = [
|
||||
document({ tabId: 11, sessionId: "miss-session" }, 801, undefined, "miss-frame"),
|
||||
];
|
||||
const graph = semantics(documents);
|
||||
|
||||
const first = await probeTooltipNames(cdp, 11, documents, graph);
|
||||
expect(first.size).toBe(0);
|
||||
const hoversAfterFirst = vi
|
||||
.mocked(cdp.send)
|
||||
.mock.calls.filter(([, method]) => method === "Input.dispatchMouseEvent").length;
|
||||
expect(hoversAfterFirst).toBeGreaterThan(0);
|
||||
|
||||
const second = await probeTooltipNames(cdp, 11, documents, graph);
|
||||
expect(second.size).toBe(0);
|
||||
expect(
|
||||
vi.mocked(cdp.send).mock.calls.filter(([, method]) => method === "Input.dispatchMouseEvent")
|
||||
.length,
|
||||
).toBe(hoversAfterFirst);
|
||||
});
|
||||
|
||||
it("keeps tooltip evidence isolated across sibling frame sessions", async () => {
|
||||
let hoveredX = -10;
|
||||
const cdp: CdpRunner = {
|
||||
|
||||
@@ -9,7 +9,7 @@ import { evaluateHoverTrigger } from "@/lib/hover-trigger-policy";
|
||||
import { isOverlayHostNode, OVERLAY_HOST_SELECTOR } from "../../lib/overlay-bridge";
|
||||
import { childFrameProjection, type GeometryProjection, projectRectToViewport } from "../geometry";
|
||||
import type { CdpRunner } from "../shared";
|
||||
import { clearHover, waitForHover } from "./hover-perception";
|
||||
import { clearHover, ProbeBudget, waitForHover } from "./hover-perception";
|
||||
|
||||
const REQUESTED_STYLES = ["position", "pointer-events", "cursor", "visibility", "opacity"] as const;
|
||||
const STYLE_COL = Object.fromEntries(
|
||||
@@ -71,14 +71,11 @@ export interface CapturedViewModel {
|
||||
/** Explicit DOMSnapshot frame ancestry; never inferred from backend node ids. */
|
||||
frameParentIds?: Map<string, string>;
|
||||
rootFrameId?: string;
|
||||
surfaceProbes?: CapturedSurfaceProbe[];
|
||||
/** Backend node ids belonging to the agent overlay host + its shadow subtree. */
|
||||
excludedBackendNodeIds: Set<number>;
|
||||
}
|
||||
|
||||
export interface CaptureViewModelOptions {
|
||||
conditionalSurfaceProbe?: boolean;
|
||||
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
@@ -193,9 +190,9 @@ interface CapturedFormState {
|
||||
sensitive?: boolean;
|
||||
}
|
||||
|
||||
interface CapturedFrameFormState {
|
||||
controls: CapturedFormState[];
|
||||
childFrames: CapturedFrameFormState[];
|
||||
interface DeepSerializedValue {
|
||||
type: string;
|
||||
value?: unknown;
|
||||
}
|
||||
|
||||
function formStateBatchExpression(maxControls: number): string {
|
||||
@@ -218,43 +215,63 @@ function formStateBatchExpression(maxControls: number): string {
|
||||
...(sensitive ? {} : { value: rawValue, defaultValue }),
|
||||
};
|
||||
};
|
||||
const controls = [];
|
||||
const collect = (doc) => {
|
||||
const controls = [];
|
||||
for (const el of Array.from(doc.querySelectorAll(controlSelector))) {
|
||||
if (remaining <= 0) break;
|
||||
controls.push(controlState(el));
|
||||
// Deep serialization supplies the node's backend id. Keep the
|
||||
// state as JSON so decoding needs no general-purpose V8 deserializer.
|
||||
controls.push([el, JSON.stringify(controlState(el))]);
|
||||
remaining -= 1;
|
||||
}
|
||||
const childFrames = [];
|
||||
for (const frame of Array.from(doc.querySelectorAll("iframe"))) {
|
||||
if (remaining <= 0) break;
|
||||
let childDoc = null;
|
||||
try { childDoc = frame.contentDocument; } catch { childDoc = null; }
|
||||
childFrames.push(childDoc && remaining > 0 ? collect(childDoc) : { controls: [], childFrames: [] });
|
||||
if (childDoc) collect(childDoc);
|
||||
}
|
||||
return { controls, childFrames };
|
||||
};
|
||||
return collect(document);
|
||||
collect(document);
|
||||
return controls;
|
||||
})()`;
|
||||
}
|
||||
|
||||
function flattenFrameFormStates(
|
||||
frame: CapturedFrameFormState | undefined,
|
||||
out: CapturedFormState[][] = [],
|
||||
): CapturedFormState[][] {
|
||||
if (!frame) return out;
|
||||
out.push(Array.isArray(frame.controls) ? frame.controls : []);
|
||||
for (const child of Array.isArray(frame.childFrames) ? frame.childFrames : []) {
|
||||
flattenFrameFormStates(child, out);
|
||||
function formStatesByBackendId(result: RuntimeEvaluateReply): Map<number, CapturedFormState> {
|
||||
const states = new Map<number, CapturedFormState>();
|
||||
const serialized = result.result?.deepSerializedValue;
|
||||
if (serialized?.type !== "array" || !Array.isArray(serialized.value)) return states;
|
||||
for (const entry of serialized.value as DeepSerializedValue[]) {
|
||||
if (entry?.type !== "array" || !Array.isArray(entry.value)) continue;
|
||||
const [element, json] = entry.value as DeepSerializedValue[];
|
||||
if (element?.type !== "node" || json?.type !== "string" || typeof json.value !== "string") {
|
||||
continue;
|
||||
}
|
||||
const backendNodeId = (element.value as { backendNodeId?: number } | undefined)?.backendNodeId;
|
||||
if (typeof backendNodeId !== "number" || !Number.isSafeInteger(backendNodeId)) continue;
|
||||
try {
|
||||
const state = JSON.parse(json.value) as CapturedFormState | null;
|
||||
if (
|
||||
!state ||
|
||||
!["empty", "filled", "default"].includes(state.state ?? "") ||
|
||||
typeof state.sensitive !== "boolean" ||
|
||||
typeof state.placeholder !== "string" ||
|
||||
(!state.sensitive &&
|
||||
(typeof state.value !== "string" || typeof state.defaultValue !== "string"))
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
states.set(backendNodeId, state);
|
||||
} catch {
|
||||
// A malformed entry must not overwrite the snapshot's own state.
|
||||
}
|
||||
}
|
||||
return out;
|
||||
return states;
|
||||
}
|
||||
|
||||
function applyFormStates(nodes: CapturedNode[], states: CapturedFormState[]): void {
|
||||
let index = 0;
|
||||
function applyFormStates(nodes: CapturedNode[], states: Map<number, CapturedFormState>): void {
|
||||
for (const node of nodes) {
|
||||
if (!isFormControlTag(node.tag)) continue;
|
||||
const state = states[index];
|
||||
index += 1;
|
||||
const state = states.get(node.backendNodeId);
|
||||
if (!state) continue;
|
||||
node.formState = state.state;
|
||||
node.formPlaceholder = state.placeholder ?? "";
|
||||
@@ -279,26 +296,37 @@ async function enrichFormControlStates(
|
||||
nodes.some((node) => isFormControlTag(node.tag)),
|
||||
);
|
||||
if (!hasControls) return;
|
||||
const objectGroup = `bsk-vom-forms-${crypto.randomUUID()}`;
|
||||
try {
|
||||
throwIfAborted(signal);
|
||||
const result = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
|
||||
expression: formStateBatchExpression(MAX_FORM_ENRICH_CONTROLS),
|
||||
returnByValue: true,
|
||||
objectGroup,
|
||||
serializationOptions: {
|
||||
serialization: "deep",
|
||||
maxDepth: 3,
|
||||
additionalParameters: { maxNodeDepth: 0, includeShadowTree: "none" },
|
||||
},
|
||||
});
|
||||
throwIfAborted(signal);
|
||||
const frameStates = flattenFrameFormStates(runtimeValue<CapturedFrameFormState>(result));
|
||||
for (let i = 0; i < frameNodeGroups.length; i += 1) {
|
||||
applyFormStates(frameNodeGroups[i], frameStates[i] ?? []);
|
||||
// Backend ids are scoped to this capture's CDP target. OOPIF targets
|
||||
// use their own captureViewModel call and never share this lookup.
|
||||
const states = formStatesByBackendId(result);
|
||||
for (const nodes of frameNodeGroups) {
|
||||
applyFormStates(nodes, states);
|
||||
}
|
||||
} catch (error) {
|
||||
if (isAbortError(error)) throw error;
|
||||
// Best-effort enrichment. DOMSnapshot/AX data still carries the nodes.
|
||||
} finally {
|
||||
await cdp.send(tabId, "Runtime.releaseObjectGroup", { objectGroup }).catch(() => undefined);
|
||||
}
|
||||
}
|
||||
|
||||
interface RuntimeEvaluateReply {
|
||||
result?: {
|
||||
value?: unknown;
|
||||
deepSerializedValue?: DeepSerializedValue;
|
||||
};
|
||||
}
|
||||
|
||||
@@ -368,7 +396,16 @@ function collectBackendIdsFromDomNode(node: CdpDomNode | undefined, out: Set<num
|
||||
}
|
||||
}
|
||||
|
||||
const MAX_HOVER_PROBE_MS = 2_000;
|
||||
/**
|
||||
* Ceiling for the whole hover-surface phase.
|
||||
*
|
||||
* The previous 2000 was only a floor: the loop checked elapsed time at the top,
|
||||
* so a candidate could start with 1ms left and still run its two settle
|
||||
* windows, pushing real cost to ~2.6s. This is that true ceiling, now actually
|
||||
* enforced by an up-front affordability check, so the same number of candidates
|
||||
* get probed under a bound that no longer lies.
|
||||
*/
|
||||
const MAX_HOVER_PROBE_MS = 2_600;
|
||||
const MAX_HOVER_TRIGGERS = 6;
|
||||
const MAX_HOVER_SURFACES = 3;
|
||||
const HOVER_SETTLE_MS = 300;
|
||||
@@ -614,13 +651,35 @@ function confidenceForHover(
|
||||
return "low";
|
||||
}
|
||||
|
||||
async function probeHoverSurfaces(
|
||||
/**
|
||||
* One candidate costs two settle windows (baseline + post-hover) plus a few
|
||||
* CDP round trips. Used to decide whether the next candidate still fits the
|
||||
* budget before paying for it.
|
||||
*/
|
||||
const HOVER_CANDIDATE_COST_MS = HOVER_SETTLE_MS * 2;
|
||||
|
||||
export interface HoverSurfaceProbeOptions {
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
/**
|
||||
* Hovers a bounded set of likely menu triggers and reports the sub-items each
|
||||
* one reveals.
|
||||
*
|
||||
* Runs after DOM *and* accessibility capture. Hovering can open menus and
|
||||
* change layout, so probing between the two captures would leave the DOM half
|
||||
* of an observation describing the page before the change and the AX half
|
||||
* describing it after.
|
||||
*
|
||||
* The caller owns the overlay bypass span (see `withOverlayBypass`).
|
||||
*/
|
||||
export async function probeHoverSurfaces(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
nodes: CapturedNode[],
|
||||
options: CaptureViewModelOptions,
|
||||
options: HoverSurfaceProbeOptions = {},
|
||||
): Promise<CapturedSurfaceProbe[]> {
|
||||
const started = Date.now();
|
||||
const budget = new ProbeBudget(MAX_HOVER_PROBE_MS);
|
||||
try {
|
||||
throwIfAborted(options.signal);
|
||||
const cssScan = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
|
||||
@@ -635,60 +694,54 @@ async function probeHoverSurfaces(
|
||||
const results: CapturedSurfaceProbe[] = [];
|
||||
const seen = new Set<number>();
|
||||
throwIfAborted(options.signal);
|
||||
await options.hoverProbeBypassOverlay?.(tabId, true).catch(() => undefined);
|
||||
try {
|
||||
for (const candidate of candidates.slice(0, MAX_HOVER_TRIGGERS)) {
|
||||
throwIfAborted(options.signal);
|
||||
for (const candidate of candidates.slice(0, MAX_HOVER_TRIGGERS)) {
|
||||
if (!budget.canAfford(HOVER_CANDIDATE_COST_MS)) break;
|
||||
if (results.length >= MAX_HOVER_SURFACES) break;
|
||||
if (seen.has(candidate.backendNodeId)) continue;
|
||||
try {
|
||||
await clearHover(cdp, tabId);
|
||||
throwIfAborted(options.signal);
|
||||
if (Date.now() - started > MAX_HOVER_PROBE_MS) break;
|
||||
if (results.length >= MAX_HOVER_SURFACES) break;
|
||||
if (seen.has(candidate.backendNodeId)) continue;
|
||||
try {
|
||||
await clearHover(cdp, tabId);
|
||||
throwIfAborted(options.signal);
|
||||
await waitForHover(HOVER_SETTLE_MS, options.signal);
|
||||
const baselineReply = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
|
||||
expression: hoverStateExpression(),
|
||||
returnByValue: true,
|
||||
});
|
||||
throwIfAborted(options.signal);
|
||||
const baselineItems = runtimeValue<HoverRuntimeItem[]>(baselineReply) ?? [];
|
||||
await waitForHover(HOVER_SETTLE_MS, options.signal);
|
||||
const baselineReply = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
|
||||
expression: hoverStateExpression(),
|
||||
returnByValue: true,
|
||||
});
|
||||
throwIfAborted(options.signal);
|
||||
const baselineItems = runtimeValue<HoverRuntimeItem[]>(baselineReply) ?? [];
|
||||
|
||||
throwIfAborted(options.signal);
|
||||
await cdp.send(tabId, "Input.dispatchMouseEvent", {
|
||||
type: "mouseMoved",
|
||||
x: candidate.x,
|
||||
y: candidate.y,
|
||||
});
|
||||
throwIfAborted(options.signal);
|
||||
await waitForHover(HOVER_SETTLE_MS, options.signal);
|
||||
const collected = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
|
||||
expression: hoverStateExpression(),
|
||||
returnByValue: true,
|
||||
});
|
||||
throwIfAborted(options.signal);
|
||||
const subItems = diffHoverItems(
|
||||
baselineItems,
|
||||
runtimeValue<HoverRuntimeItem[]>(collected) ?? [],
|
||||
);
|
||||
if (subItems.length === 0) continue;
|
||||
seen.add(candidate.backendNodeId);
|
||||
results.push({
|
||||
triggerBackendNodeId: candidate.backendNodeId,
|
||||
triggerPoint: { x: candidate.x, y: candidate.y },
|
||||
triggerAction: "hover",
|
||||
subItems,
|
||||
confidence: confidenceForHover(candidate, subItems),
|
||||
});
|
||||
} catch (error) {
|
||||
if (isAbortError(error)) throw error;
|
||||
continue;
|
||||
} finally {
|
||||
await clearHover(cdp, tabId);
|
||||
}
|
||||
throwIfAborted(options.signal);
|
||||
await cdp.send(tabId, "Input.dispatchMouseEvent", {
|
||||
type: "mouseMoved",
|
||||
x: candidate.x,
|
||||
y: candidate.y,
|
||||
});
|
||||
throwIfAborted(options.signal);
|
||||
await waitForHover(HOVER_SETTLE_MS, options.signal);
|
||||
const collected = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
|
||||
expression: hoverStateExpression(),
|
||||
returnByValue: true,
|
||||
});
|
||||
throwIfAborted(options.signal);
|
||||
const subItems = diffHoverItems(
|
||||
baselineItems,
|
||||
runtimeValue<HoverRuntimeItem[]>(collected) ?? [],
|
||||
);
|
||||
if (subItems.length === 0) continue;
|
||||
seen.add(candidate.backendNodeId);
|
||||
results.push({
|
||||
triggerBackendNodeId: candidate.backendNodeId,
|
||||
triggerPoint: { x: candidate.x, y: candidate.y },
|
||||
triggerAction: "hover",
|
||||
subItems,
|
||||
confidence: confidenceForHover(candidate, subItems),
|
||||
});
|
||||
} catch (error) {
|
||||
if (isAbortError(error)) throw error;
|
||||
continue;
|
||||
} finally {
|
||||
await clearHover(cdp, tabId);
|
||||
}
|
||||
} finally {
|
||||
await options.hoverProbeBypassOverlay?.(tabId, false).catch(() => undefined);
|
||||
}
|
||||
return results;
|
||||
} catch (err) {
|
||||
@@ -1042,7 +1095,6 @@ export async function captureViewModel(
|
||||
nodes: [],
|
||||
viewport,
|
||||
iframeNodes: new Map(),
|
||||
surfaceProbes: [],
|
||||
excludedBackendNodeIds: new Set(),
|
||||
};
|
||||
}
|
||||
@@ -1071,11 +1123,6 @@ export async function captureViewModel(
|
||||
await enrichFormControlStates(cdp, tabId, [nodes, ...iframeNodes.values()], options.signal);
|
||||
throwIfAborted(options.signal);
|
||||
|
||||
const surfaceProbes = options.conditionalSurfaceProbe
|
||||
? await probeHoverSurfaces(cdp, tabId, nodes, options)
|
||||
: [];
|
||||
throwIfAborted(options.signal);
|
||||
|
||||
const frameNodes = new Map<string, CapturedNode[]>();
|
||||
if (topContext.frameId) frameNodes.set(topContext.frameId, nodes);
|
||||
for (const iframeFrameNodes of iframeNodes.values()) {
|
||||
@@ -1096,7 +1143,6 @@ export async function captureViewModel(
|
||||
frameOwnerBackendNodeIds: frameParsed.frameOwnerBackendNodeIds,
|
||||
frameParentIds: frameParsed.frameParentIds,
|
||||
...(topContext.frameId ? { rootFrameId: topContext.frameId } : {}),
|
||||
surfaceProbes,
|
||||
excludedBackendNodeIds,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -35,3 +35,91 @@ export async function clearHover(cdp: CdpRunner, tabId: number): Promise<void> {
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Wall-clock budget for a probe phase.
|
||||
*
|
||||
* Callers ask whether the *next* candidate still fits before starting it,
|
||||
* rather than checking elapsed time at the top of the loop. A top-of-loop
|
||||
* check lets a candidate start with 1ms of budget left and then run to
|
||||
* completion, overshooting by a full settle window.
|
||||
*/
|
||||
export class ProbeBudget {
|
||||
private readonly startedAt = Date.now();
|
||||
|
||||
constructor(private readonly totalMs: number) {}
|
||||
|
||||
get elapsedMs(): number {
|
||||
return Date.now() - this.startedAt;
|
||||
}
|
||||
|
||||
/** True when a candidate costing `estimatedMs` still fits in the budget. */
|
||||
canAfford(estimatedMs: number): boolean {
|
||||
return this.elapsedMs + estimatedMs <= this.totalMs;
|
||||
}
|
||||
}
|
||||
|
||||
export type OverlayBypass = (tabId: number, enabled: boolean) => Promise<void>;
|
||||
|
||||
export interface OverlayBypassScopeOptions {
|
||||
/**
|
||||
* Called when the overlay could not be restored. Defaults to `console.error`
|
||||
* so a stuck-bypassed overlay is never silent; tests inject a spy.
|
||||
*/
|
||||
onRestoreFailure?: (error: unknown) => void;
|
||||
}
|
||||
|
||||
const overlayBypassDepth = new Map<number, number>();
|
||||
|
||||
/**
|
||||
* Runs `body` with the agent overlay bypassed, restoring it afterwards.
|
||||
*
|
||||
* Reference counted per tab so nested probe phases share a single bypass span
|
||||
* instead of toggling the overlay once per phase. Restore failures are
|
||||
* reported rather than swallowed, and the depth is released even when restore
|
||||
* throws so a failure cannot pin the overlay off for the tab's lifetime.
|
||||
*/
|
||||
export async function withOverlayBypass<T>(
|
||||
bypass: OverlayBypass | undefined,
|
||||
tabId: number,
|
||||
body: () => Promise<T>,
|
||||
options: OverlayBypassScopeOptions = {},
|
||||
): Promise<T> {
|
||||
if (!bypass) return body();
|
||||
|
||||
const depth = overlayBypassDepth.get(tabId) ?? 0;
|
||||
overlayBypassDepth.set(tabId, depth + 1);
|
||||
if (depth === 0) {
|
||||
try {
|
||||
await bypass(tabId, true);
|
||||
} catch (error) {
|
||||
// Failing to hide the overlay only risks probing our own UI, so continue.
|
||||
console.debug("[bsk hover] overlay bypass could not be enabled", error);
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
return await body();
|
||||
} finally {
|
||||
const current = overlayBypassDepth.get(tabId) ?? 1;
|
||||
if (current <= 1) {
|
||||
overlayBypassDepth.delete(tabId);
|
||||
try {
|
||||
await bypass(tabId, false);
|
||||
} catch (error) {
|
||||
(options.onRestoreFailure ?? defaultRestoreFailure)(error);
|
||||
}
|
||||
} else {
|
||||
overlayBypassDepth.set(tabId, current - 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function defaultRestoreFailure(error: unknown): void {
|
||||
console.error("[bsk hover] agent overlay could not be restored after probing", error);
|
||||
}
|
||||
|
||||
/** Test seam: drops any bypass depth left behind by a crashed probe. */
|
||||
export function resetOverlayBypassDepth(): void {
|
||||
overlayBypassDepth.clear();
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { isVomReferenceNode } from "@browser-skill/vom";
|
||||
import { type CdpRunner, sendToCdpTarget } from "../shared";
|
||||
import type { FrameDocument, FrameOwnedAxNode } from "./frame-document";
|
||||
import { clearHover, waitForHover } from "./hover-perception";
|
||||
import { clearHover, ProbeBudget, waitForHover } from "./hover-perception";
|
||||
import { stableIdentifierName } from "./semantic-graph/name-evidence";
|
||||
import { frameBackendKey, type ResolvedSemanticGraph } from "./semantic-graph/types";
|
||||
|
||||
@@ -9,11 +9,18 @@ const MAX_TOOLTIP_CANDIDATES = 8;
|
||||
const MAX_TOOLTIP_PROBE_MS = 2_400;
|
||||
const TOOLTIP_SETTLE_MS = 250;
|
||||
const CACHE_TTL_MS = 5 * 60_000;
|
||||
/**
|
||||
* Misses expire sooner than hits: a control that gains a tooltip once the app
|
||||
* finishes booting should still be discoverable, but not at the cost of
|
||||
* re-probing every unlabelled control on every observation.
|
||||
*/
|
||||
const MISS_CACHE_TTL_MS = 60_000;
|
||||
const MAX_CACHE_ENTRIES = 500;
|
||||
const FORM_CONTROL_TAGS = new Set(["input", "select", "textarea"]);
|
||||
|
||||
interface TooltipCacheEntry {
|
||||
name: string;
|
||||
/** `null` records "probed, revealed no tooltip" so the miss is not re-paid. */
|
||||
name: string | null;
|
||||
expiresAt: number;
|
||||
}
|
||||
|
||||
@@ -27,7 +34,6 @@ interface TooltipCandidate<T extends FrameOwnedAxNode> {
|
||||
|
||||
export interface TooltipNameProbeOptions {
|
||||
signal?: AbortSignal;
|
||||
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
|
||||
}
|
||||
|
||||
const tooltipCache = new Map<string, TooltipCacheEntry>();
|
||||
@@ -39,10 +45,11 @@ function cacheKey<T extends FrameOwnedAxNode>(
|
||||
return `${document.target.tabId}\u0000${document.frameId}\u0000${document.url ?? ""}\u0000${backendNodeId}`;
|
||||
}
|
||||
|
||||
function cachedName<T extends FrameOwnedAxNode>(
|
||||
/** Returns the cache entry, or `undefined` when this control was never probed. */
|
||||
function cachedEntry<T extends FrameOwnedAxNode>(
|
||||
document: FrameDocument<T>,
|
||||
backendNodeId: number,
|
||||
): string | undefined {
|
||||
): TooltipCacheEntry | undefined {
|
||||
const key = cacheKey(document, backendNodeId);
|
||||
const entry = tooltipCache.get(key);
|
||||
if (!entry) return undefined;
|
||||
@@ -50,13 +57,13 @@ function cachedName<T extends FrameOwnedAxNode>(
|
||||
tooltipCache.delete(key);
|
||||
return undefined;
|
||||
}
|
||||
return entry.name;
|
||||
return entry;
|
||||
}
|
||||
|
||||
function storeName<T extends FrameOwnedAxNode>(
|
||||
document: FrameDocument<T>,
|
||||
backendNodeId: number,
|
||||
name: string,
|
||||
name: string | null,
|
||||
): void {
|
||||
if (tooltipCache.size >= MAX_CACHE_ENTRIES) {
|
||||
const oldest = tooltipCache.keys().next().value;
|
||||
@@ -64,7 +71,7 @@ function storeName<T extends FrameOwnedAxNode>(
|
||||
}
|
||||
tooltipCache.set(cacheKey(document, backendNodeId), {
|
||||
name,
|
||||
expiresAt: Date.now() + CACHE_TTL_MS,
|
||||
expiresAt: Date.now() + (name === null ? MISS_CACHE_TTL_MS : CACHE_TTL_MS),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -237,22 +244,27 @@ export async function probeTooltipNames<T extends FrameOwnedAxNode>(
|
||||
const names = new Map<string, string>();
|
||||
const pending: TooltipCandidate<T>[] = [];
|
||||
for (const candidate of candidates(documents, graph)) {
|
||||
const cached = cachedName(candidate.document, candidate.backendNodeId);
|
||||
const cached = cachedEntry(candidate.document, candidate.backendNodeId);
|
||||
if (cached) {
|
||||
names.set(frameBackendKey(candidate.document.frameId, candidate.backendNodeId), cached);
|
||||
// A cached miss is still a decision: skip the control instead of re-probing.
|
||||
if (cached.name !== null) {
|
||||
names.set(
|
||||
frameBackendKey(candidate.document.frameId, candidate.backendNodeId),
|
||||
cached.name,
|
||||
);
|
||||
}
|
||||
} else if (pending.length < MAX_TOOLTIP_CANDIDATES) {
|
||||
pending.push(candidate);
|
||||
}
|
||||
}
|
||||
if (pending.length === 0) return names;
|
||||
|
||||
const started = Date.now();
|
||||
const budget = new ProbeBudget(MAX_TOOLTIP_PROBE_MS);
|
||||
const contexts = new Map<string, number>();
|
||||
await options.hoverProbeBypassOverlay?.(tabId, true).catch(() => undefined);
|
||||
try {
|
||||
for (const candidate of pending) {
|
||||
if (options.signal?.aborted) throw new DOMException("observation aborted", "AbortError");
|
||||
if (Date.now() - started >= MAX_TOOLTIP_PROBE_MS) break;
|
||||
if (!budget.canAfford(TOOLTIP_SETTLE_MS)) break;
|
||||
let objectId: string | undefined;
|
||||
try {
|
||||
let contextId = contexts.get(candidate.document.frameId);
|
||||
@@ -278,9 +290,9 @@ export async function probeTooltipNames<T extends FrameOwnedAxNode>(
|
||||
await waitForHover(TOOLTIP_SETTLE_MS, options.signal);
|
||||
const after = await tooltipTexts(cdp, candidate.document, objectId);
|
||||
const name = cleanTooltipDiff(before, after);
|
||||
storeName(candidate.document, candidate.backendNodeId, name ?? null);
|
||||
if (!name) continue;
|
||||
names.set(frameBackendKey(candidate.document.frameId, candidate.backendNodeId), name);
|
||||
storeName(candidate.document, candidate.backendNodeId, name);
|
||||
} catch (error) {
|
||||
if (error instanceof Error && error.name === "AbortError") throw error;
|
||||
} finally {
|
||||
@@ -290,7 +302,6 @@ export async function probeTooltipNames<T extends FrameOwnedAxNode>(
|
||||
}
|
||||
} finally {
|
||||
await clearHover(cdp, tabId);
|
||||
await options.hoverProbeBypassOverlay?.(tabId, false).catch(() => undefined);
|
||||
}
|
||||
return names;
|
||||
}
|
||||
|
||||
@@ -37,6 +37,17 @@ export interface CaptureVomObservationResult {
|
||||
frames: CaptureVomFrame[];
|
||||
matchNodes: CaptureVomMatchNode[];
|
||||
surfaceProbes?: CapturedSurfaceProbe[];
|
||||
/**
|
||||
* Whether this observation actively hovered the page, and whether that
|
||||
* revealed content the static tree does not contain. Lets callers judge how
|
||||
* far the page may have drifted from the returned snapshot.
|
||||
*/
|
||||
hoverProbe?: HoverProbeReport;
|
||||
}
|
||||
|
||||
export interface HoverProbeReport {
|
||||
performed: boolean;
|
||||
revealedContent: boolean;
|
||||
}
|
||||
|
||||
function projectFrames(documents: CapturedFrameDocument<FrameAxNode>[]): CaptureVomFrame[] {
|
||||
@@ -75,6 +86,7 @@ export function projectRecordSafeObservation(input: {
|
||||
frameDocuments: CapturedFrameDocument<FrameAxNode>[];
|
||||
rendered: VomResult;
|
||||
surfaceProbes?: CapturedSurfaceProbe[];
|
||||
hoverProbe?: HoverProbeReport;
|
||||
}): CaptureVomObservationResult {
|
||||
return {
|
||||
text: input.rendered.text,
|
||||
@@ -84,5 +96,6 @@ export function projectRecordSafeObservation(input: {
|
||||
frames: projectFrames(input.frameDocuments),
|
||||
matchNodes: projectMatchNodes(input.frameDocuments),
|
||||
surfaceProbes: input.surfaceProbes,
|
||||
...(input.hoverProbe ? { hoverProbe: input.hoverProbe } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -24,20 +24,40 @@ export type RpcErrorReason =
|
||||
| "agent_window_scope"
|
||||
| "element_not_visible"
|
||||
| "ref_not_found"
|
||||
| "invalid_selector"
|
||||
| "selector_not_found"
|
||||
| "target_not_fillable"
|
||||
| "fill_value_invalid"
|
||||
| "fill_target_changed"
|
||||
| "fill_focus_lost"
|
||||
| "fill_value_mismatch"
|
||||
| "fill_failed"
|
||||
| "target_not_select"
|
||||
| "option_not_found"
|
||||
| "single_select_value_count"
|
||||
| "tab_not_active"
|
||||
| "restricted_tab_url"
|
||||
| "cdp_extension_access_denied"
|
||||
| "borrow_conflict"
|
||||
| "screenshot_capture_failed"
|
||||
| "file_input_probe_failed"
|
||||
| "file_input_not_activated"
|
||||
| "set_file_input_failed"
|
||||
| "upload_mechanism_unsupported"
|
||||
| "file_drop_target_unavailable"
|
||||
| "file_drop_failed"
|
||||
| "download_capture_failed"
|
||||
| "transfer_outcome_unknown"
|
||||
| "transfer_timeout"
|
||||
| "cleanup_failed";
|
||||
|
||||
export type TransferEffectState = "none" | "committed" | "unknown";
|
||||
export type TransferCleanupState = "complete" | "failed";
|
||||
|
||||
export interface RpcErrorData {
|
||||
reason?: RpcErrorReason;
|
||||
effect_state?: TransferEffectState;
|
||||
phase?: string;
|
||||
cleanup_state?: TransferCleanupState;
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
@@ -319,9 +339,14 @@ export interface SnapshotResult {
|
||||
|
||||
export interface ObserveParams extends SnapshotParams {
|
||||
debug_surfaces?: boolean;
|
||||
probe_hover?: boolean;
|
||||
}
|
||||
|
||||
export interface ObserveResult extends SnapshotResult {
|
||||
hover_probe?: {
|
||||
performed: boolean;
|
||||
revealed_content: boolean;
|
||||
};
|
||||
debug?: {
|
||||
surface_probes?: Array<{
|
||||
trigger_backend_node_id: number;
|
||||
@@ -501,6 +526,53 @@ export interface SelectResult {
|
||||
dialogs?: JavaScriptDialogInfo[];
|
||||
}
|
||||
|
||||
export interface UploadFile {
|
||||
transfer_id: string;
|
||||
name: string;
|
||||
staged_path?: string;
|
||||
}
|
||||
|
||||
export type UploadMode = "input" | "drop";
|
||||
|
||||
export interface UploadParams {
|
||||
session_id: string;
|
||||
ref?: string;
|
||||
selector?: string;
|
||||
tab_id?: number;
|
||||
files: UploadFile[];
|
||||
mode?: UploadMode;
|
||||
timeout_ms?: number;
|
||||
}
|
||||
|
||||
export interface UploadResult {
|
||||
tab_id: number;
|
||||
used_ref?: string;
|
||||
used_selector?: string;
|
||||
file_names: string[];
|
||||
}
|
||||
|
||||
export interface DownloadParams {
|
||||
session_id: string;
|
||||
ref?: string;
|
||||
selector?: string;
|
||||
tab_id?: number;
|
||||
timeout_ms?: number;
|
||||
browser_relative_dir?: string;
|
||||
max_byte_size?: number;
|
||||
}
|
||||
|
||||
export interface DownloadResult {
|
||||
tab_id: number;
|
||||
used_ref?: string;
|
||||
used_selector?: string;
|
||||
suggested_filename: string;
|
||||
byte_size: number;
|
||||
mime?: string;
|
||||
danger?: string;
|
||||
browser_path?: string;
|
||||
transfer_id?: string;
|
||||
}
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
// M9 tool payloads — evaluate / wait_for_navigation / wait_ms
|
||||
// --------------------------------------------------------------------------
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
import type { DiagnosticSink } from "@/lib/diagnostics";
|
||||
import type { ConnectionStateHandler, Disposable, FrameHandler, Transport } from "./transport";
|
||||
import type { ConnectionState, ProtocolFrame } from "./types";
|
||||
|
||||
@@ -19,7 +18,6 @@ export interface WSTransportOptions {
|
||||
* Inject a fake WebSocket constructor in tests; defaults to `globalThis.WebSocket`.
|
||||
*/
|
||||
webSocketFactory?: WebSocketFactory;
|
||||
diagnostics?: DiagnosticSink;
|
||||
}
|
||||
|
||||
interface CloseLikeEvent {
|
||||
@@ -46,7 +44,6 @@ export class WSTransport implements Transport {
|
||||
private readonly factory: WebSocketFactory;
|
||||
private readonly initialDelayMs: number;
|
||||
private readonly maxDelayMs: number;
|
||||
private readonly diagnostics?: DiagnosticSink;
|
||||
|
||||
private socket: WebSocket | null = null;
|
||||
private currentState: ConnectionState = "disconnected";
|
||||
@@ -66,7 +63,6 @@ export class WSTransport implements Transport {
|
||||
this.factory = options.webSocketFactory ?? ((url: string) => new WebSocket(url));
|
||||
this.initialDelayMs = options.reconnect?.initialDelayMs ?? DEFAULT_INITIAL_DELAY_MS;
|
||||
this.maxDelayMs = options.reconnect?.maxDelayMs ?? DEFAULT_MAX_DELAY_MS;
|
||||
this.diagnostics = options.diagnostics;
|
||||
}
|
||||
|
||||
get state(): ConnectionState {
|
||||
@@ -74,13 +70,6 @@ export class WSTransport implements Transport {
|
||||
}
|
||||
|
||||
connect(): Promise<void> {
|
||||
this.diagnostics?.("transport.connect.requested", {
|
||||
state: this.currentState,
|
||||
socket_generation: this.socketGeneration,
|
||||
has_socket: this.socket !== null,
|
||||
has_connecting_promise: this.connectingPromise !== null,
|
||||
has_reconnect_timer: this.reconnectTimer !== null,
|
||||
});
|
||||
if (this.currentState === "connected") return Promise.resolve();
|
||||
|
||||
this.explicitlyClosed = false;
|
||||
@@ -105,11 +94,6 @@ export class WSTransport implements Transport {
|
||||
}
|
||||
|
||||
async disconnect(): Promise<void> {
|
||||
this.diagnostics?.("transport.disconnect.requested", {
|
||||
state: this.currentState,
|
||||
socket_generation: this.socketGeneration,
|
||||
has_socket: this.socket !== null,
|
||||
});
|
||||
this.explicitlyClosed = true;
|
||||
if (this.reconnectTimer) {
|
||||
clearTimeout(this.reconnectTimer);
|
||||
@@ -140,12 +124,6 @@ export class WSTransport implements Transport {
|
||||
|
||||
send(msg: ProtocolFrame): void {
|
||||
if (!this.socket || this.socket.readyState !== 1 /* OPEN */) {
|
||||
this.diagnostics?.("transport.send.rejected", {
|
||||
state: this.currentState,
|
||||
socket_generation: this.socketGeneration,
|
||||
socket_ready_state: this.socket?.readyState ?? null,
|
||||
frame_kind: "method" in msg ? msg.method : "event" in msg ? msg.event : "response",
|
||||
});
|
||||
throw new Error("[WSTransport] cannot send while not connected");
|
||||
}
|
||||
this.socket.send(JSON.stringify(msg));
|
||||
@@ -174,16 +152,10 @@ export class WSTransport implements Transport {
|
||||
const generation = ++this.socketGeneration;
|
||||
const socket = this.factory(this.url);
|
||||
this.socket = socket;
|
||||
this.diagnostics?.("transport.socket.created", {
|
||||
socket_generation: generation,
|
||||
reconnect_attempt: this.reconnectAttempt,
|
||||
url: this.url,
|
||||
});
|
||||
|
||||
socket.addEventListener("open", () => {
|
||||
if (!this.isCurrentSocket(socket, generation)) return;
|
||||
this.reconnectAttempt = 0;
|
||||
this.diagnostics?.("transport.socket.open", { socket_generation: generation });
|
||||
this.setState("connected");
|
||||
const resolve = this.resolveConnect;
|
||||
this.resolveConnect = null;
|
||||
@@ -202,7 +174,6 @@ export class WSTransport implements Transport {
|
||||
});
|
||||
|
||||
socket.addEventListener("error", () => {
|
||||
this.diagnostics?.("transport.socket.error", { socket_generation: generation });
|
||||
// The 'close' handler always fires after 'error' in browsers, so we
|
||||
// just record the error and let close drive reconnect logic.
|
||||
});
|
||||
@@ -226,16 +197,7 @@ export class WSTransport implements Transport {
|
||||
}
|
||||
|
||||
private handleClose(socket: WebSocket, generation: number, _ev: CloseLikeEvent): void {
|
||||
const isCurrent = this.isCurrentSocket(socket, generation);
|
||||
this.diagnostics?.("transport.socket.close", {
|
||||
socket_generation: generation,
|
||||
current_socket_generation: this.socketGeneration,
|
||||
is_current: isCurrent,
|
||||
explicitly_closed: this.explicitlyClosed,
|
||||
close_code: _ev.code ?? null,
|
||||
close_reason: _ev.reason ?? "",
|
||||
});
|
||||
if (!isCurrent) return;
|
||||
if (!this.isCurrentSocket(socket, generation)) return;
|
||||
this.socket = null;
|
||||
if (this.explicitlyClosed) {
|
||||
this.setState("disconnected");
|
||||
@@ -249,18 +211,9 @@ export class WSTransport implements Transport {
|
||||
if (this.reconnectTimer) return;
|
||||
const delay = Math.min(this.initialDelayMs * 2 ** this.reconnectAttempt, this.maxDelayMs);
|
||||
this.reconnectAttempt += 1;
|
||||
this.diagnostics?.("transport.reconnect.scheduled", {
|
||||
delay_ms: delay,
|
||||
reconnect_attempt: this.reconnectAttempt,
|
||||
socket_generation: this.socketGeneration,
|
||||
});
|
||||
this.reconnectTimer = setTimeout(() => {
|
||||
this.reconnectTimer = null;
|
||||
if (this.explicitlyClosed) return;
|
||||
this.diagnostics?.("transport.reconnect.timer_fired", {
|
||||
reconnect_attempt: this.reconnectAttempt,
|
||||
socket_generation: this.socketGeneration,
|
||||
});
|
||||
void this.connect().catch((err) => {
|
||||
console.debug("[WSTransport] reconnect attempt failed", err);
|
||||
});
|
||||
@@ -273,13 +226,7 @@ export class WSTransport implements Transport {
|
||||
|
||||
private setState(next: ConnectionState): void {
|
||||
if (this.currentState === next) return;
|
||||
const previous = this.currentState;
|
||||
this.currentState = next;
|
||||
this.diagnostics?.("transport.state.changed", {
|
||||
previous,
|
||||
next,
|
||||
socket_generation: this.socketGeneration,
|
||||
});
|
||||
for (const h of this.stateHandlers) {
|
||||
try {
|
||||
h(next);
|
||||
|
||||
@@ -22,8 +22,7 @@ export default defineConfig({
|
||||
outDir: "dist",
|
||||
modules: ["@wxt-dev/module-react"],
|
||||
manifest: {
|
||||
name: "BrowserSkill (Lumi Diagnostics)",
|
||||
version_name: `${EXTENSION_VERSION}-lumi-session-loss-v2`,
|
||||
name: "BrowserSkill",
|
||||
description:
|
||||
"Let AI agents use your logged-in browser in a separate Agent Window—without interrupting your work. Powered by the bsk CLI.",
|
||||
// Flat debugger sessions are required to address out-of-process iframes.
|
||||
@@ -31,6 +30,7 @@ export default defineConfig({
|
||||
permissions: [
|
||||
"alarms",
|
||||
"debugger",
|
||||
"downloads",
|
||||
"idle",
|
||||
"notifications",
|
||||
"tabs",
|
||||
@@ -46,7 +46,7 @@ export default defineConfig({
|
||||
128: "icon/logo.png",
|
||||
},
|
||||
action: {
|
||||
default_title: "BrowserSkill (Lumi Diagnostics)",
|
||||
default_title: "BrowserSkill",
|
||||
default_icon: {
|
||||
16: "icon/logo.png",
|
||||
32: "icon/logo.png",
|
||||
|
||||
@@ -53,6 +53,7 @@ flate2 = { workspace = true }
|
||||
tar = { workspace = true }
|
||||
zip = { workspace = true }
|
||||
sha2 = { workspace = true }
|
||||
base64 = { workspace = true }
|
||||
|
||||
[target.'cfg(unix)'.dependencies]
|
||||
nix = { workspace = true }
|
||||
@@ -63,6 +64,7 @@ windows-sys = { version = "0.59", features = [
|
||||
"Win32_Foundation",
|
||||
"Win32_System_Threading",
|
||||
"Win32_Security",
|
||||
"Win32_Storage_FileSystem",
|
||||
] }
|
||||
|
||||
[dev-dependencies]
|
||||
|
||||
+132
-275
@@ -1,327 +1,184 @@
|
||||
---
|
||||
name: browser-skill
|
||||
description: |
|
||||
Use when the user asks to perform browser automation tasks against their
|
||||
logged-in browser: visit and read pages, fill forms, scrape data, click
|
||||
through a flow, regression-test a PR's UI, validate a deployed page.
|
||||
Requires the bsk CLI installed and the browser-skill extension loaded.
|
||||
Use when the user asks to automate their logged-in Chromium browser: visit
|
||||
and read pages, fill forms, scrape data, click through flows, regression-test
|
||||
a PR's UI, validate a deployed page, or operate a tab they identify. Requires
|
||||
the bsk CLI and browser extension.
|
||||
---
|
||||
|
||||
# browser-skill
|
||||
|
||||
Drive the user's **real Chromium browser** (with their logins and cookies) through the `bsk` CLI. The extension opens an isolated **Agent Window** for automation; the user's normal windows stay protected unless you explicitly borrow a tab.
|
||||
Drive the user's real Chromium browser through `bsk`. Automation runs in an isolated **Agent
|
||||
Window** with the user's existing logins and cookies. User-window tabs remain protected unless they
|
||||
are explicitly borrowed.
|
||||
|
||||
## When to use
|
||||
Do not use this skill for tasks with no browser, for extension installation, or when the user only
|
||||
wants instructions. Never extract credentials, cookies, tokens, or other secrets from pages.
|
||||
|
||||
- Open pages, read titles/text, scrape structured data from sites the user can already access
|
||||
- Fill forms, click through multi-step flows, smoke-test a UI change
|
||||
- Understand pages with `bsk snapshot` first; use `bsk get-html` or `bsk screenshot` only when the snapshot is insufficient
|
||||
- Operate on a specific user tab they point you at (after `bsk tab borrow`)
|
||||
## Required lifecycle
|
||||
|
||||
## When NOT to use
|
||||
Every browser task owns a bounded session:
|
||||
|
||||
- Tasks with **no browser** involved (files, APIs, databases only)
|
||||
- Installing or configuring the extension (point the user to setup docs instead)
|
||||
- **Credential harvesting** — never run `bsk evaluate` on banking, SSO, or password-manager pages to extract tokens, cookies, or secrets
|
||||
- Long-lived control of a user's personal login window — borrow only for the immediate step, then `bsk tab return` or end the session
|
||||
- Replacing the user's manual browsing when they only wanted an explanation
|
||||
|
||||
## Prerequisites
|
||||
|
||||
1. `bsk` on `PATH` (Rust CLI from browser-skill)
|
||||
2. browser-skill **extension** loaded in Chromium and connected (popup shows green, or an upgrade reminder — still connected; continue)
|
||||
3. Any `bsk` command auto-starts background services as needed; use `bsk doctor` if anything fails
|
||||
|
||||
## Mandatory workflow
|
||||
|
||||
Every automation task **must** follow this lifecycle. Do **not** rely on idle timeouts (default session idle is 5 minutes).
|
||||
|
||||
```
|
||||
1. bsk session start → capture the 4-letter session id printed on stdout
|
||||
2. … every tool command … → always pass --session <id>
|
||||
3. bsk session stop <id> → REQUIRED when done (even on error paths)
|
||||
```text
|
||||
1. bsk session start # retain the printed 4-letter session id
|
||||
2. bsk ... --session <id> # pass it to every session-scoped command
|
||||
3. bsk session stop <id> # always run on success and error paths
|
||||
```
|
||||
|
||||
Optional: `bsk session start --browser <instance-id-or-label>` when multiple browsers are connected (`bsk browsers` / error output lists them). Add `--no-focus` to open the Agent Window in the background without stealing focus from the user's current window.
|
||||
Do not rely on the idle timeout for cleanup. Stop the session as soon as the goal is met unless the
|
||||
user explicitly asks to keep it open. Stopping also returns borrowed tabs.
|
||||
|
||||
Emergency cleanup: `bsk session stop --all` or the Agent Window overlay **Stop all**.
|
||||
Any `bsk` command auto-starts the background services it needs; never manage the daemon by hand.
|
||||
When multiple browsers are connected, use `bsk browsers` and start with
|
||||
`bsk session start --browser <id-or-label>`. Add `--no-focus` to that same start command when the
|
||||
Agent Window does not need to interrupt the user's current work; it is not a flag on other commands.
|
||||
Run `bsk doctor` when startup or transport problems persist after one retry.
|
||||
|
||||
## Stop when the goal is met
|
||||
## Work toward one observable goal
|
||||
|
||||
Every task is a **bounded goal**, not open-ended browsing. The goal may come from the user's request, a recorded `trace.json`, or both.
|
||||
- Derive a concrete success condition from the user's request or a supplied trace.
|
||||
- Take the shortest purposeful path: observe, act, then make at most one observation to confirm an
|
||||
ambiguous result.
|
||||
- Once success is visible, do not click, refresh, navigate, switch tabs, or perform extra checks.
|
||||
- If a human-only step appears or two attempts make no progress, request help instead of
|
||||
brute-forcing.
|
||||
|
||||
1. **Define success first** — one concrete, observable condition derived from the user's words, `purpose`, or the last meaningful step in a trace (e.g. "form submitted", "item added to cart", "playback started").
|
||||
2. **Take the shortest path** — snapshot → act → at most one check. Do not wander, re-try unrelated actions, or stack exploratory steps.
|
||||
3. **Stop as soon as success is reached** — run `bsk session stop <id>` immediately unless the user explicitly asked to keep the session open (e.g. "don't close yet", "keep browsing").
|
||||
4. **No post-success work** — once the goal is met, do not click, refresh, navigate, re-search, switch tabs, or "double-check" that it worked. Further verification is a new task.
|
||||
5. **When blocked, pause — do not brute-force** — if the page requires human input (login, captcha, OTP, payment confirmation) or an action fails twice with no progress, call `bsk request-help` instead of retrying blindly. See **Ask the human for help** below.
|
||||
6. **When unsure** — at most one extra `bsk snapshot`. If success looks met, stop. If not, ask the user; do not keep clicking.
|
||||
With a trace, follow its semantic target information and values in order, but treat its refs as
|
||||
record-local hints. Stop when its purpose or last meaningful effect is satisfied. A trace guides the
|
||||
task; it does not expand the user's goal or authorize additional actions.
|
||||
|
||||
**With a trace:** replay steps in order using `target` role/name/tag and raw `value`/`selection` fields. After the last step (or when its `effect.navigated_to` / success hint is satisfied), apply rules 3–4 immediately. The trace guides execution; it does not extend control beyond the goal.
|
||||
## Observe, act, observe
|
||||
|
||||
**Without a trace:** the user's request *is* the success condition. Satisfying it ends the task — same stop rules apply.
|
||||
Use this default loop:
|
||||
|
||||
## Core interaction loop
|
||||
|
||||
Write operations only affect tabs in the **Agent Window** (or tabs you **borrowed** into it).
|
||||
|
||||
```
|
||||
```text
|
||||
bsk navigate <url> --session <id>
|
||||
bsk observe --session <id> → primary semantic VOM view; reveals hover/focus surfaces
|
||||
bsk snapshot --session <id> → static aria tree fallback when VOM is insufficient
|
||||
bsk hover @e3 --session <id> → reveal hover-triggered menus before re-observing/clicking
|
||||
bsk click @e4 --session <id> → or bsk fill, bsk select, bsk press
|
||||
bsk observe --session <id> → again after navigation / DOM change
|
||||
bsk observe --session <id>
|
||||
bsk click|hover|fill|select|press ... --session <id>
|
||||
bsk observe --session <id> # after navigation or a meaningful DOM change
|
||||
```
|
||||
|
||||
**Refs invalidate after navigation** — always re-snapshot before clicking, filling, or selecting on a new page.
|
||||
Prefer fresh `@eN` refs over CSS selectors. Navigation invalidates refs; large DOM changes may also
|
||||
make them stale. Observe again before the next interaction.
|
||||
|
||||
Prefer `@eN` refs from the latest snapshot over raw CSS selectors. Use `--ref` / `--selector` when ambiguous (`bsk click --help`).
|
||||
An observation marks a hover-only surface as `@e1 button "Products" [hover first: Shoes | Bags]`.
|
||||
The listed items are labels, not usable refs: hover the trigger, observe again, then act on the
|
||||
revealed item's own ref. Do not click the trigger itself unless the user wants the trigger's action.
|
||||
`[has-submenu]` and `[expanded]` mark the same kind of trigger without listing what it hides.
|
||||
|
||||
When VOM renders `[hover first: …]` on an element, the listed items are not currently clickable refs. Run `bsk hover <that-ref> --session <id>`, then immediately run `bsk snapshot` or `bsk observe` again and click the newly visible menu item ref. Do not click the trigger itself unless the user explicitly wants the trigger action.
|
||||
`bsk observe` does not hover the page on its own. Reach for `--probe-hover` when a control you have
|
||||
good reason to expect is absent **and** no marker points at a trigger — that combination is what a
|
||||
CSS-only hover menu looks like from here. It hovers a bounded set of likely triggers, so it costs a
|
||||
few seconds and touches the live page; once you know which element hides the menu, `bsk hover <ref>`
|
||||
is cheaper and more precise.
|
||||
|
||||
## Observation priority
|
||||
Escalate page reading only as needed:
|
||||
|
||||
Start with `bsk observe` to understand page structure, text, controls, element refs, and conditional hover/focus surfaces. Use `bsk snapshot` only when you need the stricter static accessibility tree or VOM is insufficient. Only escalate to raw HTML or screenshots when the latest observation cannot answer the question:
|
||||
1. `bsk observe` for normal semantic understanding, text, controls, and refs.
|
||||
2. `bsk observe --probe-hover` once when an expected control is missing and no marker points at a
|
||||
trigger.
|
||||
3. `bsk snapshot` when a stricter static accessibility tree is more useful.
|
||||
4. `bsk get-html` for exact markup or hidden metadata that semantic views cannot provide.
|
||||
5. `bsk screenshot` for layout, styling, canvas, images, or requested visual evidence.
|
||||
|
||||
1. `bsk observe` — primary semantic VOM observation; may run bounded perception probes such as hover-surface discovery
|
||||
2. `bsk snapshot` — strict static accessibility tree fallback
|
||||
3. `bsk get-html` — when hidden DOM, metadata, or markup details are required
|
||||
4. `bsk screenshot` — when visual layout, canvas/image content, or styling cannot be inferred from the observation. Use `--ref @eN` (from the latest snapshot/observe) to crop to one element; omit `--ref` for the full visible tab.
|
||||
Do not start with raw HTML or screenshots merely to discover ordinary controls. When interaction is
|
||||
needed, obtain a fresh observation before acting on screenshot or HTML findings.
|
||||
|
||||
Do **not** call `bsk get-html` or `bsk screenshot` first just to inspect a page.
|
||||
## Respect the Agent Window boundary
|
||||
|
||||
## Sandbox rules
|
||||
Normal page writes affect only Agent Window tabs. To operate a user tab, first list it with
|
||||
`bsk tab list --scope user --session <id>`, then `bsk tab borrow <tab-id>`. Return it immediately
|
||||
after the relevant step with `bsk tab return <tab-id>`; never invent a tab id or keep a personal tab
|
||||
borrowed across unrelated work.
|
||||
|
||||
| Rule | Detail |
|
||||
|------|--------|
|
||||
| Agent Window | `bsk tab create`, `bsk navigate`, `bsk click`, etc. work on agent tabs by default |
|
||||
| User tabs | Read-only until borrowed: `bsk tab list --session <id> --scope user` then `bsk tab borrow <tab-id> --session <id>` |
|
||||
| Return borrowed tabs | Call `bsk tab return <tab-id> --session <id>` when finished; unreturned tabs are **auto-returned** on `bsk session stop` |
|
||||
| Writes off-agent | Commands that mutate the page fail if the tab is not in the Agent Window — borrow or create a tab first |
|
||||
## Ask the human when needed
|
||||
|
||||
## Global flags
|
||||
Use `bsk request-help` for login, captcha, OTP, payment confirmation, consent, or another step the
|
||||
user must complete. Give a precise prompt and pass fresh `--target` refs/selectors when concrete
|
||||
controls can be highlighted. Use completion criteria only when the page has a clear stable success
|
||||
signal.
|
||||
|
||||
| Flag | Purpose |
|
||||
|------|---------|
|
||||
| `--json` | Machine-readable JSON on stdout (errors too) |
|
||||
| `--quiet` | Suppress informational stderr |
|
||||
| `-v` / `-vv` | More verbose logging |
|
||||
The result `outcome` is one of `continued`, `completed`, `cancelled`, `timed_out`, or `disabled`
|
||||
(`navigated` is deprecated — never treat navigation as a completion signal). Resume only after
|
||||
`continued` or `completed`. Treat `cancelled` as rejection, and `timed_out` or `disabled` as a
|
||||
blocker rather than a reason to retry. After control returns, run a fresh `bsk observe` before
|
||||
reasoning about the page or using refs.
|
||||
|
||||
Auto-update is **on by default** — the background daemon upgrades `bsk` itself when a new release is available (postponed while a session is active). Set `BSK_AUTO_UPDATE=off` to disable it and upgrade manually with `bsk update`.
|
||||
## Command inventory
|
||||
|
||||
Command-specific flags (timeouts, `--tab-id`, `--wait-until`, …): **`bsk <cmd> --help`**
|
||||
This list of names is complete. Never invent a command outside it; read
|
||||
`bsk <command...> --help` for flags instead of guessing them.
|
||||
|
||||
## CLI command reference (one line each)
|
||||
|
||||
Details and flags: **`bsk <cmd> --help`**
|
||||
|
||||
### Diagnostics
|
||||
|
||||
| Command | Summary |
|
||||
|---------|---------|
|
||||
| `bsk status` | Connection health, connected browsers, active sessions |
|
||||
| `bsk doctor` | Deep diagnostics and repair hints |
|
||||
| `bsk browsers` | List connected browser instances (ids, labels, versions) |
|
||||
|
||||
### Session
|
||||
|
||||
| Command | Summary |
|
||||
|---------|---------|
|
||||
| `bsk session start` | Open Agent Window (`--width`/`--height` for initial size); prints **4-letter session id** |
|
||||
| `bsk session start --no-focus` | Open Agent Window in the background without stealing focus |
|
||||
| `bsk session stop <id>` | End session, close Agent Window, auto-return borrowed tabs |
|
||||
| `bsk session stop --all` | Stop every active session |
|
||||
| `bsk session list` | List active sessions |
|
||||
|
||||
### Window (require `--session <id>`)
|
||||
|
||||
| Command | Summary |
|
||||
|---------|---------|
|
||||
| `bsk window resize` | Resize the Agent Window (`--width`, `--height`; 100..=7680 CSS px) |
|
||||
|
||||
### Device emulation — `bsk emulate` (requires `--session <id>`)
|
||||
|
||||
Emulate a mobile device environment on the agent tab via CDP — viewport, User-Agent, and touch — to debug a page's mobile layout and behaviour:
|
||||
|
||||
```bash
|
||||
bsk emulate --session <id> --device iphone-14
|
||||
bsk emulate --session <id> --width 390 --height 844 --dpr 3 --mobile --ua "Mozilla/5.0 (iPhone…" --touch
|
||||
bsk emulate --session <id> --off
|
||||
```text
|
||||
session start|stop|list browsers status doctor update logs
|
||||
navigate navigate-back navigate-forward reload wait-for-navigation wait-ms
|
||||
observe snapshot get-html screenshot console network
|
||||
click hover fill select press evaluate
|
||||
tab list|create|close|select|borrow|return window resize emulate
|
||||
upload download request-help record start|stop
|
||||
```
|
||||
|
||||
- Presets (`--device`): `iphone-14`, `iphone-14-pro-max`, `iphone-se`, `pixel-7`, `galaxy-s23`, `ipad-mini`, `galaxy-tab-s8`. Manual flags (`--width`/`--height`/`--dpr`/`--mobile`/`--ua`/`--accept-language`/`--touch`/`--max-touch-points`) also work without a preset, or override individual preset fields; `--no-mobile`/`--no-touch` turn a preset's mobile viewport / touch emulation back off.
|
||||
- Repeated runs merge field by field onto the tab's current emulation state — only the flags you pass change. E.g. after `--device iphone-14`, `bsk emulate --session <id> --width 390 --height 844` keeps the preset's dpr (`3`) and mobile viewport. (The extension remembers the state per tab; after an extension reload the next run applies only the fields it carries.)
|
||||
- `--off` clears every override (viewport, UA, touch) and the remembered state, restoring the tab's real environment.
|
||||
- Scope: overrides apply to **one tab only** (CDP per-target) and are **not inherited by new tabs** — re-run `bsk emulate` after opening or switching to another tab (default target is the session's active tab; `--tab-id` overrides).
|
||||
- Emulation covers viewport/UA/touch only; it does not throttle the network, fake geolocation, or synthesise real touch-event streams.
|
||||
Required flags that are easy to get wrong:
|
||||
|
||||
### Tabs (require `--session <id>`)
|
||||
|
||||
| Command | Summary |
|
||||
|---------|---------|
|
||||
| `bsk tab list` | List tabs (`--scope user\|agent\|all`, default `all`) |
|
||||
| `bsk tab create` | New tab in Agent Window (`--url`, `--no-active`, `--index`) |
|
||||
| `bsk tab close <tab-id>` | Close an agent tab |
|
||||
| `bsk tab select <tab-id>` | Focus an agent tab |
|
||||
| `bsk tab borrow <tab-id>` | Move a user tab into the Agent Window |
|
||||
| `bsk tab return <tab-id>` | Return a borrowed tab to its original window |
|
||||
|
||||
### Observation (require `--session` unless noted)
|
||||
|
||||
| Command | Summary |
|
||||
|---------|---------|
|
||||
| `bsk snapshot` | First-choice static page understanding: accessibility tree with `@eN` element refs |
|
||||
| `bsk observe` | Semantic VOM observation with bounded perception probes for conditional surfaces |
|
||||
| `bsk get-html` | Raw HTML dump after snapshot is insufficient (high token cost) |
|
||||
| `bsk screenshot` | PNG capture after snapshot is insufficient: full visible tab, or `--ref @eN` to crop to one element (`--out` path optional) |
|
||||
|
||||
### Console & network debugging (read-only; require `--session`)
|
||||
|
||||
| Command | Summary |
|
||||
|---------|---------|
|
||||
| `bsk console` | Buffered page console messages, JS exceptions, and browser log entries (`--include-stack` for stack traces) |
|
||||
| `bsk network` | Buffered network responses (status, method, URL, MIME/resource type) and failures (`net::ERR_*` reason) |
|
||||
|
||||
Both capture from the moment the tab is attached and read a bounded per-tab buffer: `--since <seq>` pages from a cursor (`next_since` in the result), `--limit` (default 50, max 200), `--max-text-chars` (default 1000, max 4096), `--tab-id` to target a non-active tab. Both are strictly read-only — they never intercept or modify traffic, and request/response headers, bodies, and timings are not captured.
|
||||
|
||||
### Navigation
|
||||
|
||||
| Command | Summary |
|
||||
|---------|---------|
|
||||
| `bsk navigate <url>` | Go to URL in agent tab (`--wait-until`, `--timeout`) |
|
||||
| `bsk navigate-back` | History back one step |
|
||||
| `bsk navigate-forward` | History forward one step |
|
||||
| `bsk reload` | Reload current tab (`--hard` bypass cache) |
|
||||
|
||||
(`bsk navigate back` / `bsk navigate forward` are equivalent subcommands.)
|
||||
|
||||
### Interaction
|
||||
|
||||
| Command | Summary |
|
||||
|---------|---------|
|
||||
| `bsk click <ref-or-selector>` | Click element (`--button`, `--click-count`, `--modifiers`) |
|
||||
| `bsk hover <ref-or-selector>` | Move the mouse to an element and wait for hover UI to settle (`--settle`, `--modifiers`) |
|
||||
| `bsk fill <ref-or-selector> --value <text>` | Clear and type into input |
|
||||
| `bsk select <ref-or-selector> --value <v>` | Set `<select>` option(s) by `value` (repeat `--value` for multi-select) |
|
||||
| `bsk press <key>` | Key/combo (`Enter`, `Ctrl+A`, …; optional `--ref` to focus first) |
|
||||
|
||||
### Scripting & timing
|
||||
|
||||
| Command | Summary |
|
||||
|---------|---------|
|
||||
| `bsk evaluate <expression>` | Run JS in agent tab (see red lines). JS throw → stderr, **exit 0** (RPC success); use `--json` and check `.ok` to detect JS errors |
|
||||
| `bsk wait-for-navigation` | Block until load/DOM idle/etc. (`--wait-until`, `--timeout`) |
|
||||
| `bsk wait-ms <duration>` | Sleep (`500ms`, `2s`, `1m`; **no** `--session`) |
|
||||
|
||||
### Ask the human for help — `bsk request-help`
|
||||
|
||||
When a step needs a human (captcha, login, OTP) or you want the user to
|
||||
confirm an important action, pause and ask:
|
||||
|
||||
bsk request-help --session <id> --prompt "Solve the captcha, then click Done only after the site accepts it" \
|
||||
--title "Captcha required" --target @e7 --target "#submit" --timeout 5m
|
||||
|
||||
- `--prompt` (required): what the user should do.
|
||||
- `--title` (optional): custom title for the overlay panel. When omitted,
|
||||
the extension shows its default localized title.
|
||||
- `--target` (repeatable): a snapshot ref (`@e7`) or CSS selector
|
||||
(`#submit`) to scroll to and flash-highlight. **Strongly recommended** —
|
||||
whenever the prompt refers to a concrete element (a button to click, a
|
||||
field to fill, a checkbox to toggle), pass its `@eN` ref / selector so the
|
||||
user is guided straight to the right spot instead of hunting for it. For
|
||||
interaction scenarios, always include the relevant target(s); reserve a
|
||||
prompt with no `--target` for cases where there is genuinely no specific
|
||||
element to point at (e.g. "wait for the page to finish loading").
|
||||
- `--timeout` (default `5m`): how long to wait.
|
||||
- `--completion-criteria` (optional): JSON success detector. Use it only
|
||||
when there is a concrete post-help success signal, e.g.
|
||||
`{"any":[{"url_contains":"/dashboard"},{"selector_exists":"[data-testid='account-menu']"}],"stable_for_ms":1000}`.
|
||||
|
||||
The target tab is brought to the foreground; the page stays interactive
|
||||
while the agent control mask is hidden. The call blocks until the user
|
||||
explicitly acts, the timeout expires, cancellation arrives, or explicit
|
||||
completion criteria match. Page reloads, SPA route changes, and captcha
|
||||
refreshes do not return control by themselves. The result `outcome` is one of:
|
||||
|
||||
- `continued` — the user finished and clicked Done / return control (treat as confirm).
|
||||
- `cancelled` — the user clicked Cancel (treat as reject/abort).
|
||||
- `timed_out` — nobody acted within the timeout.
|
||||
- `completed` — the explicit `--completion-criteria` matched while the user had control.
|
||||
- `navigated` — deprecated legacy outcome. Do not rely on navigation as a completion signal.
|
||||
|
||||
`note` carries any text the user typed back. `resolved_targets` reports
|
||||
which refs/selectors matched a live element.
|
||||
|
||||
`request-help` does not refresh the page model after the user returns
|
||||
control. After a `continued` or `completed` result, issue a separate
|
||||
observation tool call (usually `bsk snapshot --session <id>`) before using
|
||||
new refs or reasoning about the post-help page state.
|
||||
#### Disabling request-help (unattended mode)
|
||||
Set `BSK_REQUEST_HELP=off` on unattended servers: `bsk request-help` then
|
||||
returns immediately with `outcome="disabled"` (no overlay, no waiting,
|
||||
exit 0). Any other value keeps it enabled. If you get `disabled`, do not
|
||||
retry — complete the task autonomously or stop gracefully.
|
||||
|
||||
### Recording — `bsk record`
|
||||
|
||||
Capture the user's own actions in the Agent Window to a **trace bundle**, for later LLM-driven automation. New CLI builds request **trace v3** (page observations + action chain); older extensions may still return **trace v2** (actions only), which the CLI exports as a single `trace.json` without `states/`.
|
||||
|
||||
```bash
|
||||
bsk record start --browser <instance-id-or-label> \
|
||||
[--url https://…] [--purpose "publish a wiki doc"] \
|
||||
[--max-page-tokens 3000] [--redact-values] \
|
||||
[--output trace]
|
||||
# `--url` is optional; default https://example.com/ when omitted (must be http(s)).
|
||||
# Blocks until the user clicks Finish in the recording panel, then writes:
|
||||
# trace/trace.json — action chain (+ state index when v3)
|
||||
# trace/states/ — v3 only: one `sN.txt` observe snapshot per settled page state
|
||||
|
||||
bsk record stop [--output trace] # terminal fallback if the browser panel is unavailable
|
||||
```text
|
||||
bsk fill <ref> --value <text> bsk select <ref> --value <option-value>
|
||||
bsk screenshot --out <path> bsk emulate --device <preset-id>
|
||||
bsk upload <ref> --file <path> bsk download <ref> --out <path>
|
||||
```
|
||||
|
||||
- **v3 bundle (preferred):** `--output` is a directory (default `./trace`) containing `trace.json` and `states/`. `trace.json` lists `states[]` (page observation ids) and `steps[]` (each step binds `state` = observe snapshot *before* the action and `result.state` = snapshot *after* settle). Page bodies live in `states/sN.txt` using the same VOM format as `bsk observe`.
|
||||
- **v2 fallback:** when the connected extension is older, export may contain only legacy `trace.json` with `pages[]` action context and no `states/` observation files. Update the extension for full v3 bundles.
|
||||
- Each `states[]` entry is one **settled page observation** (captured after recording start, navigation landing, or action settle — not on a timer).
|
||||
- `target.ref` values like `@e12` exist **only inside** the bundle for disambiguation; do **not** copy `@eN` refs into SKILL.md or agent runbooks — use visible names from the observation text instead.
|
||||
- `--purpose` is optional context metadata; it does **not** change what gets captured.
|
||||
- `--redact-values` masks all form values in page files as `[filled]` / `[empty]`.
|
||||
- Do **not** record on banking/SSO/password-manager pages; passwords are redacted but traces may still contain sensitive text.
|
||||
`select` matches an option's `value` attribute, not its visible label. Device preset ids are
|
||||
lowercase and hyphenated, such as `iphone-14`.
|
||||
|
||||
## Error handling
|
||||
- `console` and `network` provide bounded, read-only debugging evidence.
|
||||
- `emulate` applies viewport, user-agent, and touch overrides to one tab; new tabs do not inherit
|
||||
them. Use `--off` to restore the real environment.
|
||||
- `evaluate` is a last resort when observe plus normal interactions cannot complete the task. With
|
||||
`--json`, inspect `.ok`: a JavaScript exception may still have CLI exit code 0 because the RPC
|
||||
succeeded. Never evaluate credential surfaces to read storage, cookies, or auth data.
|
||||
- `record` captures a user's actions for later replay. Read `bsk record start --help` before use,
|
||||
and never record banking, SSO, password-manager, or other sensitive pages.
|
||||
|
||||
### Exit codes (`echo $?` after `bsk …`)
|
||||
## File transfer
|
||||
|
||||
| Code | Meaning | What to do |
|
||||
|------|---------|------------|
|
||||
| `0` | Success (including `evaluate` where JS threw but RPC succeeded) | Continue; for `evaluate`, check `--json` `.ok` to distinguish JS success from RPC success |
|
||||
| `1` | User error — bad args, unknown session, tab not in Agent Window, stale ref | Fix args; `bsk session list`; re-snapshot |
|
||||
| `2` | Protocol / transport — service unreachable, IPC failure | `bsk doctor`; check extension connected; retry the command |
|
||||
| `3` | Browser / CDP execution failed | Retry; simplify selector; check tab still open |
|
||||
| `4` | Timeout | Increase `--timeout`; try `--wait-until domcontentloaded` |
|
||||
| `5` | Unknown RPC method | This command is not implemented in the current build. Continue the task with other commands. If a newly added command is missing, suggest `bsk update`. |
|
||||
`upload` and `download` stage files through the daemon; the agent never touches browser-internal
|
||||
paths. Treat upload as disclosure to the website, download as accepting website-controlled bytes.
|
||||
|
||||
Human errors print `error:` + `hint:` on stderr; `--json` includes `code`, `message`, `hint`, `exit_code`.
|
||||
Upload has two independent mechanisms — choose explicitly, never rely on automatic fallback:
|
||||
|
||||
### When to run diagnostics
|
||||
- **Default (input mode):** for upload buttons, file-input labels, or "upload from computer"
|
||||
actions. The command clicks the target and intercepts the native file chooser.
|
||||
- **`--mode drop`:** for reliably identified attachment-receiving areas — an explicit drop zone,
|
||||
chat composer, email editor, or form attachment area. Do not target page whitespace, generic
|
||||
containers, or areas whose attachment ownership is ambiguous.
|
||||
|
||||
| Situation | Command |
|
||||
|-----------|---------|
|
||||
| Before first task in a session | `bsk status` — extension connected? |
|
||||
| Any failure you cannot fix in one retry | `bsk doctor` |
|
||||
| Multiple browsers / wrong target | `bsk browsers` then `bsk session start --browser <id>` |
|
||||
Decision sequence when uploading:
|
||||
|
||||
Always **`bsk session stop <id>`** in a `finally`-style path so the Agent Window closes and borrowed tabs return.
|
||||
1. Try input mode (the default).
|
||||
2. If it returns `reason=file_input_not_activated` with `effect_state=none`, re-observe. When a
|
||||
reliable attachment target exists, try `--mode drop` once against that target.
|
||||
3. Otherwise fall back to `request-help`.
|
||||
4. **Never** switch mechanisms or repeat when `effect_state` is `unknown` or `committed` — the
|
||||
browser may already have applied the file.
|
||||
|
||||
## Red lines
|
||||
A successful drop means Chrome dispatched the native file-drop event; it does not prove the site
|
||||
accepted the attachment. Observe the page once after the command.
|
||||
|
||||
1. **No token theft** — do not `bsk evaluate` on sensitive sites to read `localStorage`, cookies, or auth headers for exfiltration.
|
||||
2. **No long borrow** — do not leave a user's personal tab in the Agent Window across unrelated tasks.
|
||||
3. **No skip stop** — always `bsk session stop <id>`; never assume idle timeout will clean up.
|
||||
4. **No post-success control** — once the user’s goal (or last trace step) is met, do not keep operating the page; stop the session unless they asked to keep it open.
|
||||
5. **No raw observe escalation before snapshot/observe** — use `bsk snapshot` first; use `bsk observe` when VOM semantics or conditional surfaces help. Only use `bsk get-html` or `bsk screenshot` when snapshot/observe is insufficient. Element screenshots (`--ref @eN`) still require a fresh snapshot/observe ref — never skip observation just to grab a visual.
|
||||
6. **`evaluate` is powerful and risky** — use only when snapshot + click/fill/select cannot suffice; never on credential surfaces.
|
||||
Download default-refuses to overwrite; pass `--overwrite` when replacing an existing file is
|
||||
intended. Read `bsk upload --help` and `bsk download --help` for all flags and error details.
|
||||
|
||||
---
|
||||
## Recover without wandering
|
||||
|
||||
**More detail for any command:** `bsk <cmd> --help`
|
||||
- Stale ref: observe again and retry the intended action once.
|
||||
- Unknown tab or session: list current tabs/sessions; never guess identifiers.
|
||||
- Timeout: inspect current page state before deciding whether one longer purposeful wait is useful.
|
||||
- Fill result unconfirmed (`fill_value_mismatch`): observe the field first; the page may have
|
||||
formatted the value. Continue if the visible result satisfies the user's intent. Otherwise correct
|
||||
the remaining difference; do not blindly repeat fill or immediately request human help. For other
|
||||
fill errors, follow the returned hint and inspect current state before retrying.
|
||||
- Unsupported command: continue with available capabilities; suggest updating only when the missing
|
||||
command is necessary.
|
||||
- Unrecoverable failure: report the blocker and stop the session in a finally-style path.
|
||||
|
||||
The CLI's current help and error hints are authoritative for flags, parameters, and recovery
|
||||
details.
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
//! Atomic visibility boundary for files received into a same-directory temp.
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
pub fn commit(temp: &Path, out: &Path, overwrite: bool) -> std::io::Result<()> {
|
||||
if !overwrite {
|
||||
std::fs::hard_link(temp, out)?;
|
||||
std::fs::remove_file(temp)?;
|
||||
return Ok(());
|
||||
}
|
||||
replace(temp, out)
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
fn replace(temp: &Path, out: &Path) -> std::io::Result<()> {
|
||||
// POSIX rename replaces an existing non-directory destination atomically.
|
||||
std::fs::rename(temp, out)
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
fn replace(temp: &Path, out: &Path) -> std::io::Result<()> {
|
||||
use std::os::windows::ffi::OsStrExt;
|
||||
use windows_sys::Win32::Storage::FileSystem::{
|
||||
MOVEFILE_REPLACE_EXISTING, MOVEFILE_WRITE_THROUGH, MoveFileExW,
|
||||
};
|
||||
|
||||
let from: Vec<u16> = temp.as_os_str().encode_wide().chain(Some(0)).collect();
|
||||
let to: Vec<u16> = out.as_os_str().encode_wide().chain(Some(0)).collect();
|
||||
// SAFETY: both buffers are NUL-terminated and remain alive for the call.
|
||||
let ok = unsafe {
|
||||
MoveFileExW(
|
||||
from.as_ptr(),
|
||||
to.as_ptr(),
|
||||
MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH,
|
||||
)
|
||||
};
|
||||
if ok == 0 {
|
||||
Err(std::io::Error::last_os_error())
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(any(unix, windows)))]
|
||||
fn replace(temp: &Path, out: &Path) -> std::io::Result<()> {
|
||||
std::fs::rename(temp, out)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn no_replace_never_overwrites_existing_output() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let temp = dir.path().join("new.part");
|
||||
let out = dir.path().join("out.bin");
|
||||
std::fs::write(&temp, b"new").unwrap();
|
||||
std::fs::write(&out, b"old").unwrap();
|
||||
assert!(commit(&temp, &out, false).is_err());
|
||||
assert_eq!(std::fs::read(&out).unwrap(), b"old");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn overwrite_replaces_existing_output_without_predelete() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let temp = dir.path().join("new.part");
|
||||
let out = dir.path().join("out.bin");
|
||||
std::fs::write(&temp, b"new").unwrap();
|
||||
std::fs::write(&out, b"old").unwrap();
|
||||
commit(&temp, &out, true).unwrap();
|
||||
assert_eq!(std::fs::read(&out).unwrap(), b"new");
|
||||
assert!(!temp.exists());
|
||||
}
|
||||
}
|
||||
@@ -73,6 +73,14 @@ where
|
||||
P: Serialize + Send + 'static,
|
||||
R: DeserializeOwned + Send + 'static,
|
||||
{
|
||||
let parent_cancel = stdin_cancellation();
|
||||
if parent_cancel.as_ref().is_some_and(|rx| *rx.borrow()) {
|
||||
return Err(CliError::from_rpc(RpcError {
|
||||
code: ErrorCode::Cancelled,
|
||||
message: "parent closed the cancellation pipe".into(),
|
||||
data: Some(serde_json::json!({"reason": "cancelled_before_dispatch"})),
|
||||
}));
|
||||
}
|
||||
let rpc_id: RpcId = format!("{}-{}", rpc_id_prefix, random_short_id());
|
||||
let mut client = IpcClient::connect(sock.clone()).await?;
|
||||
let rpc_id_for_call = rpc_id.clone();
|
||||
@@ -88,16 +96,16 @@ where
|
||||
let outcome = tokio::select! {
|
||||
biased;
|
||||
res = &mut main_fut => res?,
|
||||
sig = wait_for_sigint() => {
|
||||
sig.context("install SIGINT handler").map_err(CliError::Local)?;
|
||||
debug!(rpc_id = %rpc_id_for_cancel, "SIGINT: forwarding cancel to daemon");
|
||||
sig = wait_for_cancel(parent_cancel) => {
|
||||
sig.context("wait for cancellation").map_err(CliError::Local)?;
|
||||
debug!(rpc_id = %rpc_id_for_cancel, "forwarding cancel to daemon");
|
||||
let _ = send_cancel(&sock, &rpc_id_for_cancel).await;
|
||||
match tokio::time::timeout(CANCEL_RESPONSE_GRACE, &mut main_fut).await {
|
||||
Ok(res) => res?,
|
||||
Err(_) => {
|
||||
debug!(
|
||||
rpc_id = %rpc_id_for_cancel,
|
||||
"SIGINT: cancel grace elapsed; synthesising cancelled error"
|
||||
"cancel grace elapsed; synthesising cancelled error"
|
||||
);
|
||||
return Err(CliError::from_rpc(RpcError {
|
||||
code: ErrorCode::Cancelled,
|
||||
@@ -112,6 +120,46 @@ where
|
||||
outcome.map_err(CliError::from_rpc)
|
||||
}
|
||||
|
||||
/// Release caller-owned staging even after cancellation. This deliberately
|
||||
/// exposes only transfer.release, not a general bypass for business RPCs.
|
||||
/// The entire batch, including connection setup, shares one bounded budget.
|
||||
pub fn release_transfers<'a>(
|
||||
sock: &Path,
|
||||
transfer_ids: impl IntoIterator<Item = &'a str>,
|
||||
) -> Result<(), CliError> {
|
||||
use bsk_protocol::tools::{TransferIdParams, TransferReleaseResult};
|
||||
const CLEANUP_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
let ids: Vec<_> = transfer_ids.into_iter().collect();
|
||||
if ids.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
let rt = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.context("build runtime for transfer cleanup")?;
|
||||
rt.block_on(async {
|
||||
tokio::time::timeout(CLEANUP_TIMEOUT, async {
|
||||
let mut client = IpcClient::connect(sock).await?;
|
||||
for id in ids {
|
||||
client
|
||||
.call::<_, TransferReleaseResult>(
|
||||
"transfer-release",
|
||||
Method::TransferRelease,
|
||||
Some(TransferIdParams {
|
||||
transfer_id: id.to_string(),
|
||||
}),
|
||||
CLEANUP_TIMEOUT,
|
||||
)
|
||||
.await?
|
||||
.map_err(CliError::from_rpc)?;
|
||||
}
|
||||
Ok::<_, CliError>(())
|
||||
})
|
||||
.await
|
||||
.context("transfer cleanup exceeded its total time budget")?
|
||||
})
|
||||
}
|
||||
|
||||
/// Send a `cancel { rpc_id }` frame over a fresh connection so it
|
||||
/// lands on the daemon while the original call is still parked.
|
||||
///
|
||||
@@ -146,8 +194,57 @@ fn random_short_id() -> String {
|
||||
bytes.iter().map(|b| format!("{b:02x}")).collect()
|
||||
}
|
||||
|
||||
/// Wait for either Ctrl-C or process SIGTERM. Returning here means
|
||||
/// the user has asked us to stop.
|
||||
/// Opt-in control pipe for parent processes that cannot send Ctrl-C (Node on
|
||||
/// Windows). One reader survives the short-lived runtimes used by multi-RPC
|
||||
/// commands such as upload. Ordinary CLI stdin and terminal Ctrl-C are unchanged.
|
||||
fn stdin_cancellation() -> Option<tokio::sync::watch::Receiver<bool>> {
|
||||
if std::env::var("BSK_CANCEL_ON_STDIN_CLOSE").as_deref() != Ok("1") {
|
||||
return None;
|
||||
}
|
||||
static CANCEL: std::sync::OnceLock<tokio::sync::watch::Receiver<bool>> =
|
||||
std::sync::OnceLock::new();
|
||||
Some(
|
||||
CANCEL
|
||||
.get_or_init(|| {
|
||||
let (tx, rx) = tokio::sync::watch::channel(false);
|
||||
std::thread::spawn(move || {
|
||||
use std::io::Read;
|
||||
let mut stdin = std::io::stdin().lock();
|
||||
let mut buf = [0; 64];
|
||||
loop {
|
||||
match stdin.read(&mut buf) {
|
||||
Ok(0) => break,
|
||||
Ok(_) => {}
|
||||
Err(err) if err.kind() == std::io::ErrorKind::Interrupted => continue,
|
||||
Err(_) => break,
|
||||
}
|
||||
}
|
||||
let _ = tx.send(true);
|
||||
});
|
||||
rx
|
||||
})
|
||||
.clone(),
|
||||
)
|
||||
}
|
||||
|
||||
async fn wait_for_cancel(
|
||||
parent_cancel: Option<tokio::sync::watch::Receiver<bool>>,
|
||||
) -> anyhow::Result<()> {
|
||||
let parent_closed = async move {
|
||||
match parent_cancel {
|
||||
Some(mut rx) => {
|
||||
let _ = rx.wait_for(|closed| *closed).await;
|
||||
}
|
||||
None => std::future::pending::<()>().await,
|
||||
}
|
||||
};
|
||||
tokio::select! {
|
||||
result = wait_for_sigint() => result,
|
||||
_ = parent_closed => Ok(()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Wait for terminal Ctrl-C.
|
||||
async fn wait_for_sigint() -> anyhow::Result<()> {
|
||||
tokio::signal::ctrl_c().await.context("listen for SIGINT")?;
|
||||
Ok(())
|
||||
|
||||
@@ -0,0 +1,140 @@
|
||||
//! `bsk download` — capture one browser download and atomically commit it.
|
||||
|
||||
use std::fs::OpenOptions;
|
||||
use std::io::Write;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::Context;
|
||||
use base64::Engine;
|
||||
use bsk_protocol::Method;
|
||||
use bsk_protocol::tools::{
|
||||
DownloadParams, DownloadResult, TransferChunkParams, TransferChunkResult,
|
||||
};
|
||||
use clap::Args;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::cli::atomic_output;
|
||||
use crate::cli::ensure_daemon::ensure_daemon;
|
||||
use crate::cli::error::{CliError, Format};
|
||||
use crate::cli::interaction::split_target;
|
||||
use crate::cli::navigate::parse_timeout_ms;
|
||||
|
||||
#[derive(Debug, Clone, Args)]
|
||||
pub struct DownloadArgs {
|
||||
/// Snapshot ref (`@e3`) or CSS selector for the download trigger.
|
||||
pub target: Option<String>,
|
||||
#[arg(long = "ref")]
|
||||
pub ref_: Option<String>,
|
||||
#[arg(long = "selector")]
|
||||
pub selector: Option<String>,
|
||||
#[arg(long)]
|
||||
pub out: PathBuf,
|
||||
#[arg(long)]
|
||||
pub session: String,
|
||||
#[arg(long = "tab-id")]
|
||||
pub tab_id: Option<i64>,
|
||||
#[arg(long, default_value = "2m", value_parser = parse_timeout_ms)]
|
||||
pub timeout: u32,
|
||||
#[arg(long)]
|
||||
pub overwrite: bool,
|
||||
}
|
||||
|
||||
pub fn dispatch(args: DownloadArgs, format: Format) -> Result<(), CliError> {
|
||||
if args.out.exists() && !args.overwrite {
|
||||
return Err(CliError::Local(anyhow::anyhow!(
|
||||
"output already exists (pass --overwrite to replace it): {}",
|
||||
args.out.display()
|
||||
)));
|
||||
}
|
||||
let info = ensure_daemon().context("ensure daemon is running")?;
|
||||
let (ref_, selector) = split_target(args.target, args.ref_, args.selector)?;
|
||||
let params = DownloadParams {
|
||||
session_id: args.session,
|
||||
ref_,
|
||||
selector,
|
||||
tab_id: args.tab_id,
|
||||
timeout_ms: Some(args.timeout),
|
||||
browser_relative_dir: None,
|
||||
max_byte_size: None,
|
||||
};
|
||||
let reply: DownloadResult = crate::cli::business_rpc::call(
|
||||
info.sock_path.clone(),
|
||||
"download",
|
||||
Method::ToolDownload,
|
||||
Some(params),
|
||||
ipc_timeout(args.timeout),
|
||||
)?;
|
||||
let transfer_id = reply.transfer_id.clone().ok_or_else(|| {
|
||||
CliError::Local(anyhow::anyhow!("daemon returned no download transfer id"))
|
||||
})?;
|
||||
let write_result = write_transfer(&info.sock_path, &transfer_id, &args.out, args.overwrite);
|
||||
let _ = crate::cli::business_rpc::release_transfers(&info.sock_path, [transfer_id.as_str()]);
|
||||
write_result?;
|
||||
match format {
|
||||
Format::Json => {
|
||||
let mut value = serde_json::to_value(&reply).unwrap();
|
||||
value["path"] = serde_json::json!(args.out.to_string_lossy());
|
||||
if let Some(obj) = value.as_object_mut() {
|
||||
obj.remove("transfer_id");
|
||||
obj.remove("browser_path");
|
||||
}
|
||||
println!("{}", serde_json::to_string_pretty(&value).unwrap());
|
||||
}
|
||||
Format::Human => println!("{}", args.out.display()),
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn write_transfer(sock: &Path, id: &str, out: &Path, overwrite: bool) -> Result<(), CliError> {
|
||||
let parent = out
|
||||
.parent()
|
||||
.filter(|p| !p.as_os_str().is_empty())
|
||||
.unwrap_or(Path::new("."));
|
||||
let temp = parent.join(format!(".bsk-download-{}.part", Uuid::new_v4().simple()));
|
||||
let result = (|| {
|
||||
let mut file = OpenOptions::new()
|
||||
.write(true)
|
||||
.create_new(true)
|
||||
.open(&temp)
|
||||
.with_context(|| format!("create temporary download {}", temp.display()))
|
||||
.map_err(CliError::Local)?;
|
||||
let mut offset = 0u64;
|
||||
loop {
|
||||
let chunk: TransferChunkResult = crate::cli::business_rpc::call(
|
||||
sock.to_path_buf(),
|
||||
"transfer-read",
|
||||
Method::TransferRead,
|
||||
Some(TransferChunkParams {
|
||||
transfer_id: id.to_string(),
|
||||
offset,
|
||||
data_base64: String::new(),
|
||||
}),
|
||||
Duration::from_secs(30),
|
||||
)?;
|
||||
let encoded = chunk.data_base64.as_deref().unwrap_or("");
|
||||
let bytes = base64::engine::general_purpose::STANDARD
|
||||
.decode(encoded)
|
||||
.map_err(|e| CliError::Local(anyhow::anyhow!("decode download chunk: {e}")))?;
|
||||
file.write_all(&bytes)
|
||||
.map_err(|e| CliError::Local(e.into()))?;
|
||||
offset = chunk.next_offset;
|
||||
if chunk.eof {
|
||||
break;
|
||||
}
|
||||
}
|
||||
file.sync_all().map_err(|e| CliError::Local(e.into()))?;
|
||||
drop(file);
|
||||
atomic_output::commit(&temp, out, overwrite)
|
||||
.with_context(|| format!("atomically commit download to {}", out.display()))
|
||||
.map_err(CliError::Local)
|
||||
})();
|
||||
if result.is_err() {
|
||||
let _ = std::fs::remove_file(&temp);
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
fn ipc_timeout(timeout_ms: u32) -> Duration {
|
||||
Duration::from_millis(u64::from(timeout_ms) + 5_000)
|
||||
}
|
||||
@@ -497,21 +497,4 @@ mod tests {
|
||||
.contains("CSS selector")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_selector_explains_css_contract_in_json() {
|
||||
let cli = CliError::from_rpc(RpcError {
|
||||
code: ErrorCode::InvalidParams,
|
||||
message: "selector is not valid CSS: text=submit".into(),
|
||||
data: Some(serde_json::json!({ "reason": "invalid_selector" })),
|
||||
});
|
||||
let exit = cli.exit_code();
|
||||
let hint = hint_for(&cli, render_info_for(&cli).as_ref());
|
||||
let json = json_error_string(&cli, exit, hint);
|
||||
let parsed: serde_json::Value = serde_json::from_str(&json).unwrap();
|
||||
let hint = parsed.get("hint").and_then(|v| v.as_str()).unwrap();
|
||||
assert!(hint.contains("CSS selector"));
|
||||
assert!(hint.contains("text="));
|
||||
assert!(hint.contains("@eN"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -71,7 +71,7 @@ pub(crate) fn parse_modifiers(input: &str) -> Result<Vec<KeyModifier>, String> {
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
fn split_target(
|
||||
pub(crate) fn split_target(
|
||||
positional: Option<String>,
|
||||
explicit_ref: Option<String>,
|
||||
explicit_selector: Option<String>,
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
mod atomic_output;
|
||||
pub mod browser_wait;
|
||||
pub mod browsers;
|
||||
pub mod business_rpc;
|
||||
@@ -9,6 +10,7 @@ pub mod console;
|
||||
pub mod daemon;
|
||||
pub mod dialogs;
|
||||
pub mod doctor;
|
||||
pub mod download;
|
||||
pub mod emulate;
|
||||
pub mod ensure_daemon;
|
||||
pub mod error;
|
||||
@@ -31,6 +33,7 @@ pub mod snapshot;
|
||||
pub mod status;
|
||||
pub mod tab;
|
||||
pub mod update;
|
||||
pub mod upload;
|
||||
pub mod waits;
|
||||
pub mod window;
|
||||
|
||||
@@ -38,6 +41,7 @@ use clap::{Args, Parser, Subcommand};
|
||||
|
||||
use crate::cli::console::ConsoleArgs;
|
||||
use crate::cli::daemon::DaemonCmd;
|
||||
use crate::cli::download::DownloadArgs;
|
||||
use crate::cli::emulate::EmulateArgs;
|
||||
use crate::cli::evaluate::EvaluateArgs;
|
||||
use crate::cli::get_html::GetHtmlArgs;
|
||||
@@ -53,6 +57,7 @@ use crate::cli::session::SessionCmd;
|
||||
use crate::cli::snapshot::SnapshotArgs;
|
||||
use crate::cli::tab::TabCmd;
|
||||
use crate::cli::update::UpdateArgs;
|
||||
use crate::cli::upload::UploadArgs;
|
||||
use crate::cli::waits::{WaitForNavigationArgs, WaitMsArgs};
|
||||
use crate::cli::window::WindowCmd;
|
||||
|
||||
@@ -177,6 +182,12 @@ pub enum Command {
|
||||
/// Set `<select>` option values by `value` attribute.
|
||||
Select(SelectArgs),
|
||||
|
||||
/// Upload files through a page file input or explicit drop target.
|
||||
Upload(UploadArgs),
|
||||
|
||||
/// Capture one browser download and write it to a local path.
|
||||
Download(DownloadArgs),
|
||||
|
||||
/// Evaluate a JavaScript expression inside the Agent Window.
|
||||
Evaluate(EvaluateArgs),
|
||||
|
||||
|
||||
@@ -33,6 +33,12 @@ pub struct ObserveArgs {
|
||||
/// Include conditional surface probe diagnostics in JSON output.
|
||||
#[arg(long = "debug-surfaces")]
|
||||
pub debug_surfaces: bool,
|
||||
|
||||
/// Actively hover page controls to reveal hover-only menus and tooltips.
|
||||
/// Costs seconds of wall clock and touches the live page, so it is off
|
||||
/// unless a static observation looks like it is missing hover content.
|
||||
#[arg(long = "probe-hover")]
|
||||
pub probe_hover: bool,
|
||||
}
|
||||
|
||||
pub fn dispatch(args: ObserveArgs, format: Format) -> Result<(), CliError> {
|
||||
@@ -47,6 +53,7 @@ fn run(sock: PathBuf, args: ObserveArgs, format: Format) -> Result<(), CliError>
|
||||
max_depth: args.max_depth,
|
||||
max_tokens: args.max_tokens,
|
||||
debug_surfaces: args.debug_surfaces,
|
||||
probe_hover: args.probe_hover,
|
||||
};
|
||||
let reply: ObserveResult = call(sock, params)?;
|
||||
match format {
|
||||
|
||||
@@ -41,15 +41,29 @@ pub mod reason {
|
||||
pub const AGENT_WINDOW_SCOPE: &str = "agent_window_scope";
|
||||
pub const ELEMENT_NOT_VISIBLE: &str = "element_not_visible";
|
||||
pub const REF_NOT_FOUND: &str = "ref_not_found";
|
||||
pub const INVALID_SELECTOR: &str = "invalid_selector";
|
||||
pub const SELECTOR_NOT_FOUND: &str = "selector_not_found";
|
||||
pub const TARGET_NOT_FILLABLE: &str = "target_not_fillable";
|
||||
pub const FILL_VALUE_INVALID: &str = "fill_value_invalid";
|
||||
pub const FILL_TARGET_CHANGED: &str = "fill_target_changed";
|
||||
pub const FILL_FOCUS_LOST: &str = "fill_focus_lost";
|
||||
pub const FILL_VALUE_MISMATCH: &str = "fill_value_mismatch";
|
||||
pub const FILL_FAILED: &str = "fill_failed";
|
||||
pub const TARGET_NOT_SELECT: &str = "target_not_select";
|
||||
pub const OPTION_NOT_FOUND: &str = "option_not_found";
|
||||
pub const SINGLE_SELECT_VALUE_COUNT: &str = "single_select_value_count";
|
||||
pub const TAB_NOT_ACTIVE: &str = "tab_not_active";
|
||||
pub const CDP_EXTENSION_ACCESS_DENIED: &str = "cdp_extension_access_denied";
|
||||
pub const BORROW_CONFLICT: &str = "borrow_conflict";
|
||||
pub const SCREENSHOT_CAPTURE_FAILED: &str = "screenshot_capture_failed";
|
||||
pub const FILE_INPUT_PROBE_FAILED: &str = "file_input_probe_failed";
|
||||
pub const FILE_INPUT_NOT_ACTIVATED: &str = "file_input_not_activated";
|
||||
pub const SET_FILE_INPUT_FAILED: &str = "set_file_input_failed";
|
||||
pub const UPLOAD_MECHANISM_UNSUPPORTED: &str = "upload_mechanism_unsupported";
|
||||
pub const FILE_DROP_TARGET_UNAVAILABLE: &str = "file_drop_target_unavailable";
|
||||
pub const FILE_DROP_FAILED: &str = "file_drop_failed";
|
||||
pub const DOWNLOAD_CAPTURE_FAILED: &str = "download_capture_failed";
|
||||
pub const TRANSFER_OUTCOME_UNKNOWN: &str = "transfer_outcome_unknown";
|
||||
pub const TRANSFER_TIMEOUT: &str = "transfer_timeout";
|
||||
pub const SESSION_BUSY: &str = crate::rpc_reason::SESSION_BUSY;
|
||||
pub const RECORD_START_PAGE_UNREACHABLE: &str = "record_start_page_unreachable";
|
||||
}
|
||||
@@ -198,6 +212,27 @@ pub fn info_for_error(code: ErrorCode, data: Option<&serde_json::Value>) -> Rend
|
||||
return base;
|
||||
};
|
||||
match (code, reason) {
|
||||
(ErrorCode::CdpFailed, reason::CDP_EXTENSION_ACCESS_DENIED) => RenderInfo {
|
||||
summary: "Chrome blocked CDP access to another extension's content in this tab",
|
||||
hint: Some(
|
||||
"a web page can contain a restricted extension frame; disable the conflicting extension and reload, or use `bsk navigate <url>` to leave this page; reconnecting BrowserSkill alone does not remove the restriction",
|
||||
),
|
||||
exit_code: base.exit_code,
|
||||
},
|
||||
(_, reason::TRANSFER_OUTCOME_UNKNOWN) => RenderInfo {
|
||||
summary: "the file transfer outcome could not be confirmed",
|
||||
hint: Some(
|
||||
"do not retry the transfer: the browser may already have applied it; inspect the page or stop safely",
|
||||
),
|
||||
exit_code: base.exit_code,
|
||||
},
|
||||
(ErrorCode::Timeout, reason::TRANSFER_TIMEOUT) => RenderInfo {
|
||||
summary: "the file transfer timed out after browser dispatch",
|
||||
hint: Some(
|
||||
"do not retry when effect_state is unknown or committed; inspect the page before taking another action",
|
||||
),
|
||||
exit_code: base.exit_code,
|
||||
},
|
||||
(ErrorCode::PermissionDenied, reason::ELEMENT_NOT_VISIBLE) => RenderInfo {
|
||||
summary: "target element has no visible geometry",
|
||||
hint: Some(
|
||||
@@ -222,17 +257,45 @@ pub fn info_for_error(code: ErrorCode, data: Option<&serde_json::Value>) -> Rend
|
||||
hint: Some("verify the CSS selector or wait for the element to appear before retrying"),
|
||||
exit_code: base.exit_code,
|
||||
},
|
||||
(ErrorCode::InvalidParams, reason::INVALID_SELECTOR) => RenderInfo {
|
||||
summary: "selector is not valid CSS",
|
||||
hint: Some(
|
||||
"use a standard CSS selector or a fresh @eN ref; Playwright text= selectors and XPath are not supported",
|
||||
),
|
||||
exit_code: base.exit_code,
|
||||
},
|
||||
(ErrorCode::InvalidParams, reason::TARGET_NOT_FILLABLE) => RenderInfo {
|
||||
summary: "target element is not fillable",
|
||||
hint: Some(
|
||||
"choose an input, textarea, or contenteditable element from the latest snapshot",
|
||||
"observe the page and choose an enabled, editable text input, textarea, or contenteditable; use the appropriate interaction for other control types",
|
||||
),
|
||||
exit_code: base.exit_code,
|
||||
},
|
||||
(ErrorCode::InvalidParams, reason::FILL_VALUE_INVALID) => RenderInfo {
|
||||
summary: "the requested text does not fit the field constraints",
|
||||
hint: Some(
|
||||
"check the field's input type and maxlength; use a compatible value without silently truncating or changing the user's intended data",
|
||||
),
|
||||
exit_code: base.exit_code,
|
||||
},
|
||||
(ErrorCode::CdpFailed, reason::FILL_TARGET_CHANGED) => RenderInfo {
|
||||
summary: "the fill target changed during the action",
|
||||
hint: Some(
|
||||
"observe the current page and field value; if the intended result is missing, use a fresh editable target before retrying",
|
||||
),
|
||||
exit_code: base.exit_code,
|
||||
},
|
||||
(ErrorCode::CdpFailed, reason::FILL_FOCUS_LOST) => RenderInfo {
|
||||
summary: "the page moved focus away from the fill target",
|
||||
hint: Some(
|
||||
"observe the page for a dialog or rerender and resolve it before retrying; fill manages field focus and does not require bringing the browser to the foreground",
|
||||
),
|
||||
exit_code: base.exit_code,
|
||||
},
|
||||
(ErrorCode::CdpFailed, reason::FILL_VALUE_MISMATCH) => RenderInfo {
|
||||
summary: "the fill result could not be confirmed",
|
||||
hint: Some(
|
||||
"observe the field before retrying; the page may have formatted the value. Continue if the visible result satisfies the user's intent; otherwise correct the remaining difference. Do not blindly repeat fill or immediately request human help",
|
||||
),
|
||||
exit_code: base.exit_code,
|
||||
},
|
||||
(ErrorCode::CdpFailed, reason::FILL_FAILED) => RenderInfo {
|
||||
summary: "fill encountered a browser or page-script error",
|
||||
hint: Some(
|
||||
"observe the page before retrying because the field may already have changed; retry only if the intended result is missing and the target is still available",
|
||||
),
|
||||
exit_code: base.exit_code,
|
||||
},
|
||||
@@ -288,6 +351,60 @@ pub fn info_for_error(code: ErrorCode, data: Option<&serde_json::Value>) -> Rend
|
||||
),
|
||||
exit_code: base.exit_code,
|
||||
},
|
||||
(ErrorCode::Unsupported, reason::FILE_INPUT_NOT_ACTIVATED) => RenderInfo {
|
||||
summary: "the upload trigger did not activate a file input",
|
||||
hint: Some(
|
||||
"the page may use a non-input picker such as showOpenFilePicker(); when effect_state is none, re-observe and use `--mode drop` once only for a reliably identified attachment-receiving area — otherwise use `bsk request-help` with the exact original local path",
|
||||
),
|
||||
exit_code: base.exit_code,
|
||||
},
|
||||
(ErrorCode::Timeout | ErrorCode::CdpFailed, reason::FILE_INPUT_PROBE_FAILED) => {
|
||||
RenderInfo {
|
||||
summary: "the browser upload transaction could not be established",
|
||||
hint: Some(
|
||||
"do not retry the same upload blindly; use `bsk request-help` when human help is available",
|
||||
),
|
||||
exit_code: base.exit_code,
|
||||
}
|
||||
}
|
||||
(ErrorCode::Timeout | ErrorCode::CdpFailed, reason::SET_FILE_INPUT_FAILED) => RenderInfo {
|
||||
summary: "the browser could not attach the staged file to the input",
|
||||
hint: Some(
|
||||
"check that BrowserSkill has Chrome's 'Allow access to file URLs' permission; otherwise use `bsk request-help`",
|
||||
),
|
||||
exit_code: base.exit_code,
|
||||
},
|
||||
(ErrorCode::Unsupported, reason::UPLOAD_MECHANISM_UNSUPPORTED) => RenderInfo {
|
||||
summary: "the browser does not support native file-drop upload",
|
||||
hint: Some(
|
||||
"do not retry the same drop; use a standard file-input target or `bsk request-help`",
|
||||
),
|
||||
exit_code: base.exit_code,
|
||||
},
|
||||
(
|
||||
ErrorCode::PermissionDenied | ErrorCode::Timeout | ErrorCode::CdpFailed,
|
||||
reason::FILE_DROP_TARGET_UNAVAILABLE,
|
||||
) => RenderInfo {
|
||||
summary: "the file-drop target could not be safely resolved",
|
||||
hint: Some(
|
||||
"rerun observe and select the attachment-receiving editor or drop zone itself; do not guess another target — no file was delivered when effect_state is none",
|
||||
),
|
||||
exit_code: base.exit_code,
|
||||
},
|
||||
(ErrorCode::Timeout | ErrorCode::CdpFailed, reason::FILE_DROP_FAILED) => RenderInfo {
|
||||
summary: "the browser could not complete the native file drop",
|
||||
hint: Some(
|
||||
"do not retry when effect_state is unknown; observe the page first, otherwise use `bsk request-help`",
|
||||
),
|
||||
exit_code: base.exit_code,
|
||||
},
|
||||
(ErrorCode::CdpFailed, reason::DOWNLOAD_CAPTURE_FAILED) => RenderInfo {
|
||||
summary: "the browser download could not be attributed or completed",
|
||||
hint: Some(
|
||||
"do not retry blindly; the target browser may not expose a reliable download intent signal",
|
||||
),
|
||||
exit_code: base.exit_code,
|
||||
},
|
||||
_ => base,
|
||||
}
|
||||
}
|
||||
@@ -373,6 +490,56 @@ mod tests {
|
||||
ErrorCode::NoBrowserConnected,
|
||||
];
|
||||
|
||||
#[test]
|
||||
fn fill_reasons_provide_recovery_without_changing_exit_codes() {
|
||||
for (code, reason, hint_fragment) in [
|
||||
(
|
||||
ErrorCode::InvalidParams,
|
||||
reason::TARGET_NOT_FILLABLE,
|
||||
"enabled, editable",
|
||||
),
|
||||
(
|
||||
ErrorCode::InvalidParams,
|
||||
reason::FILL_VALUE_INVALID,
|
||||
"maxlength",
|
||||
),
|
||||
(
|
||||
ErrorCode::CdpFailed,
|
||||
reason::FILL_TARGET_CHANGED,
|
||||
"fresh editable target",
|
||||
),
|
||||
(
|
||||
ErrorCode::CdpFailed,
|
||||
reason::FILL_FOCUS_LOST,
|
||||
"does not require bringing the browser",
|
||||
),
|
||||
(
|
||||
ErrorCode::CdpFailed,
|
||||
reason::FILL_VALUE_MISMATCH,
|
||||
"Continue if the visible result satisfies",
|
||||
),
|
||||
(
|
||||
ErrorCode::CdpFailed,
|
||||
reason::FILL_FAILED,
|
||||
"may already have changed",
|
||||
),
|
||||
] {
|
||||
let data = serde_json::json!({ "reason": reason });
|
||||
let info = info_for_error(code, Some(&data));
|
||||
assert!(info.hint.unwrap().contains(hint_fragment), "{reason}");
|
||||
assert_eq!(info.exit_code, info_for(code).exit_code);
|
||||
}
|
||||
let mismatch = serde_json::json!({ "reason": reason::FILL_VALUE_MISMATCH });
|
||||
let info = info_for_error(ErrorCode::CdpFailed, Some(&mismatch));
|
||||
assert_eq!(info.summary, "the fill result could not be confirmed");
|
||||
assert!(info.hint.unwrap().contains("Do not blindly repeat fill"));
|
||||
let unknown = serde_json::json!({ "reason": "future_fill_reason" });
|
||||
assert_eq!(
|
||||
info_for_error(ErrorCode::CdpFailed, Some(&unknown)),
|
||||
info_for(ErrorCode::CdpFailed)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn element_not_visible_overrides_permission_denied_copy() {
|
||||
let data = serde_json::json!({ "reason": reason::ELEMENT_NOT_VISIBLE });
|
||||
@@ -385,6 +552,18 @@ mod tests {
|
||||
assert!(!info.summary.contains("sandbox"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extension_access_denied_explains_the_frame_restriction_and_navigation_recovery() {
|
||||
let data = serde_json::json!({ "reason": reason::CDP_EXTENSION_ACCESS_DENIED });
|
||||
let info = info_for_error(ErrorCode::CdpFailed, Some(&data));
|
||||
assert!(info.summary.contains("another extension"));
|
||||
let hint = info.hint.unwrap();
|
||||
assert!(hint.contains("restricted extension frame"));
|
||||
assert!(hint.contains("bsk navigate <url>"));
|
||||
assert!(hint.contains("disable the conflicting extension"));
|
||||
assert_eq!(info.exit_code, 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn session_busy_overrides_timeout_copy() {
|
||||
let data = serde_json::json!({ "reason": reason::SESSION_BUSY });
|
||||
@@ -427,6 +606,64 @@ mod tests {
|
||||
assert_eq!(info.exit_code, 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn file_transfer_reasons_render_actionable_fallbacks() {
|
||||
let unsupported = serde_json::json!({ "reason": reason::FILE_INPUT_NOT_ACTIVATED });
|
||||
let info = info_for_error(ErrorCode::Unsupported, Some(&unsupported));
|
||||
assert_eq!(
|
||||
info.summary,
|
||||
"the upload trigger did not activate a file input"
|
||||
);
|
||||
let hint = info.hint.unwrap();
|
||||
assert!(hint.contains("--mode drop"));
|
||||
assert!(hint.contains("reliably identified attachment-receiving area"));
|
||||
assert!(hint.contains("exact original local path"));
|
||||
|
||||
let control = serde_json::json!({ "reason": reason::FILE_INPUT_PROBE_FAILED });
|
||||
let info = info_for_error(ErrorCode::Timeout, Some(&control));
|
||||
assert!(
|
||||
info.summary
|
||||
.contains("transaction could not be established")
|
||||
);
|
||||
assert!(info.hint.unwrap().contains("do not retry"));
|
||||
|
||||
let unsupported_drop =
|
||||
serde_json::json!({ "reason": reason::UPLOAD_MECHANISM_UNSUPPORTED });
|
||||
let info = info_for_error(ErrorCode::Unsupported, Some(&unsupported_drop));
|
||||
assert_eq!(
|
||||
info.summary,
|
||||
"the browser does not support native file-drop upload"
|
||||
);
|
||||
assert!(info.hint.unwrap().contains("standard file-input"));
|
||||
|
||||
let unavailable_target =
|
||||
serde_json::json!({ "reason": reason::FILE_DROP_TARGET_UNAVAILABLE });
|
||||
let info = info_for_error(ErrorCode::PermissionDenied, Some(&unavailable_target));
|
||||
assert_eq!(
|
||||
info.summary,
|
||||
"the file-drop target could not be safely resolved"
|
||||
);
|
||||
assert!(info.hint.unwrap().contains("effect_state is none"));
|
||||
|
||||
let failed_drop = serde_json::json!({ "reason": reason::FILE_DROP_FAILED });
|
||||
let info = info_for_error(ErrorCode::Timeout, Some(&failed_drop));
|
||||
assert!(info.summary.contains("native file drop"));
|
||||
assert!(info.hint.unwrap().contains("effect_state is unknown"));
|
||||
|
||||
let download = serde_json::json!({ "reason": reason::DOWNLOAD_CAPTURE_FAILED });
|
||||
let info = info_for_error(ErrorCode::CdpFailed, Some(&download));
|
||||
assert!(info.summary.contains("download could not be attributed"));
|
||||
|
||||
let unknown = serde_json::json!({ "reason": reason::TRANSFER_OUTCOME_UNKNOWN });
|
||||
let info = info_for_error(ErrorCode::ProtocolError, Some(&unknown));
|
||||
assert!(info.summary.contains("outcome could not be confirmed"));
|
||||
assert!(info.hint.unwrap().contains("do not retry"));
|
||||
|
||||
let timeout = serde_json::json!({ "reason": reason::TRANSFER_TIMEOUT });
|
||||
let info = info_for_error(ErrorCode::Timeout, Some(&timeout));
|
||||
assert!(info.summary.contains("timed out after browser dispatch"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ref_not_found_overrides_not_found_copy() {
|
||||
let data = serde_json::json!({ "reason": reason::REF_NOT_FOUND });
|
||||
|
||||
@@ -6,9 +6,7 @@ use std::path::{Path, PathBuf};
|
||||
use std::time::{Duration, SystemTime};
|
||||
|
||||
#[cfg(windows)]
|
||||
use std::os::windows::process::CommandExt;
|
||||
#[cfg(windows)]
|
||||
use std::process::{Command, Stdio};
|
||||
mod windows;
|
||||
|
||||
use anyhow::{Context, Result, bail};
|
||||
use clap::Args;
|
||||
@@ -363,7 +361,8 @@ fn install_candidate_with_client(
|
||||
crate::daemon::start::run_stop().context("stop bsk daemon before update")?;
|
||||
}
|
||||
|
||||
let action = replace_binary_for_update(&target, &binary, daemon_was_running)?;
|
||||
let restart_args = daemon_was_running.then(StartArgs::default);
|
||||
let action = replace_binary_for_update(&target, &binary, restart_args.as_ref())?;
|
||||
|
||||
if daemon_was_running && matches!(action, InstallAction::Replaced) {
|
||||
crate::daemon::start::run_start(StartArgs::default())
|
||||
@@ -394,8 +393,8 @@ pub(crate) fn download_candidate_binary(
|
||||
|
||||
/// Daemon-side install: download, verify, and replace the executable at
|
||||
/// `target` (on Windows: stage the replacement next to it). Unlike the
|
||||
/// CLI path this never stops or starts the daemon — the daemon drives
|
||||
/// its own restart once the binary is replaced.
|
||||
/// CLI path this never stops the daemon. On Windows the helper starts
|
||||
/// its replacement after exit; elsewhere the daemon drives its own restart.
|
||||
///
|
||||
/// `target` must be captured *before* any replacement happens: on Linux
|
||||
/// `std::env::current_exe` starts returning a ` (deleted)`-suffixed
|
||||
@@ -403,10 +402,11 @@ pub(crate) fn download_candidate_binary(
|
||||
pub(crate) fn self_install_candidate(
|
||||
candidate: &UpdateCandidate,
|
||||
target: &Path,
|
||||
restart_args: &StartArgs,
|
||||
) -> Result<InstallAction> {
|
||||
let client = update_http_client(ARCHIVE_FETCH_TIMEOUT)?;
|
||||
let binary = download_candidate_binary(candidate, &client)?;
|
||||
replace_binary_at_path(target, &binary)
|
||||
replace_binary_for_update(target, &binary, Some(restart_args))
|
||||
}
|
||||
|
||||
/// Outcome of one daemon auto-update step (see [`auto_update_step`]).
|
||||
@@ -423,8 +423,8 @@ pub(crate) enum AutoUpdateOutcome {
|
||||
/// The new binary replaced the old one; the daemon should now
|
||||
/// restart into it.
|
||||
Replaced { latest: String },
|
||||
/// Windows staged the replacement next to the running binary; a
|
||||
/// daemon/terminal restart finishes it.
|
||||
/// Windows started a helper that will replace the executable and
|
||||
/// restart the daemon once the current process exits.
|
||||
Staged { latest: String },
|
||||
}
|
||||
|
||||
@@ -668,6 +668,7 @@ fn render_report(format: Format, report: &UpdateReport) -> Result<()> {
|
||||
}
|
||||
if matches!(report.install_action, Some("staged")) {
|
||||
println!("the detached update helper will apply the replacement after exit");
|
||||
println!("if it fails, see the *.update-*.log file next to the executable");
|
||||
}
|
||||
}
|
||||
Format::Json => {
|
||||
@@ -688,22 +689,22 @@ pub fn extract_bsk_binary(archive_bytes: &[u8], kind: ArchiveKind) -> Result<Vec
|
||||
}
|
||||
|
||||
pub fn replace_binary_at_path(target: &Path, binary: &[u8]) -> Result<InstallAction> {
|
||||
replace_binary_for_update(target, binary, false)
|
||||
replace_binary_for_update(target, binary, None)
|
||||
}
|
||||
|
||||
fn replace_binary_for_update(
|
||||
target: &Path,
|
||||
binary: &[u8],
|
||||
restart_daemon: bool,
|
||||
restart_args: Option<&StartArgs>,
|
||||
) -> Result<InstallAction> {
|
||||
#[cfg(windows)]
|
||||
{
|
||||
stage_windows_replacement(target, binary, restart_daemon)
|
||||
stage_windows_replacement(target, binary, restart_args)
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
{
|
||||
let _ = restart_daemon;
|
||||
let _ = restart_args;
|
||||
replace_binary_atomically(target, binary)
|
||||
}
|
||||
}
|
||||
@@ -749,33 +750,69 @@ fn replace_binary_atomically(target: &Path, binary: &[u8]) -> Result<InstallActi
|
||||
fn stage_windows_replacement(
|
||||
target: &Path,
|
||||
binary: &[u8],
|
||||
restart_daemon: bool,
|
||||
restart_args: Option<&StartArgs>,
|
||||
) -> Result<InstallAction> {
|
||||
let paths = staged_replacement_paths(target, std::process::id())?;
|
||||
std::fs::write(&paths.binary_path, binary)
|
||||
.with_context(|| format!("write {}", paths.binary_path.display()))?;
|
||||
std::fs::write(
|
||||
&paths.script_path,
|
||||
windows_replacement_script(target, &paths.binary_path, restart_daemon),
|
||||
)
|
||||
.with_context(|| format!("write {}", paths.script_path.display()))?;
|
||||
// The helper owns replacement/restart after this process exits. Do not
|
||||
// report Staged until it has actually begun executing the script.
|
||||
let _child = spawn_windows_replacement(target, binary, restart_args, 120)?;
|
||||
Ok(InstallAction::Staged)
|
||||
}
|
||||
|
||||
const CREATE_NO_WINDOW: u32 = 0x0800_0000;
|
||||
let command = format!("\"{}\"", paths.script_path.display());
|
||||
if let Err(err) = Command::new("cmd.exe")
|
||||
.args(["/D", "/S", "/C"])
|
||||
.arg(command)
|
||||
.stdin(Stdio::null())
|
||||
.stdout(Stdio::null())
|
||||
.stderr(Stdio::null())
|
||||
.creation_flags(CREATE_NO_WINDOW)
|
||||
.spawn()
|
||||
{
|
||||
#[cfg(windows)]
|
||||
fn spawn_windows_replacement(
|
||||
target: &Path,
|
||||
binary: &[u8],
|
||||
restart_args: Option<&StartArgs>,
|
||||
attempts: u32,
|
||||
) -> Result<windows::Helper> {
|
||||
let paths = staged_replacement_paths(target, std::process::id())?;
|
||||
let ready_path = paths.script_path.with_extension("ready");
|
||||
let log_path = paths.script_path.with_extension("log");
|
||||
let result = (|| {
|
||||
// A previous failed attempt by this process must not acknowledge a
|
||||
// new helper. All paths are private to this target and process id.
|
||||
match std::fs::remove_file(&ready_path) {
|
||||
Ok(()) => {}
|
||||
Err(err) if err.kind() == std::io::ErrorKind::NotFound => {}
|
||||
Err(err) => return Err(err.into()),
|
||||
}
|
||||
std::fs::write(&paths.binary_path, binary)
|
||||
.with_context(|| format!("write {}", paths.binary_path.display()))?;
|
||||
std::fs::write(
|
||||
&paths.script_path,
|
||||
windows_replacement_script(restart_args, attempts),
|
||||
)
|
||||
.with_context(|| format!("write {}", paths.script_path.display()))?;
|
||||
let mut child = windows::spawn(
|
||||
&paths.script_path,
|
||||
&paths.binary_path,
|
||||
target,
|
||||
&ready_path,
|
||||
&log_path,
|
||||
)
|
||||
.context("launch detached Windows update helper")?;
|
||||
let deadline = std::time::Instant::now() + Duration::from_secs(5);
|
||||
loop {
|
||||
if std::fs::remove_file(&ready_path).is_ok() {
|
||||
return Ok(child);
|
||||
}
|
||||
if let Some(status) = child.try_wait()? {
|
||||
bail!("Windows update helper exited before becoming ready: {status}");
|
||||
}
|
||||
if std::time::Instant::now() >= deadline {
|
||||
let _ = child.kill();
|
||||
let _ = child.wait();
|
||||
bail!("Windows update helper did not become ready within 5 seconds");
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(20));
|
||||
}
|
||||
})();
|
||||
if result.is_err() {
|
||||
let _ = std::fs::remove_file(&paths.binary_path);
|
||||
let _ = std::fs::remove_file(&paths.script_path);
|
||||
return Err(err).context("launch detached Windows update helper");
|
||||
let _ = std::fs::remove_file(&ready_path);
|
||||
}
|
||||
Ok(InstallAction::Staged)
|
||||
result.with_context(|| format!("Windows update helper failed; see {}", log_path.display()))
|
||||
}
|
||||
|
||||
pub fn staged_replacement_paths(target: &Path, pid: u32) -> Result<StagedReplacementPaths> {
|
||||
@@ -792,25 +829,43 @@ pub fn staged_replacement_paths(target: &Path, pid: u32) -> Result<StagedReplace
|
||||
}
|
||||
|
||||
#[cfg(any(windows, test))]
|
||||
fn windows_replacement_script(target: &Path, staged_binary: &Path, restart_daemon: bool) -> String {
|
||||
let target = target.display().to_string().replace('%', "%%");
|
||||
let staged_binary = staged_binary.display().to_string().replace('%', "%%");
|
||||
let restart = if restart_daemon {
|
||||
format!("start \"\" /B \"{target}\" daemon start >nul 2>nul\r\n")
|
||||
} else {
|
||||
String::new()
|
||||
};
|
||||
fn windows_replacement_script(restart_args: Option<&StartArgs>, attempts: u32) -> String {
|
||||
// Only fixed switches and numeric values go into the ASCII script. Paths
|
||||
// stay in Unicode environment variables, including %, ! and shell symbols.
|
||||
let restart = restart_args.map_or_else(String::new, |args| {
|
||||
format!(
|
||||
"\"%BSK_UPDATE_TARGET%\" daemon start --port {} --session-idle {}ms --daemon-idle {}ms\r\nif errorlevel 1 goto failed_restart\r\n",
|
||||
args.resolved_port(),
|
||||
args.resolved_session_idle().as_millis(),
|
||||
args.resolved_daemon_idle().as_millis(),
|
||||
)
|
||||
});
|
||||
format!(
|
||||
"@echo off\r\n\
|
||||
setlocal\r\n\
|
||||
setlocal DisableDelayedExpansion\r\n\
|
||||
> \"%BSK_UPDATE_READY%\" echo ready\r\n\
|
||||
if errorlevel 1 exit /b 1\r\n\
|
||||
set /a attempts=0\r\n\
|
||||
:retry\r\n\
|
||||
move /Y \"{staged_binary}\" \"{target}\" >nul 2>nul\r\n\
|
||||
if errorlevel 1 (\r\n\
|
||||
timeout /t 1 /nobreak >nul\r\n\
|
||||
goto retry\r\n\
|
||||
)\r\n\
|
||||
move /Y \"%BSK_UPDATE_SOURCE%\" \"%BSK_UPDATE_TARGET%\" >nul\r\n\
|
||||
if not errorlevel 1 goto replaced\r\n\
|
||||
set /a attempts+=1\r\n\
|
||||
if %attempts% geq {attempts} goto failed_replace\r\n\
|
||||
if not exist \"%BSK_UPDATE_SOURCE%\" goto failed_replace\r\n\
|
||||
ping -n 2 127.0.0.1 >nul\r\n\
|
||||
goto retry\r\n\
|
||||
:failed_replace\r\n\
|
||||
echo Failed to replace the executable after %attempts% attempts.\r\n\
|
||||
goto failed\r\n\
|
||||
:replaced\r\n\
|
||||
{restart}\
|
||||
del /F /Q \"%~f0\" >nul 2>nul\r\n"
|
||||
del /F /Q \"%BSK_UPDATE_LOG%\" >nul 2>nul\r\n\
|
||||
(del /F /Q \"%BSK_UPDATE_SCRIPT%\" >nul 2>nul & exit 0)\r\n\
|
||||
:failed_restart\r\n\
|
||||
echo Replaced the executable but failed to restart the daemon.\r\n\
|
||||
:failed\r\n\
|
||||
del /F /Q \"%BSK_UPDATE_SOURCE%\" >nul 2>nul\r\n\
|
||||
(del /F /Q \"%BSK_UPDATE_SCRIPT%\" >nul 2>nul & exit 1)\r\n"
|
||||
)
|
||||
}
|
||||
|
||||
@@ -964,6 +1019,7 @@ mod tests {
|
||||
assert_eq!(extracted, b"windows binary");
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
#[test]
|
||||
fn replaces_binary_at_path() {
|
||||
let tmp = tempfile::TempDir::new().unwrap();
|
||||
@@ -998,29 +1054,139 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn windows_replacement_script_retries_restarts_and_cleans_itself() {
|
||||
let script = windows_replacement_script(
|
||||
Path::new(r"C:\Program Files\bsk.exe"),
|
||||
Path::new(r"C:\Program Files\bsk.exe.update-42"),
|
||||
true,
|
||||
fn windows_replacement_script_preserves_restart_config() {
|
||||
let args = StartArgs {
|
||||
port: Some(54321),
|
||||
session_idle: Some(Duration::from_millis(1234)),
|
||||
daemon_idle: Some(Duration::from_secs(75)),
|
||||
..Default::default()
|
||||
};
|
||||
let script = windows_replacement_script(Some(&args), 120);
|
||||
assert!(script.is_ascii());
|
||||
assert!(
|
||||
script
|
||||
.contains("daemon start --port 54321 --session-idle 1234ms --daemon-idle 75000ms")
|
||||
);
|
||||
|
||||
assert!(script.contains(":retry"));
|
||||
assert!(script.contains("move /Y"));
|
||||
assert!(script.contains(r#"start "" /B "C:\Program Files\bsk.exe" daemon start"#));
|
||||
assert!(script.contains(r#"del /F /Q "%~f0""#));
|
||||
assert!(!windows_replacement_script(None, 120).contains("daemon start"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn windows_replacement_script_omits_restart_when_daemon_was_not_running() {
|
||||
let script = windows_replacement_script(
|
||||
Path::new(r"C:\bsk.exe"),
|
||||
Path::new(r"C:\bsk.exe.update-42"),
|
||||
false,
|
||||
);
|
||||
#[cfg(windows)]
|
||||
fn wait_for_helper(mut child: windows::Helper) -> std::process::ExitStatus {
|
||||
let deadline = std::time::Instant::now() + Duration::from_secs(10);
|
||||
loop {
|
||||
if let Some(status) = child.try_wait().unwrap() {
|
||||
return status;
|
||||
}
|
||||
if std::time::Instant::now() >= deadline {
|
||||
let _ = child.kill();
|
||||
let _ = child.wait();
|
||||
panic!("Windows update helper did not exit");
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(20));
|
||||
}
|
||||
}
|
||||
|
||||
assert!(!script.contains("daemon start"));
|
||||
assert!(script.contains(r#"del /F /Q "%~f0""#));
|
||||
#[cfg(windows)]
|
||||
#[test]
|
||||
fn windows_helper_replaces_in_unicode_spaces_and_shell_symbol_paths() {
|
||||
let tmp = tempfile::TempDir::new().unwrap();
|
||||
for name in [
|
||||
"ascii",
|
||||
"with space",
|
||||
"中文目录",
|
||||
"literal %PATH% ! & (folder)",
|
||||
] {
|
||||
let dir = tmp.path().join(name);
|
||||
std::fs::create_dir(&dir).unwrap();
|
||||
let target = dir.join("bsk.exe");
|
||||
std::fs::write(&target, b"old binary").unwrap();
|
||||
let child = spawn_windows_replacement(&target, b"new binary", None, 5).unwrap();
|
||||
let paths = staged_replacement_paths(&target, std::process::id()).unwrap();
|
||||
let status = wait_for_helper(child);
|
||||
assert!(
|
||||
status.success(),
|
||||
"{name}: {:?}",
|
||||
std::fs::read_to_string(paths.script_path.with_extension("log"))
|
||||
);
|
||||
assert_eq!(std::fs::read(&target).unwrap(), b"new binary", "{name}");
|
||||
// Successful updates leave no helper, staged binary, ready file or log.
|
||||
assert_eq!(std::fs::read_dir(&dir).unwrap().count(), 1, "{name}");
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
#[test]
|
||||
fn windows_helper_waits_for_unlock_then_replaces() {
|
||||
use std::os::windows::fs::OpenOptionsExt;
|
||||
let tmp = tempfile::TempDir::new().unwrap();
|
||||
let target = tmp.path().join("bsk.exe");
|
||||
std::fs::write(&target, b"old binary").unwrap();
|
||||
// Deny delete sharing, like a running Windows executable.
|
||||
let lock = std::fs::OpenOptions::new()
|
||||
.read(true)
|
||||
.share_mode(1)
|
||||
.open(&target)
|
||||
.unwrap();
|
||||
let mut child = spawn_windows_replacement(&target, b"new binary", None, 5).unwrap();
|
||||
std::thread::sleep(Duration::from_millis(150));
|
||||
assert!(child.try_wait().unwrap().is_none());
|
||||
assert_eq!(std::fs::read(&target).unwrap(), b"old binary");
|
||||
drop(lock);
|
||||
assert!(wait_for_helper(child).success());
|
||||
assert_eq!(std::fs::read(&target).unwrap(), b"new binary");
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
#[test]
|
||||
fn windows_helper_bounds_retries_and_preserves_old_binary_on_failure() {
|
||||
use std::os::windows::fs::OpenOptionsExt;
|
||||
let tmp = tempfile::TempDir::new().unwrap();
|
||||
let target = tmp.path().join("bsk.exe");
|
||||
std::fs::write(&target, b"old binary").unwrap();
|
||||
let _lock = std::fs::OpenOptions::new()
|
||||
.read(true)
|
||||
.share_mode(1)
|
||||
.open(&target)
|
||||
.unwrap();
|
||||
let child = spawn_windows_replacement(&target, b"new binary", None, 2).unwrap();
|
||||
assert!(!wait_for_helper(child).success());
|
||||
let paths = staged_replacement_paths(&target, std::process::id()).unwrap();
|
||||
assert_eq!(std::fs::read(&target).unwrap(), b"old binary");
|
||||
assert!(!paths.binary_path.exists());
|
||||
assert!(!paths.script_path.exists());
|
||||
let log_bytes = std::fs::read(paths.script_path.with_extension("log")).unwrap();
|
||||
let log = String::from_utf8_lossy(&log_bytes);
|
||||
assert!(
|
||||
log.contains("Failed to replace the executable after 2 attempts."),
|
||||
"{log}"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
#[test]
|
||||
fn windows_helper_reports_restart_failure_without_losing_replacement() {
|
||||
let tmp = tempfile::TempDir::new().unwrap();
|
||||
let target = tmp.path().join("bsk.exe");
|
||||
std::fs::write(&target, b"old binary").unwrap();
|
||||
// Not a valid PE executable: replacement succeeds but restart fails.
|
||||
let child = spawn_windows_replacement(
|
||||
&target,
|
||||
b"invalid executable",
|
||||
Some(&StartArgs::default()),
|
||||
2,
|
||||
)
|
||||
.unwrap();
|
||||
assert!(!wait_for_helper(child).success());
|
||||
let paths = staged_replacement_paths(&target, std::process::id()).unwrap();
|
||||
let bytes = std::fs::read(paths.script_path.with_extension("log")).unwrap();
|
||||
let log = String::from_utf8_lossy(&bytes);
|
||||
assert!(
|
||||
log.contains("Replaced the executable but failed to restart the daemon."),
|
||||
"{log}"
|
||||
);
|
||||
assert_eq!(std::fs::read(&target).unwrap(), b"invalid executable");
|
||||
assert!(!paths.script_path.exists());
|
||||
assert!(!paths.binary_path.exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1279,10 +1445,9 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn auto_update_step_staged_outcome_requests_no_restart() {
|
||||
// The Windows shape: the replacement is staged next to the
|
||||
// running binary, so the outcome must not ask for the immediate
|
||||
// self-restart that `Replaced` triggers.
|
||||
fn auto_update_step_reports_staged_handoff() {
|
||||
// The helper, rather than this process, starts the new executable.
|
||||
// The daemon must exit to let that staged handoff finish.
|
||||
let candidate = test_candidate();
|
||||
let outcome =
|
||||
auto_update_step(Some(&candidate), true, 0, |_| Ok(InstallAction::Staged)).unwrap();
|
||||
|
||||
@@ -0,0 +1,203 @@
|
||||
//! Launch the update helper with only its stdio handles inherited.
|
||||
//!
|
||||
//! std::process::Command on Windows also inherits other inheritable handles.
|
||||
//! A daemon's listening socket must not survive in the helper: it would keep
|
||||
//! the port occupied after the daemon exits and prevent its replacement starting.
|
||||
|
||||
use std::ffi::OsStr;
|
||||
use std::fs::File;
|
||||
use std::io;
|
||||
use std::os::windows::ffi::OsStrExt;
|
||||
use std::os::windows::io::{AsRawHandle, FromRawHandle, OwnedHandle};
|
||||
use std::os::windows::process::ExitStatusExt;
|
||||
use std::path::Path;
|
||||
use std::process::ExitStatus;
|
||||
|
||||
use windows_sys::Win32::Foundation::{
|
||||
HANDLE, HANDLE_FLAG_INHERIT, SetHandleInformation, WAIT_OBJECT_0, WAIT_TIMEOUT,
|
||||
};
|
||||
use windows_sys::Win32::System::Threading::{
|
||||
CREATE_NEW_PROCESS_GROUP, CREATE_NO_WINDOW, CREATE_UNICODE_ENVIRONMENT, CreateProcessW,
|
||||
DeleteProcThreadAttributeList, EXTENDED_STARTUPINFO_PRESENT, GetExitCodeProcess,
|
||||
InitializeProcThreadAttributeList, PROC_THREAD_ATTRIBUTE_HANDLE_LIST, PROCESS_INFORMATION,
|
||||
STARTF_USESTDHANDLES, STARTUPINFOEXW, TerminateProcess, UpdateProcThreadAttribute,
|
||||
WaitForSingleObject,
|
||||
};
|
||||
|
||||
pub(super) struct Helper(OwnedHandle);
|
||||
|
||||
impl Helper {
|
||||
pub(super) fn try_wait(&mut self) -> io::Result<Option<ExitStatus>> {
|
||||
// SAFETY: the owned process handle remains valid for both calls.
|
||||
match unsafe { WaitForSingleObject(self.0.as_raw_handle(), 0) } {
|
||||
WAIT_OBJECT_0 => {
|
||||
let mut code = 0;
|
||||
if unsafe { GetExitCodeProcess(self.0.as_raw_handle(), &mut code) } == 0 {
|
||||
return Err(io::Error::last_os_error());
|
||||
}
|
||||
Ok(Some(ExitStatus::from_raw(code)))
|
||||
}
|
||||
WAIT_TIMEOUT => Ok(None),
|
||||
_ => Err(io::Error::last_os_error()),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn kill(&mut self) -> io::Result<()> {
|
||||
// SAFETY: this handle belongs to the helper we created.
|
||||
if unsafe { TerminateProcess(self.0.as_raw_handle(), 1) } == 0 {
|
||||
return Err(io::Error::last_os_error());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn wait(&mut self) -> io::Result<()> {
|
||||
// Only used after kill(); do not leave a live helper on startup failure.
|
||||
if unsafe { WaitForSingleObject(self.0.as_raw_handle(), 5000) } != WAIT_OBJECT_0 {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::TimedOut,
|
||||
"update helper did not exit",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
// The opaque attribute list needs pointer-aligned storage and explicit teardown.
|
||||
struct AttributeList(Vec<usize>);
|
||||
|
||||
impl Drop for AttributeList {
|
||||
fn drop(&mut self) {
|
||||
// SAFETY: this wrapper is constructed only after successful initialization.
|
||||
unsafe { DeleteProcThreadAttributeList(self.0.as_mut_ptr().cast()) };
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn spawn(
|
||||
script: &Path,
|
||||
source: &Path,
|
||||
target: &Path,
|
||||
ready: &Path,
|
||||
log_path: &Path,
|
||||
) -> io::Result<Helper> {
|
||||
let root = std::env::var_os("SystemRoot")
|
||||
.ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, "SystemRoot is not set"))?;
|
||||
let application = wide(Path::new(&root).join("System32/cmd.exe").as_os_str());
|
||||
// /S /C strips one outer pair of quotes. Environment expansion keeps paths
|
||||
// Unicode and avoids CRT escaping, batch-file encoding, and CALL expansion.
|
||||
let mut command = wide(OsStr::new(
|
||||
"cmd.exe /D /V:OFF /S /C \"\"%BSK_UPDATE_SCRIPT%\"\"",
|
||||
));
|
||||
let overrides = [
|
||||
("BSK_UPDATE_SCRIPT", script.as_os_str()),
|
||||
("BSK_UPDATE_SOURCE", source.as_os_str()),
|
||||
("BSK_UPDATE_TARGET", target.as_os_str()),
|
||||
("BSK_UPDATE_READY", ready.as_os_str()),
|
||||
("BSK_UPDATE_LOG", log_path.as_os_str()),
|
||||
];
|
||||
let mut env: Vec<_> = std::env::vars_os()
|
||||
.filter(|(key, _)| {
|
||||
let key = key.to_string_lossy();
|
||||
!overrides
|
||||
.iter()
|
||||
.any(|(name, _)| key.eq_ignore_ascii_case(name))
|
||||
&& !key.eq_ignore_ascii_case(crate::daemon::start::DAEMONIZED_ENV)
|
||||
&& !key.eq_ignore_ascii_case(crate::daemon::start::DAEMON_REPLACEMENT_WAIT_ENV)
|
||||
})
|
||||
.collect();
|
||||
env.extend(
|
||||
overrides
|
||||
.into_iter()
|
||||
.map(|(key, value)| (key.into(), value.to_owned())),
|
||||
);
|
||||
env.sort_by_key(|(key, _)| key.to_string_lossy().to_uppercase());
|
||||
let mut environment = Vec::new();
|
||||
for (key, value) in env {
|
||||
environment.extend(key.encode_wide());
|
||||
environment.push(b'=' as u16);
|
||||
environment.extend(value.encode_wide());
|
||||
environment.push(0);
|
||||
}
|
||||
environment.push(0);
|
||||
|
||||
let input = File::open("NUL")?;
|
||||
let log = File::create(log_path)?;
|
||||
let handles: [HANDLE; 2] = [input.as_raw_handle(), log.as_raw_handle()];
|
||||
for &handle in &handles {
|
||||
// SAFETY: these are dedicated helper stdio handles, held until spawn
|
||||
// finishes. The explicit handle list excludes every other parent handle.
|
||||
if unsafe { SetHandleInformation(handle, HANDLE_FLAG_INHERIT, HANDLE_FLAG_INHERIT) } == 0 {
|
||||
return Err(io::Error::last_os_error());
|
||||
}
|
||||
}
|
||||
|
||||
let mut size = 0;
|
||||
// SAFETY: the first call queries the allocation size, as required by Win32.
|
||||
unsafe { InitializeProcThreadAttributeList(std::ptr::null_mut(), 1, 0, &mut size) };
|
||||
if size == 0 {
|
||||
return Err(io::Error::last_os_error());
|
||||
}
|
||||
let mut storage = vec![0usize; size.div_ceil(std::mem::size_of::<usize>())];
|
||||
// SAFETY: storage has the requested size and pointer alignment.
|
||||
if unsafe { InitializeProcThreadAttributeList(storage.as_mut_ptr().cast(), 1, 0, &mut size) }
|
||||
== 0
|
||||
{
|
||||
return Err(io::Error::last_os_error());
|
||||
}
|
||||
let mut attributes = AttributeList(storage);
|
||||
// SAFETY: the handle array and attribute storage outlive CreateProcessW.
|
||||
if unsafe {
|
||||
UpdateProcThreadAttribute(
|
||||
attributes.0.as_mut_ptr().cast(),
|
||||
0,
|
||||
PROC_THREAD_ATTRIBUTE_HANDLE_LIST as usize,
|
||||
handles.as_ptr().cast(),
|
||||
std::mem::size_of_val(&handles),
|
||||
std::ptr::null_mut(),
|
||||
std::ptr::null(),
|
||||
)
|
||||
} == 0
|
||||
{
|
||||
return Err(io::Error::last_os_error());
|
||||
}
|
||||
// SAFETY: these Win32 structs permit zero initialization; required fields
|
||||
// are populated below before the process is created.
|
||||
let mut startup: STARTUPINFOEXW = unsafe { std::mem::zeroed() };
|
||||
startup.StartupInfo.cb = std::mem::size_of::<STARTUPINFOEXW>() as u32;
|
||||
startup.StartupInfo.dwFlags = STARTF_USESTDHANDLES;
|
||||
startup.StartupInfo.hStdInput = handles[0];
|
||||
startup.StartupInfo.hStdOutput = handles[1];
|
||||
startup.StartupInfo.hStdError = handles[1];
|
||||
startup.lpAttributeList = attributes.0.as_mut_ptr().cast();
|
||||
let mut process: PROCESS_INFORMATION = unsafe { std::mem::zeroed() };
|
||||
// SAFETY: strings are NUL-terminated UTF-16, the environment is double-NUL
|
||||
// terminated, and all buffers/handles remain alive for the duration of spawn.
|
||||
if unsafe {
|
||||
CreateProcessW(
|
||||
application.as_ptr(),
|
||||
command.as_mut_ptr(),
|
||||
std::ptr::null(),
|
||||
std::ptr::null(),
|
||||
1,
|
||||
CREATE_NO_WINDOW
|
||||
| CREATE_NEW_PROCESS_GROUP
|
||||
| CREATE_UNICODE_ENVIRONMENT
|
||||
| EXTENDED_STARTUPINFO_PRESENT,
|
||||
environment.as_ptr().cast(),
|
||||
std::ptr::null(),
|
||||
&startup.StartupInfo,
|
||||
&mut process,
|
||||
)
|
||||
} == 0
|
||||
{
|
||||
return Err(io::Error::last_os_error());
|
||||
}
|
||||
// SAFETY: successful CreateProcessW transfers these two handles to us.
|
||||
let _thread = unsafe { OwnedHandle::from_raw_handle(process.hThread) };
|
||||
Ok(Helper(unsafe {
|
||||
OwnedHandle::from_raw_handle(process.hProcess)
|
||||
}))
|
||||
}
|
||||
|
||||
fn wide(value: &OsStr) -> Vec<u16> {
|
||||
value.encode_wide().chain(Some(0)).collect()
|
||||
}
|
||||
@@ -0,0 +1,202 @@
|
||||
//! `bsk upload` — stage caller-readable files and attach them to a page.
|
||||
|
||||
use std::fs::File;
|
||||
use std::io::Read;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::Context;
|
||||
use base64::Engine;
|
||||
use bsk_protocol::Method;
|
||||
use bsk_protocol::tools::{
|
||||
TransferBeginParams, TransferBeginResult, TransferChunkParams, TransferChunkResult,
|
||||
TransferIdParams, TransferReadyResult, UploadFile, UploadMode, UploadParams, UploadResult,
|
||||
};
|
||||
use clap::{Args, ValueEnum};
|
||||
|
||||
use crate::cli::ensure_daemon::ensure_daemon;
|
||||
use crate::cli::error::{CliError, Format};
|
||||
use crate::cli::interaction::split_target;
|
||||
use crate::cli::navigate::parse_timeout_ms;
|
||||
|
||||
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, ValueEnum)]
|
||||
pub enum UploadModeArg {
|
||||
#[default]
|
||||
Input,
|
||||
Drop,
|
||||
}
|
||||
|
||||
impl From<UploadModeArg> for UploadMode {
|
||||
fn from(value: UploadModeArg) -> Self {
|
||||
match value {
|
||||
UploadModeArg::Input => Self::Input,
|
||||
UploadModeArg::Drop => Self::Drop,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Args)]
|
||||
pub struct UploadArgs {
|
||||
/// Snapshot ref (`@e3`) or CSS selector for the chooser trigger / drop target.
|
||||
pub target: Option<String>,
|
||||
#[arg(long = "ref")]
|
||||
pub ref_: Option<String>,
|
||||
#[arg(long = "selector")]
|
||||
pub selector: Option<String>,
|
||||
/// Local file to upload. Repeat for a multiple-file input.
|
||||
#[arg(long = "file", required = true)]
|
||||
pub files: Vec<PathBuf>,
|
||||
/// Browser mechanism used to deliver the staged files.
|
||||
#[arg(long, value_enum, default_value_t)]
|
||||
pub mode: UploadModeArg,
|
||||
#[arg(long)]
|
||||
pub session: String,
|
||||
#[arg(long = "tab-id")]
|
||||
pub tab_id: Option<i64>,
|
||||
#[arg(long, default_value = "2m", value_parser = parse_timeout_ms)]
|
||||
pub timeout: u32,
|
||||
}
|
||||
|
||||
pub fn dispatch(args: UploadArgs, format: Format) -> Result<(), CliError> {
|
||||
if args.files.len() > crate::daemon::file_transfer::MAX_UPLOAD_FILES {
|
||||
return Err(CliError::Local(anyhow::anyhow!(
|
||||
"upload accepts at most {} files",
|
||||
crate::daemon::file_transfer::MAX_UPLOAD_FILES
|
||||
)));
|
||||
}
|
||||
let info = ensure_daemon().context("ensure daemon is running")?;
|
||||
let (ref_, selector) = split_target(args.target, args.ref_, args.selector)?;
|
||||
let mut staged = Vec::new();
|
||||
for path in &args.files {
|
||||
if let Err(err) = stage_file(&info.sock_path, &args.session, path, &mut staged) {
|
||||
let _ = crate::cli::business_rpc::release_transfers(
|
||||
&info.sock_path,
|
||||
staged.iter().map(|(id, _)| id.as_str()),
|
||||
);
|
||||
return Err(err);
|
||||
}
|
||||
}
|
||||
let params = UploadParams {
|
||||
session_id: args.session,
|
||||
ref_,
|
||||
selector,
|
||||
tab_id: args.tab_id,
|
||||
files: staged
|
||||
.iter()
|
||||
.map(|(id, name)| UploadFile {
|
||||
transfer_id: id.clone(),
|
||||
name: name.clone(),
|
||||
staged_path: None,
|
||||
})
|
||||
.collect(),
|
||||
mode: args.mode.into(),
|
||||
timeout_ms: Some(args.timeout),
|
||||
};
|
||||
let result = crate::cli::business_rpc::call::<_, UploadResult>(
|
||||
info.sock_path.clone(),
|
||||
"upload",
|
||||
Method::ToolUpload,
|
||||
Some(params),
|
||||
ipc_timeout(args.timeout),
|
||||
);
|
||||
// The helper can reject cancellation before handing any request to the
|
||||
// daemon. In that case staging still belongs to this CLI.
|
||||
if result.as_ref().is_err_and(|err| {
|
||||
err.data()
|
||||
.is_some_and(|data| data["reason"] == "cancelled_before_dispatch")
|
||||
}) {
|
||||
let _ = crate::cli::business_rpc::release_transfers(
|
||||
&info.sock_path,
|
||||
staged.iter().map(|(id, _)| id.as_str()),
|
||||
);
|
||||
}
|
||||
// Once tool.upload is dispatched, staging ownership belongs to the
|
||||
// session. A transport timeout cannot prove that Chrome did not attach
|
||||
// the file, so releasing here could invalidate a late successful attach.
|
||||
// Session teardown remains the single cleanup boundary after dispatch.
|
||||
let reply = result?;
|
||||
match format {
|
||||
Format::Json => println!("{}", serde_json::to_string_pretty(&reply).unwrap()),
|
||||
Format::Human => println!(
|
||||
"upload ok tab={} files={}",
|
||||
reply.tab_id,
|
||||
reply.file_names.join(", ")
|
||||
),
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn stage_file(
|
||||
sock: &Path,
|
||||
session: &str,
|
||||
path: &PathBuf,
|
||||
staged: &mut Vec<(String, String)>,
|
||||
) -> Result<(), CliError> {
|
||||
let mut file = File::open(path)
|
||||
.with_context(|| format!("open upload file {}", path.display()))
|
||||
.map_err(CliError::Local)?;
|
||||
let meta = file
|
||||
.metadata()
|
||||
.with_context(|| format!("read upload file metadata {}", path.display()))
|
||||
.map_err(CliError::Local)?;
|
||||
if !meta.is_file() {
|
||||
return Err(CliError::Local(anyhow::anyhow!(
|
||||
"upload source is not a regular file: {}",
|
||||
path.display()
|
||||
)));
|
||||
}
|
||||
let name = path
|
||||
.file_name()
|
||||
.and_then(|v| v.to_str())
|
||||
.filter(|v| !v.is_empty())
|
||||
.ok_or_else(|| CliError::Local(anyhow::anyhow!("upload file has no valid basename")))?
|
||||
.to_string();
|
||||
let begin: TransferBeginResult = crate::cli::business_rpc::call(
|
||||
sock.to_path_buf(),
|
||||
"transfer-begin",
|
||||
Method::TransferBegin,
|
||||
Some(TransferBeginParams {
|
||||
session_id: session.to_string(),
|
||||
name: name.clone(),
|
||||
byte_size: meta.len(),
|
||||
}),
|
||||
Duration::from_secs(10),
|
||||
)?;
|
||||
// Record ownership immediately so one error path releases both completed
|
||||
// files and the current partially staged file.
|
||||
staged.push((begin.transfer_id.clone(), name));
|
||||
let mut offset = 0u64;
|
||||
let mut buf = vec![0u8; begin.chunk_size as usize];
|
||||
loop {
|
||||
let n = file.read(&mut buf).map_err(|e| CliError::Local(e.into()))?;
|
||||
if n == 0 {
|
||||
break;
|
||||
}
|
||||
let reply: TransferChunkResult = crate::cli::business_rpc::call(
|
||||
sock.to_path_buf(),
|
||||
"transfer-chunk",
|
||||
Method::TransferChunk,
|
||||
Some(TransferChunkParams {
|
||||
transfer_id: begin.transfer_id.clone(),
|
||||
offset,
|
||||
data_base64: base64::engine::general_purpose::STANDARD.encode(&buf[..n]),
|
||||
}),
|
||||
Duration::from_secs(30),
|
||||
)?;
|
||||
offset = reply.next_offset;
|
||||
}
|
||||
let _: TransferReadyResult = crate::cli::business_rpc::call(
|
||||
sock.to_path_buf(),
|
||||
"transfer-finish",
|
||||
Method::TransferFinish,
|
||||
Some(TransferIdParams {
|
||||
transfer_id: begin.transfer_id.clone(),
|
||||
}),
|
||||
Duration::from_secs(10),
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn ipc_timeout(timeout_ms: u32) -> Duration {
|
||||
Duration::from_millis(u64::from(timeout_ms) + 5_000)
|
||||
}
|
||||
@@ -223,14 +223,6 @@ impl BrowserRegistry {
|
||||
"browser reconnect: replacing previous registration"
|
||||
);
|
||||
}
|
||||
tracing::info!(
|
||||
diagnostic = true,
|
||||
event = "browser.registry.insert",
|
||||
id = %client.id,
|
||||
generation = client.generation,
|
||||
previous_generation = guard.get(&client.id).map(|previous| previous.generation),
|
||||
"diagnostic browser registry mutation"
|
||||
);
|
||||
guard.insert(client.id.clone(), client);
|
||||
}
|
||||
|
||||
@@ -247,31 +239,16 @@ impl BrowserRegistry {
|
||||
/// took over the same `id` while the old socket was tearing down
|
||||
/// (review M4/M5 round 2 Important #1). Returns the removed entry
|
||||
/// when a removal actually happened.
|
||||
#[track_caller]
|
||||
pub fn remove_if_generation_matches(
|
||||
&self,
|
||||
id: &BrowserId,
|
||||
generation: u64,
|
||||
) -> Option<std::sync::Arc<BrowserClient>> {
|
||||
let caller = std::panic::Location::caller();
|
||||
let mut guard = self.inner.lock().expect("browser registry poisoned");
|
||||
let current_generation = guard.get(id).map(|current| current.generation);
|
||||
let removed = match guard.get(id) {
|
||||
match guard.get(id) {
|
||||
Some(current) if current.generation == generation => guard.remove(id),
|
||||
_ => None,
|
||||
};
|
||||
tracing::info!(
|
||||
diagnostic = true,
|
||||
event = "browser.registry.remove_if_generation_matches",
|
||||
id = %id,
|
||||
requested_generation = generation,
|
||||
?current_generation,
|
||||
removed = removed.is_some(),
|
||||
caller_file = caller.file(),
|
||||
caller_line = caller.line(),
|
||||
"diagnostic browser registry mutation"
|
||||
);
|
||||
removed
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get(&self, id: &BrowserId) -> Option<std::sync::Arc<BrowserClient>> {
|
||||
|
||||
@@ -0,0 +1,728 @@
|
||||
//! Session-scoped, content-transparent staging for upload/download tools.
|
||||
//!
|
||||
//! The registry deliberately exposes no arbitrary-path operations. Upload
|
||||
//! bytes arrive from the invoking CLI in bounded chunks; browser downloads
|
||||
//! land in a daemon-minted directory and are read back by the invoking CLI.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::fs::{self, File, OpenOptions};
|
||||
use std::io::{Read, Seek, SeekFrom, Write};
|
||||
use std::path::{Component, Path, PathBuf};
|
||||
use std::sync::Mutex;
|
||||
|
||||
use base64::Engine;
|
||||
use bsk_protocol::tools::{
|
||||
TransferBeginParams, TransferBeginResult, TransferChunkParams, TransferChunkResult,
|
||||
TransferIdParams, TransferReadyResult, TransferReleaseResult,
|
||||
};
|
||||
use bsk_protocol::{ErrorCode, RpcError};
|
||||
use uuid::Uuid;
|
||||
|
||||
use super::paths;
|
||||
|
||||
pub const TRANSFER_CHUNK_SIZE: u32 = 512 * 1024;
|
||||
pub const MAX_TRANSFER_BYTES: u64 = 512 * 1024 * 1024;
|
||||
pub const MAX_UPLOAD_FILES: usize = 20;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum Direction {
|
||||
Upload,
|
||||
Download,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct Entry {
|
||||
session_id: String,
|
||||
direction: Direction,
|
||||
path: PathBuf,
|
||||
expected_size: Option<u64>,
|
||||
written: u64,
|
||||
ready: bool,
|
||||
}
|
||||
|
||||
/// Browser download path after the daemon has verified that it belongs to
|
||||
/// the capability minted for this transfer. From this point onward the daemon
|
||||
/// owns cleanup on every error path; unvalidated arbitrary paths are never
|
||||
/// removed.
|
||||
#[derive(Debug)]
|
||||
struct ValidatedBrowserFile {
|
||||
path: PathBuf,
|
||||
parent: PathBuf,
|
||||
cleanup_armed: bool,
|
||||
}
|
||||
|
||||
impl ValidatedBrowserFile {
|
||||
fn validate(reported_path: &Path, transfer_id: &str) -> Result<Self, RpcError> {
|
||||
if !reported_path.is_absolute() {
|
||||
return Err(permission(
|
||||
"download path is outside its browser capability",
|
||||
));
|
||||
}
|
||||
let expected_parent = Path::new("BrowserSkill").join(transfer_id);
|
||||
let parent = reported_path
|
||||
.parent()
|
||||
.ok_or_else(|| permission("download path has no parent directory"))?;
|
||||
if !parent.ends_with(&expected_parent) {
|
||||
return Err(permission(
|
||||
"download escaped its browser capability directory",
|
||||
));
|
||||
}
|
||||
for path in [reported_path, parent] {
|
||||
if fs::symlink_metadata(path)
|
||||
.map_err(io_error)?
|
||||
.file_type()
|
||||
.is_symlink()
|
||||
{
|
||||
return Err(permission("download capability path contains a symlink"));
|
||||
}
|
||||
}
|
||||
Ok(Self {
|
||||
path: reported_path.to_path_buf(),
|
||||
parent: parent.to_path_buf(),
|
||||
cleanup_armed: true,
|
||||
})
|
||||
}
|
||||
|
||||
fn remove_source(mut self) -> Result<(), RpcError> {
|
||||
remove_if_present(&self.path).map_err(io_error)?;
|
||||
remove_dir_if_present(&self.parent).map_err(io_error)?;
|
||||
self.cleanup_armed = false;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for ValidatedBrowserFile {
|
||||
fn drop(&mut self) {
|
||||
if !self.cleanup_armed {
|
||||
return;
|
||||
}
|
||||
let _ = remove_if_present(&self.path);
|
||||
let _ = remove_dir_if_present(&self.parent);
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct DownloadStaging {
|
||||
pub transfer_id: String,
|
||||
pub browser_relative_dir: String,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct TransferRegistry {
|
||||
root: PathBuf,
|
||||
initialized: Mutex<bool>,
|
||||
entries: Mutex<HashMap<String, Entry>>,
|
||||
}
|
||||
|
||||
impl TransferRegistry {
|
||||
pub fn new() -> anyhow::Result<Self> {
|
||||
Ok(Self {
|
||||
root: paths::bsk_home()?.join("run").join("transfers"),
|
||||
initialized: Mutex::new(false),
|
||||
entries: Mutex::new(HashMap::new()),
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn at_root(root: PathBuf) -> anyhow::Result<Self> {
|
||||
let registry = Self {
|
||||
root,
|
||||
initialized: Mutex::new(false),
|
||||
entries: Mutex::new(HashMap::new()),
|
||||
};
|
||||
registry.ensure_root()?;
|
||||
Ok(registry)
|
||||
}
|
||||
|
||||
pub fn initialize(&self) -> anyhow::Result<()> {
|
||||
self.ensure_root()
|
||||
}
|
||||
|
||||
fn ensure_root(&self) -> anyhow::Result<()> {
|
||||
let mut initialized = self.initialized.lock().unwrap();
|
||||
if *initialized {
|
||||
return Ok(());
|
||||
}
|
||||
if self.root.exists() {
|
||||
// Transfers never survive a daemon; removing stale directories on
|
||||
// startup gives crash cleanup without a second persistence model.
|
||||
let _ = fs::remove_dir_all(&self.root);
|
||||
}
|
||||
fs::create_dir_all(&self.root)?;
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
fs::set_permissions(&self.root, fs::Permissions::from_mode(0o700))?;
|
||||
}
|
||||
*initialized = true;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn begin_upload(&self, p: TransferBeginParams) -> Result<TransferBeginResult, RpcError> {
|
||||
self.ensure_root()
|
||||
.map_err(|err| protocol_error(err.to_string()))?;
|
||||
let basename = safe_upload_basename(&p.name)?;
|
||||
if p.byte_size > MAX_TRANSFER_BYTES {
|
||||
return Err(invalid(format!(
|
||||
"file size {} exceeds transfer limit {}",
|
||||
p.byte_size, MAX_TRANSFER_BYTES
|
||||
)));
|
||||
}
|
||||
let mut entries = self.entries.lock().unwrap();
|
||||
let session_bytes: u64 = entries
|
||||
.values()
|
||||
.filter(|entry| entry.session_id == p.session_id)
|
||||
.map(|entry| entry.expected_size.unwrap_or(entry.written))
|
||||
.sum();
|
||||
if session_bytes.saturating_add(p.byte_size) > MAX_TRANSFER_BYTES {
|
||||
return Err(invalid(format!(
|
||||
"session transfer staging exceeds limit {}",
|
||||
MAX_TRANSFER_BYTES
|
||||
)));
|
||||
}
|
||||
let id = new_id();
|
||||
let dir = self.root.join(&id);
|
||||
fs::create_dir(&dir).map_err(io_error)?;
|
||||
set_private_dir(&dir).map_err(io_error)?;
|
||||
let path = dir.join(basename);
|
||||
let file = OpenOptions::new()
|
||||
.write(true)
|
||||
.create_new(true)
|
||||
.open(&path)
|
||||
.map_err(io_error)?;
|
||||
set_private_file(&file).map_err(io_error)?;
|
||||
entries.insert(
|
||||
id.clone(),
|
||||
Entry {
|
||||
session_id: p.session_id,
|
||||
direction: Direction::Upload,
|
||||
path,
|
||||
expected_size: Some(p.byte_size),
|
||||
written: 0,
|
||||
ready: false,
|
||||
},
|
||||
);
|
||||
Ok(TransferBeginResult {
|
||||
transfer_id: id,
|
||||
chunk_size: TRANSFER_CHUNK_SIZE,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn write_chunk(&self, p: TransferChunkParams) -> Result<TransferChunkResult, RpcError> {
|
||||
let bytes = base64::engine::general_purpose::STANDARD
|
||||
.decode(p.data_base64)
|
||||
.map_err(|e| invalid(format!("invalid transfer chunk: {e}")))?;
|
||||
if bytes.len() > TRANSFER_CHUNK_SIZE as usize {
|
||||
return Err(invalid("transfer chunk exceeds negotiated size"));
|
||||
}
|
||||
let mut entries = self.entries.lock().unwrap();
|
||||
let entry = entries
|
||||
.get_mut(&p.transfer_id)
|
||||
.ok_or_else(|| not_found("transfer not found"))?;
|
||||
if entry.direction != Direction::Upload || entry.ready {
|
||||
return Err(invalid("transfer is not writable"));
|
||||
}
|
||||
if p.offset != entry.written {
|
||||
return Err(invalid(format!(
|
||||
"non-sequential transfer offset: expected {}, got {}",
|
||||
entry.written, p.offset
|
||||
)));
|
||||
}
|
||||
let next = entry.written.saturating_add(bytes.len() as u64);
|
||||
if next > entry.expected_size.unwrap_or(MAX_TRANSFER_BYTES) {
|
||||
return Err(invalid("transfer exceeds declared byte size"));
|
||||
}
|
||||
let mut file = OpenOptions::new()
|
||||
.append(true)
|
||||
.open(&entry.path)
|
||||
.map_err(io_error)?;
|
||||
file.write_all(&bytes).map_err(io_error)?;
|
||||
entry.written = next;
|
||||
Ok(TransferChunkResult {
|
||||
next_offset: next,
|
||||
eof: false,
|
||||
data_base64: None,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn finish_upload(&self, p: TransferIdParams) -> Result<TransferReadyResult, RpcError> {
|
||||
let mut entries = self.entries.lock().unwrap();
|
||||
let entry = entries
|
||||
.get_mut(&p.transfer_id)
|
||||
.ok_or_else(|| not_found("transfer not found"))?;
|
||||
if entry.direction != Direction::Upload {
|
||||
return Err(invalid("transfer is not an upload"));
|
||||
}
|
||||
if entry.written != entry.expected_size.unwrap_or(entry.written) {
|
||||
return Err(invalid(format!(
|
||||
"incomplete transfer: expected {} bytes, received {}",
|
||||
entry.expected_size.unwrap_or(0),
|
||||
entry.written
|
||||
)));
|
||||
}
|
||||
entry.ready = true;
|
||||
Ok(TransferReadyResult {
|
||||
transfer_id: p.transfer_id,
|
||||
byte_size: entry.written,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn resolve_uploads(
|
||||
&self,
|
||||
session_id: &str,
|
||||
ids: &[String],
|
||||
) -> Result<Vec<PathBuf>, RpcError> {
|
||||
let entries = self.entries.lock().unwrap();
|
||||
ids.iter()
|
||||
.map(|id| {
|
||||
let entry = entries
|
||||
.get(id)
|
||||
.ok_or_else(|| not_found("upload transfer not found"))?;
|
||||
if entry.session_id != session_id
|
||||
|| entry.direction != Direction::Upload
|
||||
|| !entry.ready
|
||||
{
|
||||
return Err(permission(
|
||||
"upload transfer is outside this session or not ready",
|
||||
));
|
||||
}
|
||||
Ok(entry.path.clone())
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub fn begin_download(&self, session_id: &str) -> Result<DownloadStaging, RpcError> {
|
||||
self.ensure_root()
|
||||
.map_err(|err| protocol_error(err.to_string()))?;
|
||||
let id = new_id();
|
||||
let dir = self.root.join(&id);
|
||||
fs::create_dir(&dir).map_err(io_error)?;
|
||||
set_private_dir(&dir).map_err(io_error)?;
|
||||
self.entries.lock().unwrap().insert(
|
||||
id.clone(),
|
||||
Entry {
|
||||
session_id: session_id.to_string(),
|
||||
direction: Direction::Download,
|
||||
path: dir.clone(),
|
||||
expected_size: None,
|
||||
written: 0,
|
||||
ready: false,
|
||||
},
|
||||
);
|
||||
Ok(DownloadStaging {
|
||||
browser_relative_dir: format!("BrowserSkill/{id}"),
|
||||
transfer_id: id,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn import_download(&self, id: &str, reported_path: &Path) -> Result<u64, RpcError> {
|
||||
let mut entries = self.entries.lock().unwrap();
|
||||
let entry = entries
|
||||
.get_mut(id)
|
||||
.ok_or_else(|| not_found("download transfer not found"))?;
|
||||
if entry.direction != Direction::Download {
|
||||
return Err(permission("transfer is not a download capability"));
|
||||
}
|
||||
let browser_file = ValidatedBrowserFile::validate(reported_path, id)?;
|
||||
let canonical = browser_file.path.canonicalize().map_err(io_error)?;
|
||||
let meta = fs::metadata(&canonical).map_err(io_error)?;
|
||||
if !meta.is_file() || meta.len() > MAX_TRANSFER_BYTES {
|
||||
return Err(invalid(
|
||||
"download is not a regular file or exceeds the transfer limit",
|
||||
));
|
||||
}
|
||||
|
||||
let destination = entry.path.join("payload");
|
||||
let source = File::open(&canonical).map_err(io_error)?;
|
||||
let mut destination_file = OpenOptions::new()
|
||||
.write(true)
|
||||
.create_new(true)
|
||||
.open(&destination)
|
||||
.map_err(io_error)?;
|
||||
let copied = std::io::copy(
|
||||
&mut source.take(MAX_TRANSFER_BYTES.saturating_add(1)),
|
||||
&mut destination_file,
|
||||
)
|
||||
.map_err(io_error)?;
|
||||
if copied > MAX_TRANSFER_BYTES {
|
||||
let _ = fs::remove_file(&destination);
|
||||
return Err(invalid("download exceeds the transfer limit"));
|
||||
}
|
||||
entry.path = destination;
|
||||
entry.written = copied;
|
||||
entry.expected_size = Some(copied);
|
||||
entry.ready = true;
|
||||
browser_file.remove_source()?;
|
||||
Ok(copied)
|
||||
}
|
||||
|
||||
pub fn read_chunk(&self, p: TransferChunkParams) -> Result<TransferChunkResult, RpcError> {
|
||||
let entries = self.entries.lock().unwrap();
|
||||
let entry = entries
|
||||
.get(&p.transfer_id)
|
||||
.ok_or_else(|| not_found("transfer not found"))?;
|
||||
if entry.direction != Direction::Download || !entry.ready {
|
||||
return Err(invalid("transfer is not readable"));
|
||||
}
|
||||
if p.offset > entry.written {
|
||||
return Err(invalid("read offset is beyond transfer length"));
|
||||
}
|
||||
let mut file = File::open(&entry.path).map_err(io_error)?;
|
||||
file.seek(SeekFrom::Start(p.offset)).map_err(io_error)?;
|
||||
let mut buf = vec![0u8; TRANSFER_CHUNK_SIZE as usize];
|
||||
let count = file.read(&mut buf).map_err(io_error)?;
|
||||
buf.truncate(count);
|
||||
let next = p.offset + count as u64;
|
||||
Ok(TransferChunkResult {
|
||||
next_offset: next,
|
||||
eof: next >= entry.written,
|
||||
data_base64: Some(base64::engine::general_purpose::STANDARD.encode(buf)),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn release(&self, p: TransferIdParams) -> TransferReleaseResult {
|
||||
let entry = self.entries.lock().unwrap().remove(&p.transfer_id);
|
||||
if let Some(entry) = entry {
|
||||
let dir = if entry.path.is_dir() {
|
||||
entry.path
|
||||
} else {
|
||||
entry.path.parent().unwrap_or(&self.root).to_path_buf()
|
||||
};
|
||||
let _ = fs::remove_dir_all(dir);
|
||||
TransferReleaseResult { released: true }
|
||||
} else {
|
||||
TransferReleaseResult { released: false }
|
||||
}
|
||||
}
|
||||
|
||||
pub fn release_session(&self, session_id: &str) {
|
||||
let ids: Vec<String> = self
|
||||
.entries
|
||||
.lock()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.filter(|(_, entry)| entry.session_id == session_id)
|
||||
.map(|(id, _)| id.clone())
|
||||
.collect();
|
||||
for id in ids {
|
||||
self.release(TransferIdParams { transfer_id: id });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn new_id() -> String {
|
||||
format!("tr_{}", Uuid::new_v4().simple())
|
||||
}
|
||||
|
||||
fn set_private_dir(path: &Path) -> std::io::Result<()> {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
fs::set_permissions(path, fs::Permissions::from_mode(0o700))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn set_private_file(file: &File) -> std::io::Result<()> {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
file.set_permissions(fs::Permissions::from_mode(0o600))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn remove_if_present(path: &Path) -> std::io::Result<()> {
|
||||
match fs::remove_file(path) {
|
||||
Ok(()) => Ok(()),
|
||||
Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(()),
|
||||
Err(err) => Err(err),
|
||||
}
|
||||
}
|
||||
|
||||
fn remove_dir_if_present(path: &Path) -> std::io::Result<()> {
|
||||
match fs::remove_dir(path) {
|
||||
Ok(()) => Ok(()),
|
||||
Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(()),
|
||||
Err(err) => Err(err),
|
||||
}
|
||||
}
|
||||
|
||||
fn safe_upload_basename(name: &str) -> Result<&str, RpcError> {
|
||||
if name.is_empty() || name.contains(['/', '\\', '\0']) {
|
||||
return Err(invalid("upload name must be a safe basename"));
|
||||
}
|
||||
let mut components = Path::new(name).components();
|
||||
match (components.next(), components.next()) {
|
||||
(Some(Component::Normal(_)), None) => Ok(name),
|
||||
_ => Err(invalid("upload name must be a safe basename")),
|
||||
}
|
||||
}
|
||||
|
||||
fn invalid(message: impl Into<String>) -> RpcError {
|
||||
RpcError {
|
||||
code: ErrorCode::InvalidParams,
|
||||
message: message.into(),
|
||||
data: None,
|
||||
}
|
||||
}
|
||||
fn not_found(message: impl Into<String>) -> RpcError {
|
||||
RpcError {
|
||||
code: ErrorCode::NotFound,
|
||||
message: message.into(),
|
||||
data: None,
|
||||
}
|
||||
}
|
||||
fn permission(message: impl Into<String>) -> RpcError {
|
||||
RpcError {
|
||||
code: ErrorCode::PermissionDenied,
|
||||
message: message.into(),
|
||||
data: None,
|
||||
}
|
||||
}
|
||||
fn io_error(err: std::io::Error) -> RpcError {
|
||||
RpcError {
|
||||
code: ErrorCode::ProtocolError,
|
||||
message: err.to_string(),
|
||||
data: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn protocol_error(message: impl Into<String>) -> RpcError {
|
||||
RpcError {
|
||||
code: ErrorCode::ProtocolError,
|
||||
message: message.into(),
|
||||
data: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn registry() -> (tempfile::TempDir, TransferRegistry) {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let registry = TransferRegistry::at_root(temp.path().join("transfers")).unwrap();
|
||||
(temp, registry)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn upload_is_not_resolvable_until_complete_or_from_another_session() {
|
||||
let (_temp, registry) = registry();
|
||||
let begin = registry
|
||||
.begin_upload(TransferBeginParams {
|
||||
session_id: "s1".into(),
|
||||
name: "image.png".into(),
|
||||
byte_size: 3,
|
||||
})
|
||||
.unwrap();
|
||||
assert!(
|
||||
registry
|
||||
.resolve_uploads("s1", std::slice::from_ref(&begin.transfer_id))
|
||||
.is_err()
|
||||
);
|
||||
registry
|
||||
.write_chunk(TransferChunkParams {
|
||||
transfer_id: begin.transfer_id.clone(),
|
||||
offset: 0,
|
||||
data_base64: base64::engine::general_purpose::STANDARD.encode(b"abc"),
|
||||
})
|
||||
.unwrap();
|
||||
registry
|
||||
.finish_upload(TransferIdParams {
|
||||
transfer_id: begin.transfer_id.clone(),
|
||||
})
|
||||
.unwrap();
|
||||
assert!(
|
||||
registry
|
||||
.resolve_uploads("s2", std::slice::from_ref(&begin.transfer_id))
|
||||
.is_err()
|
||||
);
|
||||
let [path] = registry
|
||||
.resolve_uploads("s1", &[begin.transfer_id])
|
||||
.unwrap()
|
||||
.try_into()
|
||||
.unwrap();
|
||||
assert_eq!(path.file_name().unwrap(), "image.png");
|
||||
assert_eq!(fs::read(path).unwrap(), b"abc");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn upload_name_must_be_one_safe_path_component() {
|
||||
let (_temp, registry) = registry();
|
||||
for name in ["", ".", "..", "../secret", "folder/file", "folder\\file"] {
|
||||
let result = registry.begin_upload(TransferBeginParams {
|
||||
session_id: "s1".into(),
|
||||
name: name.into(),
|
||||
byte_size: 0,
|
||||
});
|
||||
assert!(result.is_err(), "accepted unsafe upload name");
|
||||
}
|
||||
assert!(registry.entries.lock().unwrap().is_empty());
|
||||
assert!(fs::read_dir(®istry.root).unwrap().next().is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn same_upload_names_are_isolated_by_transfer_directory() {
|
||||
let (_temp, registry) = registry();
|
||||
let first = registry
|
||||
.begin_upload(TransferBeginParams {
|
||||
session_id: "s1".into(),
|
||||
name: "image.png".into(),
|
||||
byte_size: 0,
|
||||
})
|
||||
.unwrap();
|
||||
let second = registry
|
||||
.begin_upload(TransferBeginParams {
|
||||
session_id: "s1".into(),
|
||||
name: "image.png".into(),
|
||||
byte_size: 0,
|
||||
})
|
||||
.unwrap();
|
||||
let entries = registry.entries.lock().unwrap();
|
||||
let first_path = &entries[&first.transfer_id].path;
|
||||
let second_path = &entries[&second.transfer_id].path;
|
||||
assert_ne!(first_path.parent(), second_path.parent());
|
||||
assert_eq!(first_path.file_name().unwrap(), "image.png");
|
||||
assert_eq!(second_path.file_name().unwrap(), "image.png");
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn upload_staging_is_private() {
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
|
||||
let (_temp, registry) = registry();
|
||||
let begin = registry
|
||||
.begin_upload(TransferBeginParams {
|
||||
session_id: "s1".into(),
|
||||
name: "private.png".into(),
|
||||
byte_size: 0,
|
||||
})
|
||||
.unwrap();
|
||||
let entries = registry.entries.lock().unwrap();
|
||||
let path = &entries[&begin.transfer_id].path;
|
||||
assert_eq!(
|
||||
fs::metadata(path.parent().unwrap())
|
||||
.unwrap()
|
||||
.permissions()
|
||||
.mode()
|
||||
& 0o777,
|
||||
0o700
|
||||
);
|
||||
assert_eq!(
|
||||
fs::metadata(path).unwrap().permissions().mode() & 0o777,
|
||||
0o600
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn download_must_import_from_its_browser_capability_directory() {
|
||||
let (temp, registry) = registry();
|
||||
let staging = registry.begin_download("s1").unwrap();
|
||||
let outside = temp.path().join("outside.bin");
|
||||
fs::write(&outside, b"secret").unwrap();
|
||||
assert!(
|
||||
registry
|
||||
.import_download(&staging.transfer_id, &outside)
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(fs::read(&outside).unwrap(), b"secret");
|
||||
|
||||
let browser_dir = temp
|
||||
.path()
|
||||
.join("Downloads")
|
||||
.join("BrowserSkill")
|
||||
.join(&staging.transfer_id);
|
||||
fs::create_dir_all(&browser_dir).unwrap();
|
||||
let inside = browser_dir.join("result.bin");
|
||||
fs::write(&inside, b"result").unwrap();
|
||||
assert_eq!(
|
||||
registry
|
||||
.import_download(&staging.transfer_id, &inside)
|
||||
.unwrap(),
|
||||
6
|
||||
);
|
||||
assert!(!inside.exists());
|
||||
let chunk = registry
|
||||
.read_chunk(TransferChunkParams {
|
||||
transfer_id: staging.transfer_id,
|
||||
offset: 0,
|
||||
data_base64: String::new(),
|
||||
})
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
base64::engine::general_purpose::STANDARD
|
||||
.decode(chunk.data_base64.unwrap())
|
||||
.unwrap(),
|
||||
b"result"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validated_oversized_browser_download_is_removed_on_rejection() {
|
||||
let (temp, registry) = registry();
|
||||
let staging = registry.begin_download("s1").unwrap();
|
||||
let browser_dir = temp
|
||||
.path()
|
||||
.join("Downloads")
|
||||
.join("BrowserSkill")
|
||||
.join(&staging.transfer_id);
|
||||
fs::create_dir_all(&browser_dir).unwrap();
|
||||
let inside = browser_dir.join("oversized.bin");
|
||||
File::create(&inside)
|
||||
.unwrap()
|
||||
.set_len(MAX_TRANSFER_BYTES + 1)
|
||||
.unwrap();
|
||||
|
||||
assert!(
|
||||
registry
|
||||
.import_download(&staging.transfer_id, &inside)
|
||||
.is_err()
|
||||
);
|
||||
assert!(!inside.exists());
|
||||
assert!(!browser_dir.exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn releasing_a_session_removes_all_staging() {
|
||||
let (_temp, registry) = registry();
|
||||
let first = registry.begin_download("s1").unwrap();
|
||||
let second = registry.begin_download("s2").unwrap();
|
||||
let first_directory = registry.root.join(&first.transfer_id);
|
||||
let second_directory = registry.root.join(&second.transfer_id);
|
||||
registry.release_session("s1");
|
||||
assert!(!first_directory.exists());
|
||||
assert!(second_directory.exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn upload_staging_is_bounded_across_a_session() {
|
||||
let (_temp, registry) = registry();
|
||||
registry
|
||||
.begin_upload(TransferBeginParams {
|
||||
session_id: "s1".into(),
|
||||
name: "large.bin".into(),
|
||||
byte_size: MAX_TRANSFER_BYTES,
|
||||
})
|
||||
.unwrap();
|
||||
assert!(
|
||||
registry
|
||||
.begin_upload(TransferBeginParams {
|
||||
session_id: "s1".into(),
|
||||
name: "one-more-byte.bin".into(),
|
||||
byte_size: 1,
|
||||
})
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
registry
|
||||
.begin_upload(TransferBeginParams {
|
||||
session_id: "s2".into(),
|
||||
name: "other-session.bin".into(),
|
||||
byte_size: 1,
|
||||
})
|
||||
.is_ok()
|
||||
);
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user