mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
Compare commits
4
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 polardayandCopilot Autofix powered by AI
|
9a5d45d156 | ||
|
polardayandCopilot Autofix powered by AI
|
11cd12c9cd | ||
|
“haonan”andCursor
|
873971f4cb | ||
|
“haonan”andCursor
|
a9c7444139 |
@@ -37,35 +37,6 @@ jobs:
|
||||
- name: Run Rust tests
|
||||
run: cargo test --workspace --locked
|
||||
|
||||
windows-daemon:
|
||||
name: Windows daemon, installer and cancellation regression tests
|
||||
runs-on: windows-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Run Windows named-pipe tests
|
||||
run: cargo test -p bsk --lib --locked daemon::ipc::windows::tests
|
||||
|
||||
- name: Run Windows process liveness tests
|
||||
run: cargo test -p bsk --lib --locked daemon::lockfile::tests
|
||||
|
||||
- name: Run Windows update tests
|
||||
run: cargo test -p bsk --lib --locked cli::update::tests
|
||||
|
||||
- name: Run Windows update and cancellation process tests
|
||||
run: cargo test -p bsk --test windows_update --test windows_parent_cancel --locked
|
||||
|
||||
- name: Run installer tests (Windows PowerShell 5.1)
|
||||
shell: powershell
|
||||
run: ./scripts/install-windows.test.ps1
|
||||
|
||||
- name: Run installer tests (PowerShell 7)
|
||||
shell: pwsh
|
||||
run: ./scripts/install-windows.test.ps1
|
||||
|
||||
frontend:
|
||||
name: Frontend lint, typecheck, tests, build
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
-109
@@ -1,109 +0,0 @@
|
||||
# Changelog
|
||||
|
||||
All notable changes to BrowserSkill will be documented in this file.
|
||||
The format is based on [Keep a Changelog](https://keepachangelog.com/).
|
||||
|
||||
Starting from 0.2.0, CLI / Extension / DSH Plugin share the same version number.
|
||||
|
||||
## [0.2.1] - 2026-09-09
|
||||
|
||||
### Changed
|
||||
- DSH Plugin: clarified installation, configuration, profile restart, and upgrade instructions
|
||||
|
||||
### Fixed
|
||||
- CLI: reliable Windows self-updates, including executable replacement and daemon restart
|
||||
- CLI: Windows named-pipe connection continuity and stale daemon process detection
|
||||
- Windows installer: PowerShell 5.1 compatibility, path quoting, and staged executable replacement
|
||||
- CLI / DSH Plugin: cancellation propagation and upload/download cleanup when the parent process exits
|
||||
- Extension: tab ownership and session cleanup when agent tabs fail to open or borrowed tabs are returned or closed
|
||||
- Extension: navigation recovery when other extensions restrict Chrome DevTools Protocol access
|
||||
- Extension: background input filling, automatic append behavior, and verification of field values before reporting success
|
||||
- Extension: match observed form state to the correct element after page changes
|
||||
- Extension: UUID generation fallback for recording and overlays on insecure origins
|
||||
- Browser locale matching for regional language variants
|
||||
- DSH Plugin: observation state synchronization, thumbnail capture lifecycle, and stale preview cleanup
|
||||
- DSH Plugin: removed the obsolete client runtime host dependency
|
||||
|
||||
## [0.2.0] - 2026-09-02
|
||||
|
||||
### Added
|
||||
- File transfer: upload and download support across CLI, Extension, and DSH Plugin
|
||||
- File transfer: drag-and-drop upload (`drop-to-upload`)
|
||||
- VOM semantic graph, name enrichment, and hover perception modules
|
||||
- VOM hover probing (opt-in via `observe` parameter)
|
||||
- DSH Plugin: browser tool parity with CLI commands
|
||||
- Edge Add-ons automated publishing in CI
|
||||
- Protocol upgrade reminder when CLI / Extension protocol versions differ
|
||||
- Browser evaluation harness (`evals/browser/`)
|
||||
- Unified release script (`scripts/release.mjs`)
|
||||
|
||||
### Changed
|
||||
- **Version scheme**: all three components now share the same semver
|
||||
- VOM rendering algorithm optimizations
|
||||
- Leaner SKILL.md agent instructions
|
||||
- DSH Plugin: simplified browser commands
|
||||
- Borrow confirmation UX — proactive focus and longer timeout
|
||||
- PiP window now has a close button
|
||||
|
||||
### Fixed
|
||||
- Screenshot media type detection (was hard-coded to `image/png`)
|
||||
- DSH Plugin session lifecycle stability
|
||||
- DSH Plugin Cordis package ID mismatch
|
||||
- Observation thumbnail media type sniffing
|
||||
- Upload/download race conditions and layout bypass issues
|
||||
- VOM repeated name and safety policy issues
|
||||
|
||||
---
|
||||
|
||||
*Previous releases used independent version numbers per component.*
|
||||
|
||||
## CLI 0.1.11 / Extension 0.1.7 / DSH Plugin 0.1.2 — 2026-08-26 ~ 2026-08-29
|
||||
|
||||
### Added
|
||||
- DSH Plugin sidebar integration, session lifecycle, and archive cleanup
|
||||
- Recorder iframe and OOPIF support
|
||||
|
||||
### Changed
|
||||
- VOM functional refactor
|
||||
|
||||
### Fixed
|
||||
- Recorder safety policy and bug fixes
|
||||
|
||||
## CLI 0.1.10 / Extension 0.1.6 — 2026-08-08
|
||||
|
||||
### Added
|
||||
- VOM observation recording and settled-state detection
|
||||
- Record overlay timer
|
||||
|
||||
### Fixed
|
||||
- Browser keepalive disconnect handling
|
||||
|
||||
## CLI 0.1.9 / Extension 0.1.5 — 2026-07-29
|
||||
|
||||
### Added
|
||||
- CLI auto-update mechanism
|
||||
- Trace v3 protocol and recorder
|
||||
|
||||
### Fixed
|
||||
- MV3 keepalive disconnect
|
||||
|
||||
## CLI 0.1.8 / Extension 0.1.4 — 2026-07-22
|
||||
|
||||
### Added
|
||||
- More browser interaction actions
|
||||
|
||||
### Fixed
|
||||
- Windows named-pipe hash-only path issue
|
||||
|
||||
## CLI 0.1.7 / Extension 0.1.3 — 2026-07-07
|
||||
|
||||
Initial public release pair.
|
||||
|
||||
## CLI 0.1.6 — 2026-06-30
|
||||
|
||||
### Fixed
|
||||
- Minor CLI fixes
|
||||
|
||||
## CLI 0.1.5 / Extension 0.1.2 — 2026-06-22
|
||||
|
||||
First tagged releases.
|
||||
Generated
+2
-2
@@ -134,7 +134,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "bsk"
|
||||
version = "0.2.1"
|
||||
version = "0.1.11"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"base64",
|
||||
@@ -169,7 +169,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "bsk-protocol"
|
||||
version = "0.2.1"
|
||||
version = "0.1.11"
|
||||
dependencies = [
|
||||
"schemars",
|
||||
"semver",
|
||||
|
||||
+2
-2
@@ -7,7 +7,7 @@ members = [
|
||||
default-members = ["crates/bsk-cli"]
|
||||
|
||||
[workspace.package]
|
||||
version = "0.2.1"
|
||||
version = "0.1.11"
|
||||
edition = "2024"
|
||||
rust-version = "1.85"
|
||||
license = "MIT"
|
||||
@@ -43,7 +43,7 @@ rand = "0.8"
|
||||
uuid = { version = "1", features = ["v4", "serde"] }
|
||||
|
||||
# Workspace crates (version required for `cargo publish` of dependents)
|
||||
bsk-protocol = { version = "0.2.1", path = "crates/bsk-protocol" }
|
||||
bsk-protocol = { version = "0.1.9", path = "crates/bsk-protocol" }
|
||||
|
||||
[profile.release]
|
||||
strip = "symbols"
|
||||
|
||||
@@ -147,26 +147,20 @@ Start a new Agent session and write a prompt that needs the browser, for example
|
||||
Using [DeepSeek Harness](https://github.com/deepseek-ai/deepseek-harness) (`dsh`)?
|
||||
BrowserSkill ships a first-class dsh plugin on npm as
|
||||
[`@wxg-prc-cpg/browser-skill-dsh-plugin`](https://www.npmjs.com/package/@wxg-prc-cpg/browser-skill-dsh-plugin).
|
||||
It gives the agent native `browser_*` tools and a live view of its browser sessions
|
||||
in the Web UI. The plugin runs `bsk` on the agent's behalf.
|
||||
It injects native `browser_*` tools (no shelling out to `bsk`) and a live Web UI
|
||||
overlay of each Agent Window.
|
||||
|
||||
Install the `bsk` CLI and connect the browser extension first. Then add the plugin
|
||||
to a dsh profile and start it (replace `web` with your profile name):
|
||||
Add it to a dsh profile, then start that profile:
|
||||
|
||||
```sh
|
||||
dsh plugin --profile web add @wxg-prc-cpg/browser-skill-dsh-plugin
|
||||
dsh --profile web
|
||||
```
|
||||
|
||||
The plugin includes the `browser-skill` skill, so `bsk install-skill` is not needed
|
||||
for dsh. Installed plugins do not update automatically. To upgrade this plugin:
|
||||
|
||||
```sh
|
||||
dsh plugin --profile web update @wxg-prc-cpg/browser-skill-dsh-plugin --latest
|
||||
```
|
||||
|
||||
Restart the profile after upgrading. See the
|
||||
[plugin README](packages/dsh-plugin-browserskill/README.md) for usage and configuration.
|
||||
The plugin carries its own copy of the skill, so `bsk install-skill` is not needed
|
||||
for dsh — but the `bsk` CLI and the browser extension are still prerequisites. See
|
||||
the [plugin README](packages/dsh-plugin-browserskill/README.md) for the tool list,
|
||||
configuration, and the observation overlay.
|
||||
|
||||
## How It Works
|
||||
|
||||
|
||||
+3
-9
@@ -125,22 +125,16 @@ bsk install-skill
|
||||
|
||||
## DeepSeek Harness 插件
|
||||
|
||||
在用 [DeepSeek Harness](https://github.com/deepseek-ai/deepseek-harness)（`dsh`）？BrowserSkill 提供了官方 dsh 插件，已发布到 npm：[`@wxg-prc-cpg/browser-skill-dsh-plugin`](https://www.npmjs.com/package/@wxg-prc-cpg/browser-skill-dsh-plugin)。它为 Agent 提供原生 `browser_*` 工具，由插件代为调用 `bsk`，并在 Web UI 中实时展示浏览器会话。
|
||||
在用 [DeepSeek Harness](https://github.com/deepseek-ai/deepseek-harness)（`dsh`）？BrowserSkill 提供了官方 dsh 插件，已发布到 npm：[`@wxg-prc-cpg/browser-skill-dsh-plugin`](https://www.npmjs.com/package/@wxg-prc-cpg/browser-skill-dsh-plugin)。它会注入原生 `browser_*` 工具（无需再通过 Shell 调用 `bsk`），并在 Web UI 中实时观察每个 Agent Window。
|
||||
|
||||
先安装 `bsk` CLI 并连接浏览器扩展，再将插件装进 dsh profile 并启动（将 `web` 替换为你的 profile 名称）：
|
||||
把它装进某个 dsh profile，然后启动该 profile：
|
||||
|
||||
```sh
|
||||
dsh plugin --profile web add @wxg-prc-cpg/browser-skill-dsh-plugin
|
||||
dsh --profile web
|
||||
```
|
||||
|
||||
插件自带 `browser-skill` skill，所以在 dsh 下无需执行 `bsk install-skill`。已安装的插件不会自动更新；升级此插件请运行：
|
||||
|
||||
```sh
|
||||
dsh plugin --profile web update @wxg-prc-cpg/browser-skill-dsh-plugin --latest
|
||||
```
|
||||
|
||||
升级后重启该 profile。用法与配置见[插件 README](packages/dsh-plugin-browserskill/README.md)。
|
||||
插件自带 skill，所以在 dsh 下无需执行 `bsk install-skill`；但 `bsk` CLI 和浏览器扩展仍是前置条件。工具清单、配置项与观察浮层见[插件 README](packages/dsh-plugin-browserskill/README.md)。
|
||||
|
||||
## 工作原理
|
||||
|
||||
|
||||
@@ -57,7 +57,7 @@ The Extension requests the following Chrome permissions. Each is used solely for
|
||||
|
||||
## 6. Where Data Goes
|
||||
|
||||
All Extension activity stays on the user's local device. The only network traffic the Extension generates is a WebSocket connection to `ws://127.0.0.1:52800` (loopback only). What the AI agent connected to that local daemon does with the data afterwards (for example, sending a screenshot to an LLM provider) is governed by the privacy policy of that agent or LLM provider, **not** by this policy. BrowserSkill is not a party to those communications.
|
||||
All Extension activity stays on the user's local device. The only network traffic the Extension generates is a WebSocket connection to the local bsk daemon on `127.0.0.1` (loopback only; default port **52800**, configurable in the extension popup). What the AI agent connected to that local daemon does with the data afterwards (for example, sending a screenshot to an LLM provider) is governed by the privacy policy of that agent or LLM provider, **not** by this policy. BrowserSkill is not a party to those communications.
|
||||
|
||||
## 7. Data Retention
|
||||
|
||||
@@ -80,7 +80,7 @@ The Extension is a developer tool and is not directed at children under 13. It d
|
||||
|
||||
## 10. Security
|
||||
|
||||
Because the Extension communicates only with `127.0.0.1`, no data is exposed to the network. Users should still avoid running BrowserSkill in untrusted environments, since any local process able to bind to `127.0.0.1:52800` could send commands to the Extension. Run BrowserSkill only on machines you control.
|
||||
Because the Extension communicates only with `127.0.0.1`, no data is exposed to the network. Users should still avoid running BrowserSkill in untrusted environments, since any local process able to bind to the configured loopback port could send commands to the Extension. Run BrowserSkill only on machines you control.
|
||||
|
||||
## 11. Open Source and Auditability
|
||||
|
||||
|
||||
@@ -54,7 +54,7 @@ BrowserSkill **不会**：
|
||||
|
||||
## 6. 数据流向
|
||||
|
||||
本扩展的所有活动都停留在用户本地设备上。本扩展产生的唯一网络流量是与 `ws://127.0.0.1:52800`（仅回环地址）的 WebSocket 连接。连接到该本地守护进程的 AI 助手在拿到数据之后如何处理（例如将截图发送给某个 LLM 服务），由该助手或 LLM 提供商自身的隐私政策约束，**不在本政策范围内**。BrowserSkill 不参与那些通信。
|
||||
本扩展的所有活动都停留在用户本地设备上。本扩展产生的唯一网络流量是与本地 bsk 守护进程在 `127.0.0.1`（仅回环地址；默认端口 **52800**，可在扩展弹窗中配置）上的 WebSocket 连接。连接到该本地守护进程的 AI 助手在拿到数据之后如何处理（例如将截图发送给某个 LLM 服务），由该助手或 LLM 提供商自身的隐私政策约束，**不在本政策范围内**。BrowserSkill 不参与那些通信。
|
||||
|
||||
## 7. 数据保留
|
||||
|
||||
@@ -76,7 +76,7 @@ BrowserSkill **不会**：
|
||||
|
||||
## 10. 安全性
|
||||
|
||||
由于本扩展仅与 `127.0.0.1` 通信，因此不会向网络暴露任何数据。但用户仍应避免在不可信的环境中运行 BrowserSkill —— 任何能够绑定到 `127.0.0.1:52800` 的本地进程理论上都可以向本扩展发送指令。请仅在您本人控制的机器上运行 BrowserSkill。
|
||||
由于本扩展仅与 `127.0.0.1` 通信，因此不会向网络暴露任何数据。但用户仍应避免在不可信的环境中运行 BrowserSkill —— 任何能够绑定到所配置回环端口的本地进程理论上都可以向本扩展发送指令。请仅在您本人控制的机器上运行 BrowserSkill。
|
||||
|
||||
## 11. 开源与可审计性
|
||||
|
||||
|
||||
@@ -1,81 +0,0 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<title>BrowserSkill file-transfer fixture</title>
|
||||
<style>
|
||||
body { font: 16px system-ui; max-width: 760px; margin: 48px auto; }
|
||||
.actions { display: flex; gap: 12px; align-items: center; margin: 24px 0; }
|
||||
button, a { padding: 10px 16px; border: 1px solid #777; border-radius: 8px; }
|
||||
a[aria-disabled="true"] { opacity: .45; pointer-events: none; }
|
||||
#drop { padding: 36px; border: 3px dashed #777; border-radius: 12px; text-align: center; }
|
||||
#drop.dragging { border-color: #2563eb; background: #eff6ff; }
|
||||
img { display: block; max-width: 480px; max-height: 360px; margin-top: 20px; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Upload then download the same image</h1>
|
||||
<p id="status">No image selected</p>
|
||||
<div class="actions">
|
||||
<button id="upload" type="button">Upload through file input</button>
|
||||
<input id="file" type="file" accept="image/png" hidden />
|
||||
<a id="download" aria-disabled="true">Download uploaded image</a>
|
||||
</div>
|
||||
<div id="drop" role="button" tabindex="0">Drop an image here</div>
|
||||
<img id="preview" alt="Uploaded image preview" hidden />
|
||||
<script>
|
||||
const input = document.querySelector('#file');
|
||||
const upload = document.querySelector('#upload');
|
||||
const drop = document.querySelector('#drop');
|
||||
const download = document.querySelector('#download');
|
||||
const preview = document.querySelector('#preview');
|
||||
const status = document.querySelector('#status');
|
||||
let objectUrl;
|
||||
|
||||
window.__bskTransferFixture = { events: [], result: null };
|
||||
|
||||
function accept(file, mechanism) {
|
||||
if (!file) return;
|
||||
if (objectUrl) URL.revokeObjectURL(objectUrl);
|
||||
objectUrl = URL.createObjectURL(file);
|
||||
preview.src = objectUrl;
|
||||
preview.hidden = false;
|
||||
download.href = objectUrl;
|
||||
download.download = file.name;
|
||||
download.removeAttribute('aria-disabled');
|
||||
status.textContent = `Ready via ${mechanism}: ${file.name} (${file.size} bytes)`;
|
||||
window.__bskTransferFixture.result = {
|
||||
mechanism,
|
||||
name: file.name,
|
||||
size: file.size,
|
||||
type: file.type,
|
||||
hasPathProperty: 'path' in file,
|
||||
};
|
||||
}
|
||||
|
||||
upload.addEventListener('click', () => input.click());
|
||||
input.addEventListener('change', () => accept(input.files[0], 'input'));
|
||||
|
||||
for (const type of ['dragenter', 'dragover']) {
|
||||
drop.addEventListener(type, event => {
|
||||
const types = [...event.dataTransfer.types];
|
||||
window.__bskTransferFixture.events.push({ type, types });
|
||||
if (!types.includes('Files')) return;
|
||||
event.preventDefault();
|
||||
drop.classList.add('dragging');
|
||||
});
|
||||
}
|
||||
drop.addEventListener('dragleave', () => drop.classList.remove('dragging'));
|
||||
drop.addEventListener('drop', event => {
|
||||
window.__bskTransferFixture.events.push({
|
||||
type: 'drop',
|
||||
types: [...event.dataTransfer.types],
|
||||
fileCount: event.dataTransfer.files.length,
|
||||
});
|
||||
event.preventDefault();
|
||||
drop.classList.remove('dragging');
|
||||
accept(event.dataTransfer.files[0], 'drop');
|
||||
});
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@browser-skill/extension",
|
||||
"version": "0.2.1",
|
||||
"version": "0.1.7",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
|
||||
@@ -29,24 +29,6 @@ function fakeApi() {
|
||||
}
|
||||
|
||||
describe("ChromiumCdp", () => {
|
||||
it("changes attachment identity only across real debugger attachments", async () => {
|
||||
const { api, onEvent, onDetach } = fakeApi();
|
||||
const cdp = new ChromiumCdp(api);
|
||||
expect(cdp.getAttachmentId(4)).toBeUndefined();
|
||||
await cdp.ensureAttached(4);
|
||||
const first = cdp.getAttachmentId(4);
|
||||
expect(first).toBeTruthy();
|
||||
await cdp.ensureAttached(4);
|
||||
onEvent.fire({ tabId: 4 }, "Target.attachedToTarget", { sessionId: "child" });
|
||||
expect(cdp.getAttachmentId(4)).toBe(first);
|
||||
onDetach.fire({ tabId: 4 }, "target_closed");
|
||||
expect(cdp.getAttachmentId(4)).toBeUndefined();
|
||||
await cdp.ensureAttached(4);
|
||||
expect(cdp.getAttachmentId(4)).not.toBe(first);
|
||||
await cdp.detach(4);
|
||||
expect(cdp.getAttachmentId(4)).toBeUndefined();
|
||||
});
|
||||
|
||||
it("discovers multiple iframe targets and recursively routes nested OOPIF commands", async () => {
|
||||
const { api, onEvent } = fakeApi();
|
||||
(api.sendCommand as ReturnType<typeof vi.fn>).mockImplementation(
|
||||
@@ -272,69 +254,6 @@ describe("ChromiumCdp", () => {
|
||||
expect(cdp.isAttached(7)).toBe(false);
|
||||
});
|
||||
|
||||
it.each([
|
||||
false,
|
||||
true,
|
||||
])("waits for detach before reconnecting a returned tab (detach rejects: %s)", async (rejectDetach) => {
|
||||
const { api } = fakeApi();
|
||||
const attached = new Set<number>();
|
||||
let finishDetach!: () => void;
|
||||
vi.mocked(api.attach).mockImplementation(async ({ tabId }) => {
|
||||
if (attached.has(tabId!)) throw new Error("Another debugger is already attached");
|
||||
attached.add(tabId!);
|
||||
});
|
||||
vi.mocked(api.detach).mockImplementation(async ({ tabId }) => {
|
||||
attached.delete(tabId!);
|
||||
});
|
||||
vi.mocked(api.detach).mockImplementationOnce(
|
||||
({ tabId }) =>
|
||||
new Promise<void>((resolve, reject) => {
|
||||
finishDetach = () => {
|
||||
attached.delete(tabId!);
|
||||
if (rejectDetach) reject(new Error("tab already detached"));
|
||||
else resolve();
|
||||
};
|
||||
}),
|
||||
);
|
||||
const cdp = new ChromiumCdp(api);
|
||||
cdp.trackSessionTab("aa11", 7);
|
||||
await cdp.ensureAttached(7);
|
||||
const returning = cdp.releaseSessionTab("aa11", 7);
|
||||
await vi.waitFor(() => expect(api.detach).toHaveBeenCalledOnce());
|
||||
let duplicateDetachFinished = false;
|
||||
const duplicateDetach = cdp.detach(7).then(() => {
|
||||
duplicateDetachFinished = true;
|
||||
});
|
||||
|
||||
cdp.trackSessionTab("bb22", 7);
|
||||
// Register rejection handlers immediately; the pre-fix driver rejects
|
||||
// both commands while the browser still has the previous attachment.
|
||||
const pending = Promise.allSettled([
|
||||
cdp.send(7, "Runtime.evaluate", { expression: "document.title" }),
|
||||
cdp.send(7, "DOM.getDocument"),
|
||||
]);
|
||||
await cdp.send(8, "DOM.getDocument");
|
||||
const finishedEarly = duplicateDetachFinished;
|
||||
finishDetach();
|
||||
await Promise.all([returning, duplicateDetach]);
|
||||
|
||||
expect(await pending).toEqual([
|
||||
{ status: "fulfilled", value: { ok: true } },
|
||||
{ status: "fulfilled", value: { ok: true } },
|
||||
]);
|
||||
expect(finishedEarly).toBe(false);
|
||||
expect(vi.mocked(api.attach).mock.calls.filter(([target]) => target.tabId === 7)).toHaveLength(
|
||||
2,
|
||||
);
|
||||
expect(cdp.isAttached(7)).toBe(true);
|
||||
expect(cdp.isAttached(8)).toBe(true);
|
||||
await cdp.detachSession("bb22");
|
||||
expect(api.detach).toHaveBeenCalledTimes(2);
|
||||
expect(cdp.isAttached(7)).toBe(false);
|
||||
expect(attached.has(7)).toBe(false);
|
||||
expect(cdp.isAttached(8)).toBe(true);
|
||||
});
|
||||
|
||||
it("detachAll() iterates every cached tab", async () => {
|
||||
const { api } = fakeApi();
|
||||
const cdp = new ChromiumCdp(api);
|
||||
@@ -732,155 +651,6 @@ describe("ChromiumCdp", () => {
|
||||
expect(cdp.networkEntriesSince(33, 0, 50, 1000).entries[0].url).toBeUndefined();
|
||||
});
|
||||
|
||||
it("releases only the returning session's claim on a tab", async () => {
|
||||
const { api } = fakeApi();
|
||||
const cdp = new ChromiumCdp(api);
|
||||
await cdp.ensureAttached(1);
|
||||
await cdp.ensureAttached(2);
|
||||
cdp.trackSessionTab("aa11", 1);
|
||||
cdp.trackSessionTab("bb22", 1);
|
||||
cdp.trackSessionTab("aa11", 2);
|
||||
|
||||
await cdp.releaseSessionTab("unknown", 1);
|
||||
await cdp.releaseSessionTab("aa11", 1);
|
||||
expect(api.detach).not.toHaveBeenCalled();
|
||||
|
||||
await cdp.releaseSessionTab("bb22", 1);
|
||||
await cdp.releaseSessionTab("bb22", 1);
|
||||
expect(api.detach).toHaveBeenCalledExactlyOnceWith({ tabId: 1 });
|
||||
expect(cdp.isAttached(2)).toBe(true);
|
||||
|
||||
await cdp.detachSession("aa11");
|
||||
expect(api.detach).toHaveBeenCalledTimes(2);
|
||||
expect(api.detach).toHaveBeenLastCalledWith({ tabId: 2 });
|
||||
});
|
||||
|
||||
it.each([
|
||||
false,
|
||||
true,
|
||||
])("releases a tab with an attachment in flight (new owner: %s)", async (newOwner) => {
|
||||
const { api } = fakeApi();
|
||||
let finishAttach!: () => void;
|
||||
vi.mocked(api.attach).mockImplementationOnce(
|
||||
() =>
|
||||
new Promise<void>((resolve) => {
|
||||
finishAttach = resolve;
|
||||
}),
|
||||
);
|
||||
const cdp = new ChromiumCdp(api);
|
||||
cdp.trackSessionTab("aa11", 1);
|
||||
const attaching = cdp.ensureAttached(1);
|
||||
const releasing = cdp.releaseSessionTab("aa11", 1);
|
||||
if (newOwner) cdp.trackSessionTab("bb22", 1);
|
||||
finishAttach();
|
||||
await Promise.all([attaching, releasing]);
|
||||
|
||||
expect(cdp.isAttached(1)).toBe(newOwner);
|
||||
expect(api.detach).toHaveBeenCalledTimes(newOwner ? 0 : 1);
|
||||
});
|
||||
|
||||
it("stops accepting dialogs on a returned tab even when another session keeps CDP attached", async () => {
|
||||
const { api, onEvent } = fakeApi();
|
||||
let controlled = true;
|
||||
const cdp = new ChromiumCdp(api, { shouldAutoAcceptDialog: () => controlled });
|
||||
cdp.trackSessionTab("aa11", 1);
|
||||
cdp.trackSessionTab("bb22", 1);
|
||||
await cdp.ensureAttached(1);
|
||||
onEvent.fire({ tabId: 1 }, "Page.javascriptDialogOpening", {
|
||||
type: "confirm",
|
||||
message: "before return",
|
||||
});
|
||||
await vi.waitFor(() => expect(cdp.dialogsSince(1, 0)).toHaveLength(1));
|
||||
vi.mocked(api.sendCommand).mockClear();
|
||||
|
||||
controlled = false;
|
||||
await cdp.releaseSessionTab("aa11", 1);
|
||||
expect(cdp.isAttached(1)).toBe(true);
|
||||
onEvent.fire({ tabId: 1 }, "Page.javascriptDialogOpening", {
|
||||
type: "confirm",
|
||||
message: "after return",
|
||||
});
|
||||
await Promise.resolve();
|
||||
expect(api.sendCommand).not.toHaveBeenCalled();
|
||||
expect(cdp.dialogsSince(1, 0)).toHaveLength(1);
|
||||
|
||||
controlled = true;
|
||||
onEvent.fire({ tabId: 1 }, "Page.javascriptDialogOpening", {
|
||||
type: "alert",
|
||||
message: "borrowed again",
|
||||
});
|
||||
await vi.waitFor(() => expect(cdp.dialogsSince(1, 0)).toHaveLength(2));
|
||||
expect(api.sendCommand).toHaveBeenCalledExactlyOnceWith(
|
||||
{ tabId: 1 },
|
||||
"Page.handleJavaScriptDialog",
|
||||
{ accept: true },
|
||||
);
|
||||
});
|
||||
|
||||
it("ignores dialog events after detach, including a pending eligibility check", async () => {
|
||||
const { api, onEvent } = fakeApi();
|
||||
let resolveEligibility!: (value: boolean) => void;
|
||||
const shouldAutoAcceptDialog = vi.fn(
|
||||
() =>
|
||||
new Promise<boolean>((resolve) => {
|
||||
resolveEligibility = resolve;
|
||||
}),
|
||||
);
|
||||
const cdp = new ChromiumCdp(api, { shouldAutoAcceptDialog });
|
||||
await cdp.ensureAttached(1);
|
||||
onEvent.fire({ tabId: 1 }, "Page.javascriptDialogOpening", { type: "alert" });
|
||||
await cdp.detach(1);
|
||||
resolveEligibility(true);
|
||||
await Promise.resolve();
|
||||
onEvent.fire({ tabId: 1 }, "Page.javascriptDialogOpening", { type: "alert" });
|
||||
|
||||
expect(shouldAutoAcceptDialog).toHaveBeenCalledOnce();
|
||||
expect(api.sendCommand).not.toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
"Page.handleJavaScriptDialog",
|
||||
expect.anything(),
|
||||
);
|
||||
expect(cdp.dialogsSince(1, 0)).toEqual([]);
|
||||
});
|
||||
|
||||
it("does not accept dialogs when the tab's current scope cannot be determined", async () => {
|
||||
const { api, onEvent } = fakeApi();
|
||||
const shouldAutoAcceptDialog = vi.fn(async () => {
|
||||
throw new Error("tab is gone");
|
||||
});
|
||||
const cdp = new ChromiumCdp(api, { shouldAutoAcceptDialog });
|
||||
await cdp.ensureAttached(1);
|
||||
vi.mocked(api.sendCommand).mockClear();
|
||||
|
||||
onEvent.fire({ tabId: 1 }, "Page.javascriptDialogOpening", { type: "confirm" });
|
||||
await Promise.resolve();
|
||||
|
||||
expect(shouldAutoAcceptDialog).toHaveBeenCalledWith(1);
|
||||
expect(api.sendCommand).not.toHaveBeenCalled();
|
||||
expect(cdp.dialogsSince(1, 0)).toEqual([]);
|
||||
});
|
||||
|
||||
it("does not restore dialog state when an acceptance completes after detach", async () => {
|
||||
const { api, onEvent } = fakeApi();
|
||||
const cdp = new ChromiumCdp(api);
|
||||
await cdp.ensureAttached(1);
|
||||
let finishAccept!: () => void;
|
||||
vi.mocked(api.sendCommand).mockImplementationOnce(
|
||||
() =>
|
||||
new Promise<void>((resolve) => {
|
||||
finishAccept = resolve;
|
||||
}),
|
||||
);
|
||||
|
||||
onEvent.fire({ tabId: 1 }, "Page.javascriptDialogOpening", { type: "alert" });
|
||||
await cdp.detach(1);
|
||||
finishAccept();
|
||||
await Promise.resolve();
|
||||
|
||||
expect(cdp.dialogsSince(1, 0)).toEqual([]);
|
||||
expect(cdp.dialogCursor(1)).toBe(0);
|
||||
});
|
||||
|
||||
it("detachSession only detaches tabs no other session owns", async () => {
|
||||
const { api } = fakeApi();
|
||||
const cdp = new ChromiumCdp(api);
|
||||
|
||||
@@ -164,9 +164,7 @@ async function settleBeforeDeadline(promises: Promise<void>[], deadline: number)
|
||||
export class ChromiumCdp {
|
||||
private readonly api: CdpDebuggerApi;
|
||||
private readonly attachedTabs = new Set<number>();
|
||||
private readonly attachmentIds = new Map<number, string>();
|
||||
private readonly attachInFlight = new Map<number, Promise<void>>();
|
||||
private readonly detachInFlight = new Map<number, Promise<void>>();
|
||||
private readonly tabOwners = new Map<number, Set<string>>();
|
||||
private readonly dialogBuffers = new Map<number, JavaScriptDialogInfo[]>();
|
||||
private readonly dialogSequences = new Map<number, number>();
|
||||
@@ -184,13 +182,7 @@ export class ChromiumCdp {
|
||||
private networkSubscription: { dispose(): void } | null = null;
|
||||
private frameTargetSubscription: { dispose(): void } | null = null;
|
||||
|
||||
constructor(
|
||||
api: CdpDebuggerApi = chromeDebuggerApi,
|
||||
private readonly options: {
|
||||
/** CDP observation alone does not authorize dismissing user dialogs. */
|
||||
shouldAutoAcceptDialog?: (tabId: number) => boolean | Promise<boolean>;
|
||||
} = {},
|
||||
) {
|
||||
constructor(api: CdpDebuggerApi = chromeDebuggerApi) {
|
||||
this.api = api;
|
||||
this.bindAutoDetach();
|
||||
this.bindDialogHandler();
|
||||
@@ -199,17 +191,8 @@ export class ChromiumCdp {
|
||||
this.bindFrameTargetHandler();
|
||||
}
|
||||
|
||||
/** Identity of this actual debugger attachment, independent of frame topology. */
|
||||
getAttachmentId(tabId: number): string | undefined {
|
||||
return this.attachmentIds.get(tabId);
|
||||
}
|
||||
|
||||
/** Attach to `tabId` if we haven't already in this driver. */
|
||||
async ensureAttached(tabId: number): Promise<void> {
|
||||
// Returning a tab clears the cache before Chrome finishes detaching.
|
||||
// New observers must wait before opening the next connection to that tab.
|
||||
const detaching = this.detachInFlight.get(tabId);
|
||||
if (detaching) await detaching;
|
||||
if (this.attachedTabs.has(tabId)) return;
|
||||
const existing = this.attachInFlight.get(tabId);
|
||||
if (existing) {
|
||||
@@ -223,7 +206,6 @@ export class ChromiumCdp {
|
||||
await this.enableConsoleDomains(tabId);
|
||||
await this.enableNetworkDomainBestEffort(tabId);
|
||||
this.attachedTabs.add(tabId);
|
||||
this.attachmentIds.set(tabId, crypto.randomUUID());
|
||||
await this.enableFrameDiscovery({ tabId }).catch((err) => {
|
||||
console.debug("[bsk cdp] frame discovery unavailable", { tabId, err });
|
||||
});
|
||||
@@ -443,32 +425,20 @@ export class ChromiumCdp {
|
||||
|
||||
/** Detach if attached; never throws. */
|
||||
async detach(tabId: number): Promise<void> {
|
||||
const existing = this.detachInFlight.get(tabId);
|
||||
if (existing) {
|
||||
await existing;
|
||||
return;
|
||||
}
|
||||
this.attachInFlight.delete(tabId);
|
||||
if (!this.attachedTabs.has(tabId)) return;
|
||||
this.attachedTabs.delete(tabId);
|
||||
this.attachmentIds.delete(tabId);
|
||||
this.clearDialogState(tabId);
|
||||
this.clearConsoleState(tabId);
|
||||
this.clearNetworkState(tabId);
|
||||
this.clearFrameState(tabId);
|
||||
const detach = (async () => {
|
||||
try {
|
||||
await this.api.detach({ tabId });
|
||||
} catch (err) {
|
||||
// Tab may already be gone — Chrome auto-detaches on close. Log
|
||||
// at debug so production builds aren't noisy.
|
||||
console.debug("[bsk cdp] detach failed (likely tab already closed)", err);
|
||||
}
|
||||
})().finally(() => {
|
||||
this.detachInFlight.delete(tabId);
|
||||
});
|
||||
this.detachInFlight.set(tabId, detach);
|
||||
await detach;
|
||||
try {
|
||||
await this.api.detach({ tabId });
|
||||
} catch (err) {
|
||||
// Tab may already be gone — Chrome auto-detaches on close. Log
|
||||
// at debug so production builds aren't noisy.
|
||||
console.debug("[bsk cdp] detach failed (likely tab already closed)", err);
|
||||
}
|
||||
}
|
||||
|
||||
/** True iff `ensureAttached(tabId)` has succeeded since the last detach. */
|
||||
@@ -483,17 +453,6 @@ export class ChromiumCdp {
|
||||
this.tabOwners.set(tabId, owners);
|
||||
}
|
||||
|
||||
/** Release one session's claim, preserving attachments still used by another. */
|
||||
async releaseSessionTab(sessionId: string, tabId: number): Promise<void> {
|
||||
const owners = this.tabOwners.get(tabId);
|
||||
if (!owners?.delete(sessionId) || owners.size > 0) return;
|
||||
this.tabOwners.delete(tabId);
|
||||
// An observation may still be attaching when the tab is returned. Wait
|
||||
// for it so detach cannot miss the attachment or remove a new owner's claim.
|
||||
await this.attachInFlight.get(tabId)?.catch(() => {});
|
||||
if (!this.tabOwners.has(tabId)) await this.detach(tabId);
|
||||
}
|
||||
|
||||
/** Subscribe to all CDP events. Returned disposable removes the listener. */
|
||||
onEvent(handler: (source: CdpDebuggee, method: string, params: unknown) => void): {
|
||||
dispose(): void;
|
||||
@@ -510,7 +469,6 @@ export class ChromiumCdp {
|
||||
this.attachInFlight.clear();
|
||||
this.tabOwners.clear();
|
||||
this.attachedTabs.clear();
|
||||
this.attachmentIds.clear();
|
||||
this.dialogBuffers.clear();
|
||||
this.dialogSequences.clear();
|
||||
this.consoleBuffers.clear();
|
||||
@@ -681,23 +639,13 @@ export class ChromiumCdp {
|
||||
}
|
||||
|
||||
private async onJavaScriptDialogOpening(tabId: number, params: unknown): Promise<void> {
|
||||
if (!this.attachedTabs.has(tabId) && !this.attachInFlight.has(tabId)) return;
|
||||
const parsed = parseDialogOpeningParams(params);
|
||||
try {
|
||||
if (
|
||||
this.options.shouldAutoAcceptDialog &&
|
||||
!(await this.options.shouldAutoAcceptDialog(tabId))
|
||||
) {
|
||||
return;
|
||||
}
|
||||
// The tab may have been returned while its current scope was checked.
|
||||
if (!this.attachedTabs.has(tabId) && !this.attachInFlight.has(tabId)) return;
|
||||
const handleParams: { accept: boolean; promptText?: string } = { accept: true };
|
||||
if (parsed.type === "prompt") {
|
||||
handleParams.promptText = parsed.defaultPrompt ?? "";
|
||||
}
|
||||
await this.api.sendCommand({ tabId }, "Page.handleJavaScriptDialog", handleParams);
|
||||
if (!this.attachedTabs.has(tabId) && !this.attachInFlight.has(tabId)) return;
|
||||
const sequence = (this.dialogSequences.get(tabId) ?? 0) + 1;
|
||||
this.dialogSequences.set(tabId, sequence);
|
||||
this.appendDialog(tabId, {
|
||||
@@ -857,7 +805,6 @@ export class ChromiumCdp {
|
||||
const listener = (source: chrome.debugger.Debuggee, _reason: string) => {
|
||||
if (typeof source.tabId === "number") {
|
||||
this.attachedTabs.delete(source.tabId);
|
||||
this.attachmentIds.delete(source.tabId);
|
||||
this.attachInFlight.delete(source.tabId);
|
||||
this.tabOwners.delete(source.tabId);
|
||||
this.clearDialogState(source.tabId);
|
||||
@@ -888,9 +835,15 @@ export class ChromiumCdp {
|
||||
|
||||
/** Detach tabs only when no other live session has claimed them. */
|
||||
async detachSession(sessionId: string): Promise<void> {
|
||||
await Promise.all(
|
||||
Array.from(this.tabOwners.keys(), (tabId) => this.releaseSessionTab(sessionId, tabId)),
|
||||
);
|
||||
const tabsToDetach: number[] = [];
|
||||
for (const [tabId, owners] of this.tabOwners) {
|
||||
owners.delete(sessionId);
|
||||
if (owners.size === 0) {
|
||||
this.tabOwners.delete(tabId);
|
||||
tabsToDetach.push(tabId);
|
||||
}
|
||||
}
|
||||
await Promise.all(tabsToDetach.map((tabId) => this.detach(tabId)));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -45,32 +45,10 @@ function portHarness() {
|
||||
|
||||
describe("RecordFrameAgent", () => {
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals();
|
||||
document.documentElement.removeAttribute(RECORD_DOCUMENT_ATTRIBUTE);
|
||||
document.body.replaceChildren();
|
||||
});
|
||||
|
||||
it("starts when crypto.randomUUID is unavailable", async () => {
|
||||
const harness = portHarness();
|
||||
const getRandomValues = vi.fn((bytes: Uint8Array) => {
|
||||
bytes.fill(0x11);
|
||||
return bytes;
|
||||
});
|
||||
vi.stubGlobal("crypto", { getRandomValues });
|
||||
vi.stubGlobal("chrome", {
|
||||
runtime: {
|
||||
connect: vi.fn(() => harness.port),
|
||||
sendMessage: vi.fn(),
|
||||
},
|
||||
});
|
||||
const agent = new RecordFrameAgent();
|
||||
|
||||
await expect(
|
||||
agent.start({ type: RECORD_FRAME_START, requestId: "rec-http", startedAtMs: 10 }),
|
||||
).resolves.toEqual({ ok: true });
|
||||
expect(getRandomValues).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("keeps a failed stop retryable and flushes the final dirty fill", async () => {
|
||||
const harness = portHarness();
|
||||
const sendMessage = vi
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
import { createRandomUuid } from "@/lib/random-uuid";
|
||||
import {
|
||||
isAcceptedRecordStepAck,
|
||||
RECORD_STEP,
|
||||
@@ -19,7 +18,7 @@ export class RecordStepDelivery {
|
||||
constructor(
|
||||
requestId: string,
|
||||
send: SendRecordStepMessage = (message) => chrome.runtime.sendMessage(message),
|
||||
producerId: string = createRandomUuid(),
|
||||
producerId: string = crypto.randomUUID(),
|
||||
) {
|
||||
this.#requestId = requestId;
|
||||
this.#producerId = producerId;
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
import { createRandomUuid } from "@/lib/random-uuid";
|
||||
import {
|
||||
markRecordingDocument,
|
||||
type RecordingDocumentMarker,
|
||||
@@ -68,7 +67,7 @@ export class RecordFrameAgent {
|
||||
async #start(
|
||||
message: RecordFrameStartMessage,
|
||||
): Promise<{ ok: true } | { ok: false; error: string }> {
|
||||
const producerId = createRandomUuid();
|
||||
const producerId = crypto.randomUUID();
|
||||
const root = await waitForDocumentElement();
|
||||
if (this.#disposed) return { ok: false, error: "recording document is disposed" };
|
||||
const marker = markRecordingDocument(producerId, root);
|
||||
|
||||
@@ -4,7 +4,9 @@ import { ConnectionController } from "@/lib/connection-controller";
|
||||
import { startHeartbeat } from "@/lib/heartbeat";
|
||||
import {
|
||||
getConnectionEnabled,
|
||||
getDaemonPort,
|
||||
setConnectionEnabled as persistConnectionEnabled,
|
||||
STORAGE_KEYS,
|
||||
setLabel,
|
||||
} from "@/lib/instance-id";
|
||||
import { startKeepalive } from "@/lib/keepalive";
|
||||
@@ -25,7 +27,7 @@ import { recordFrameCoordinator } from "@/lib/recording/frame-coordinator";
|
||||
import { attachSessionsLiveFlag } from "@/lib/sessions-live-flag";
|
||||
import { createDisconnectCleanup } from "@/session-manager/disconnect-cleanup";
|
||||
import { attachSessionEventHandler } from "@/session-manager/event-handler";
|
||||
import { isAgentControlledTab, SessionManager } from "@/session-manager/manager";
|
||||
import { SessionManager } from "@/session-manager/manager";
|
||||
import {
|
||||
attachBorrowNotificationButtonHandler,
|
||||
attachBorrowNotificationClickHandler,
|
||||
@@ -41,8 +43,7 @@ import {
|
||||
attachRecordStepListener,
|
||||
type RecordRuntimeDeps,
|
||||
} from "@/tools/record";
|
||||
import { chromeTabsApi } from "@/tools/shared";
|
||||
import { chromeTabMutationApi } from "@/tools/tabs";
|
||||
import { resolveDaemonWsUrl } from "@/transport/daemon-endpoint";
|
||||
import { detectBrowserMeta } from "@/transport/handshake";
|
||||
import type { Transport } from "@/transport/transport";
|
||||
import { WSTransport } from "@/transport/ws-transport";
|
||||
@@ -51,16 +52,32 @@ export default defineBackground(() => {
|
||||
const controller = new ConnectionController();
|
||||
const transport = new WSTransport({ url: __BSK_DAEMON_WS_URL__ });
|
||||
const sessions = new SessionManager();
|
||||
const cdp = new ChromiumCdp(undefined, {
|
||||
shouldAutoAcceptDialog: async (tabId) => {
|
||||
const tab = await chrome.tabs.get(tabId);
|
||||
return sessions.findByWindowId(tab.windowId) !== null;
|
||||
},
|
||||
});
|
||||
const cdp = new ChromiumCdp();
|
||||
const sessionsLive = attachSessionsLiveFlag({ manager: sessions });
|
||||
let overlayGeneration = 0;
|
||||
const controlModes = new Map<string, OverlayMode>();
|
||||
|
||||
async function applyDaemonPort(port: number): Promise<void> {
|
||||
const url = resolveDaemonWsUrl(port);
|
||||
if (!transport.setUrl(url)) return;
|
||||
await transport.disconnect();
|
||||
if (!controller.isConnectionEnabled) return;
|
||||
try {
|
||||
await transport.connect();
|
||||
} catch (err) {
|
||||
console.debug("[browser-skill] reconnect after port change failed", err);
|
||||
}
|
||||
}
|
||||
|
||||
if (typeof chrome !== "undefined" && chrome.storage?.onChanged) {
|
||||
chrome.storage.onChanged.addListener((changes, areaName) => {
|
||||
if (areaName !== "local") return;
|
||||
const change = changes[STORAGE_KEYS.DAEMON_PORT];
|
||||
if (!change || typeof change.newValue !== "number") return;
|
||||
void applyDaemonPort(change.newValue);
|
||||
});
|
||||
}
|
||||
|
||||
function setControlMode(sessionId: string, mode: OverlayMode): void {
|
||||
if (controlModes.get(sessionId) === mode) return;
|
||||
controlModes.set(sessionId, mode);
|
||||
@@ -87,24 +104,6 @@ export default defineBackground(() => {
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Authoritative overlay state for a specific tab. Agent Window tabs are
|
||||
* free by default; only tabs explicitly claimed through session startup,
|
||||
* `tab_create`, or `tab_borrow` receive the control overlay.
|
||||
*/
|
||||
function overlayStateForTab(tabId?: number, windowId?: number): OverlayAgentStateMessage {
|
||||
if (typeof tabId === "number" && typeof windowId === "number") {
|
||||
const ctx = sessions.findByWindowId(windowId);
|
||||
if (ctx && isAgentControlledTab(ctx, tabId)) return overlayStateForWindow(windowId);
|
||||
}
|
||||
return {
|
||||
type: OVERLAY_AGENT_STATE,
|
||||
sessionId: null,
|
||||
mode: "hidden",
|
||||
generation: overlayGeneration,
|
||||
};
|
||||
}
|
||||
|
||||
async function pushOverlayStateToTab(
|
||||
tabId: number,
|
||||
state: OverlayAgentStateMessage,
|
||||
@@ -116,18 +115,13 @@ export default defineBackground(() => {
|
||||
}
|
||||
}
|
||||
|
||||
async function pushOverlayStateForTab(tabId: number, windowId?: number): Promise<void> {
|
||||
const state = overlayStateForTab(tabId, windowId);
|
||||
await pushOverlayStateToTab(tabId, state);
|
||||
}
|
||||
|
||||
async function pushOverlayStateForWindow(windowId: number): Promise<void> {
|
||||
const state = overlayStateForWindow(windowId);
|
||||
const tabs = await chrome.tabs.query({ windowId });
|
||||
await Promise.all(
|
||||
tabs.map((tab) => {
|
||||
if (typeof tab.id !== "number") return Promise.resolve();
|
||||
return pushOverlayStateForTab(tab.id, windowId);
|
||||
}),
|
||||
tabs.map((tab) =>
|
||||
typeof tab.id === "number" ? pushOverlayStateToTab(tab.id, state) : Promise.resolve(),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
@@ -166,30 +160,13 @@ export default defineBackground(() => {
|
||||
if (typeof tab.windowId !== "number") return;
|
||||
pushOverlayStateForAgentWindow(tab.windowId);
|
||||
});
|
||||
// Newly observed tabs are free unless the agent's own creation path has
|
||||
// already claimed their concrete id. This listener only renders that
|
||||
// state; it never infers or mutates ownership from event ordering.
|
||||
chrome.tabs.onCreated.addListener((tab) => {
|
||||
if (typeof tab.windowId !== "number" || typeof tab.id !== "number") return;
|
||||
if (!sessions.findByWindowId(tab.windowId)) return;
|
||||
void pushOverlayStateForTab(tab.id, tab.windowId);
|
||||
});
|
||||
chrome.tabs.onRemoved.addListener((tabId, removeInfo) => {
|
||||
sessions.forgetClosedTab(tabId, { isWindowClosing: removeInfo.isWindowClosing });
|
||||
});
|
||||
// Re-sync the storage.session flag on SW startup so a previous SW's
|
||||
// stale `true` does not keep waking us on every page load until the
|
||||
// first mutation (review M4/M5 round 3 m-R3-1).
|
||||
void sessionsLive.refresh();
|
||||
const cleanupAfterDisconnect = createDisconnectCleanup({
|
||||
manager: sessions,
|
||||
// Same contract as the dispatcher's `tool.session_stop`: without these
|
||||
// deps the agent-tab cleanup and the window-release path are dead code.
|
||||
sessionStopDeps: {
|
||||
cdp,
|
||||
tabManagement: { tabs: chromeTabMutationApi },
|
||||
tabsQuery: chromeTabsApi,
|
||||
},
|
||||
sessionStopDeps: { cdp },
|
||||
onSessionsChanged: () => {
|
||||
void sessionsLive.syncFromManager();
|
||||
},
|
||||
@@ -223,9 +200,6 @@ export default defineBackground(() => {
|
||||
recording: recordDeps,
|
||||
onSessionsChanged: onOverlaySessionStateChanged,
|
||||
onBrowserControlResumed,
|
||||
onAgentTabClaimed: (tabId, windowId) => {
|
||||
void pushOverlayStateForTab(tabId, windowId);
|
||||
},
|
||||
approveBorrow: (ctx) =>
|
||||
requestBorrowConfirmation(ctx.tabId, {
|
||||
...(ctx.signal !== undefined ? { signal: ctx.signal } : {}),
|
||||
@@ -329,6 +303,8 @@ export default defineBackground(() => {
|
||||
|
||||
void (async () => {
|
||||
const connectionEnabled = await getConnectionEnabled();
|
||||
const port = await getDaemonPort();
|
||||
transport.setUrl(resolveDaemonWsUrl(port));
|
||||
await controller.attach(transport, detectBrowserMeta(), connectionEnabled, {
|
||||
beforeDisconnect: async () => {
|
||||
const report = await cleanupAfterDisconnect();
|
||||
@@ -359,11 +335,7 @@ export default defineBackground(() => {
|
||||
}
|
||||
|
||||
if (msg.kind === OVERLAY_MSG_READY) {
|
||||
// Resolve from explicit tab ownership before sending so free tabs never
|
||||
// flash the control mask.
|
||||
if (typeof sender.tab?.id === "number") {
|
||||
void pushOverlayStateForTab(sender.tab.id, sender.tab.windowId);
|
||||
}
|
||||
sendResponse(overlayStateForWindow(sender.tab?.windowId));
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -399,11 +371,6 @@ export default defineBackground(() => {
|
||||
if (msg && typeof msg === "object" && "kind" in msg) {
|
||||
if (msg.kind === "set_label") {
|
||||
void setLabel(msg.value).then(() => controller.refreshLabel());
|
||||
} else if (msg.kind === "set_port") {
|
||||
// Placeholder for the future custom-port UI; warn loudly so
|
||||
// any reintroduced popup control is caught instead of
|
||||
// silently doing nothing (review M4/M5 C2).
|
||||
console.warn("[browser-skill] set_port is not wired yet; ignoring", msg.value);
|
||||
} else if (msg.kind === "set_connection_enabled") {
|
||||
void controller
|
||||
.setConnectionEnabled(msg.value)
|
||||
|
||||
@@ -2,6 +2,7 @@ import { cleanup, fireEvent, render, screen, waitFor } from "@testing-library/re
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import type { SnapshotInfo } from "@/lib/connection-controller";
|
||||
import { STORAGE_KEYS } from "@/lib/instance-id";
|
||||
import { DEFAULT_DAEMON_PORT } from "@/transport/daemon-endpoint";
|
||||
import { EXTENSION_VERSION } from "@/transport/handshake";
|
||||
import { App } from "./App";
|
||||
import { useConnectionState } from "./use-connection-state";
|
||||
@@ -66,9 +67,32 @@ describe("App", () => {
|
||||
render(<App />);
|
||||
|
||||
expect(screen.getByText("未连接")).toBeTruthy();
|
||||
expect(screen.getByText("无法连接，请确认 daemon 已启动且端口一致。")).toBeTruthy();
|
||||
expect(screen.queryByText("请先打开 BrowserSkill。")).toBeNull();
|
||||
});
|
||||
|
||||
it("shows the connection switch off and hides transport errors when disconnected", () => {
|
||||
mockUseConnectionState.mockReturnValue({
|
||||
snapshot: {
|
||||
...baseSnapshot,
|
||||
lastError: "[WSTransport] disconnect during connect",
|
||||
},
|
||||
statusState: "disconnected",
|
||||
setLabel,
|
||||
setConnectionEnabled,
|
||||
});
|
||||
|
||||
render(<App />);
|
||||
|
||||
expect(screen.getByText("未连接")).toBeTruthy();
|
||||
expect(screen.getByText("无法连接，请确认 daemon 已启动且端口一致。")).toBeTruthy();
|
||||
expect(screen.queryByText("端口不匹配")).toBeNull();
|
||||
expect(
|
||||
screen.getByRole("switch", { name: "BrowserSkill 连接" }).getAttribute("aria-checked"),
|
||||
).toBe("false");
|
||||
expect(screen.queryByText("[WSTransport] disconnect during connect")).toBeNull();
|
||||
});
|
||||
|
||||
it("does not render record UI on the main view", () => {
|
||||
render(<App />);
|
||||
|
||||
@@ -122,8 +146,8 @@ describe("App", () => {
|
||||
|
||||
expect(screen.queryByText(/^扩展 v/)).toBeNull();
|
||||
expect(screen.queryByText(/^daemon v/)).toBeNull();
|
||||
expect(screen.getByTitle("扩展版本").textContent).toBe(`Ext ${EXTENSION_VERSION}`);
|
||||
expect(screen.getByTitle("bsk 版本").textContent).toBe(`CLI ${mockDaemonVersion}`);
|
||||
expect(screen.getByTitle("扩展版本").textContent).toBe(EXTENSION_VERSION);
|
||||
expect(screen.getByTitle("bsk 版本").textContent).toBe(mockDaemonVersion);
|
||||
expect(screen.getByText("03c3e47f")).toBeTruthy();
|
||||
|
||||
const copyButton = screen.getByRole("button", { name: "复制实例 ID" });
|
||||
@@ -136,6 +160,13 @@ describe("App", () => {
|
||||
});
|
||||
|
||||
it("renders the connection toggle with switch semantics", () => {
|
||||
mockUseConnectionState.mockReturnValue({
|
||||
snapshot: { ...baseSnapshot, state: "connected" },
|
||||
statusState: "connected",
|
||||
setLabel,
|
||||
setConnectionEnabled,
|
||||
});
|
||||
|
||||
render(<App />);
|
||||
|
||||
const toggle = screen.getByRole("switch", { name: "BrowserSkill 连接" });
|
||||
@@ -143,6 +174,13 @@ describe("App", () => {
|
||||
});
|
||||
|
||||
it("calls setConnectionEnabled(false) when the toggle is turned off", () => {
|
||||
mockUseConnectionState.mockReturnValue({
|
||||
snapshot: { ...baseSnapshot, state: "connected" },
|
||||
statusState: "connected",
|
||||
setLabel,
|
||||
setConnectionEnabled,
|
||||
});
|
||||
|
||||
render(<App />);
|
||||
|
||||
fireEvent.click(screen.getByRole("switch", { name: "BrowserSkill 连接" }));
|
||||
@@ -160,6 +198,7 @@ describe("App", () => {
|
||||
render(<App />);
|
||||
|
||||
expect(screen.getByText("连接已关闭")).toBeTruthy();
|
||||
expect(screen.queryByText("无法连接，请确认 daemon 已启动且端口一致。")).toBeNull();
|
||||
expect(
|
||||
screen.getByRole("switch", { name: "BrowserSkill 连接" }).getAttribute("aria-checked"),
|
||||
).toBe("false");
|
||||
@@ -354,14 +393,20 @@ describe("control hints toggle", () => {
|
||||
|
||||
const info = await screen.findByRole("button", { name: "控制提示说明" });
|
||||
expect(info).toBeTruthy();
|
||||
const tooltip = screen.getByRole("tooltip");
|
||||
expect(tooltip.textContent).toBe("Agent 控制页面时显示提示条和橙色闪光。");
|
||||
const tooltip = screen.getByText("Agent 控制页面时显示提示条和橙色闪光。");
|
||||
expect(tooltip.getAttribute("role")).toBe("tooltip");
|
||||
// Hidden until the info button is hovered or focused.
|
||||
expect(tooltip.className).toContain("opacity-0");
|
||||
});
|
||||
|
||||
it("uses the same switch component and size for both settings rows", async () => {
|
||||
stubChromeStorage();
|
||||
mockUseConnectionState.mockReturnValue({
|
||||
snapshot: { ...baseSnapshot, state: "connected" },
|
||||
statusState: "connected",
|
||||
setLabel: vi.fn(),
|
||||
setConnectionEnabled: vi.fn(),
|
||||
});
|
||||
|
||||
render(<App />);
|
||||
|
||||
@@ -374,3 +419,109 @@ describe("control hints toggle", () => {
|
||||
expect(hintsToggle.className).toBe(connectionToggle.className);
|
||||
});
|
||||
});
|
||||
|
||||
describe("daemon port input", () => {
|
||||
function stubChromeStorage(initial: Record<string, unknown> = {}) {
|
||||
const store = { ...initial };
|
||||
vi.stubGlobal("chrome", {
|
||||
runtime: { lastError: undefined },
|
||||
storage: {
|
||||
local: {
|
||||
get: (keys: string | string[], cb: (items: Record<string, unknown>) => void) => {
|
||||
const items: Record<string, unknown> = {};
|
||||
for (const k of Array.isArray(keys) ? keys : [keys]) {
|
||||
if (k in store) items[k] = store[k];
|
||||
}
|
||||
cb(items);
|
||||
},
|
||||
set: (items: Record<string, unknown>, cb?: () => void) => {
|
||||
Object.assign(store, items);
|
||||
cb?.();
|
||||
},
|
||||
},
|
||||
onChanged: {
|
||||
addListener: vi.fn(),
|
||||
removeListener: vi.fn(),
|
||||
},
|
||||
},
|
||||
});
|
||||
return store;
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
cleanup();
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
it("prefills the port from storage", async () => {
|
||||
stubChromeStorage({ [STORAGE_KEYS.DAEMON_PORT]: 53200 });
|
||||
|
||||
render(<App />);
|
||||
|
||||
const input = await screen.findByRole("textbox", { name: "连接端口" });
|
||||
await waitFor(() => expect((input as HTMLInputElement).value).toBe("53200"));
|
||||
});
|
||||
|
||||
it("persists a valid port on blur", async () => {
|
||||
const store = stubChromeStorage();
|
||||
|
||||
render(<App />);
|
||||
|
||||
const input = await screen.findByRole("textbox", { name: "连接端口" });
|
||||
fireEvent.change(input, { target: { value: "53200" } });
|
||||
fireEvent.blur(input);
|
||||
|
||||
expect(store[STORAGE_KEYS.DAEMON_PORT]).toBe(53200);
|
||||
expect((input as HTMLInputElement).value).toBe("53200");
|
||||
});
|
||||
|
||||
it("persists a valid port when Enter blurs the field", async () => {
|
||||
const store = stubChromeStorage();
|
||||
|
||||
render(<App />);
|
||||
|
||||
const input = await screen.findByRole("textbox", { name: "连接端口" });
|
||||
fireEvent.change(input, { target: { value: "53200" } });
|
||||
fireEvent.keyDown(input, { key: "Enter" });
|
||||
|
||||
expect(store[STORAGE_KEYS.DAEMON_PORT]).toBe(53200);
|
||||
});
|
||||
|
||||
it("shows an error and does not write invalid ports", async () => {
|
||||
const store = stubChromeStorage();
|
||||
|
||||
render(<App />);
|
||||
|
||||
const input = await screen.findByRole("textbox", { name: "连接端口" });
|
||||
fireEvent.change(input, { target: { value: "abc" } });
|
||||
fireEvent.blur(input);
|
||||
|
||||
expect(screen.getByText("请输入 1 到 65535 之间的端口号。")).toBeTruthy();
|
||||
expect(store[STORAGE_KEYS.DAEMON_PORT]).toBeUndefined();
|
||||
});
|
||||
|
||||
it("stores the default port when the field is cleared", async () => {
|
||||
const store = stubChromeStorage({ [STORAGE_KEYS.DAEMON_PORT]: 53200 });
|
||||
|
||||
render(<App />);
|
||||
|
||||
const input = await screen.findByRole("textbox", { name: "连接端口" });
|
||||
fireEvent.change(input, { target: { value: "" } });
|
||||
fireEvent.blur(input);
|
||||
|
||||
expect(store[STORAGE_KEYS.DAEMON_PORT]).toBe(DEFAULT_DAEMON_PORT);
|
||||
expect((input as HTMLInputElement).value).toBe(String(DEFAULT_DAEMON_PORT));
|
||||
});
|
||||
|
||||
it("keeps the port hint copy in an accessible info tooltip", async () => {
|
||||
stubChromeStorage();
|
||||
|
||||
render(<App />);
|
||||
|
||||
const info = await screen.findByRole("button", { name: "连接端口说明" });
|
||||
expect(info).toBeTruthy();
|
||||
const tooltip = screen.getByText("扩展通过此端口连接本机 daemon。非必要请勿修改。");
|
||||
expect(tooltip.getAttribute("role")).toBe("tooltip");
|
||||
expect(tooltip.className).toContain("opacity-0");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -15,6 +15,7 @@ import { POPUP_FEATURES, type PopupView } from "./features";
|
||||
import { Switch } from "./switch";
|
||||
import { type PopupStatusState, useConnectionState } from "./use-connection-state";
|
||||
import { useControlHintsHidden } from "./use-control-hints-hidden";
|
||||
import { useDaemonPort } from "./use-daemon-port";
|
||||
|
||||
const STATE_LABEL_KEYS = {
|
||||
disconnected: "popup.stateLabel.disconnected",
|
||||
@@ -41,6 +42,12 @@ export function App() {
|
||||
const { t } = useTranslation("extension");
|
||||
const { snapshot, statusState, setConnectionEnabled } = useConnectionState();
|
||||
const [controlHintsHidden, setControlHintsHidden] = useControlHintsHidden();
|
||||
const {
|
||||
draft: daemonPortDraft,
|
||||
setDraft: setDaemonPortDraft,
|
||||
commit: commitDaemonPort,
|
||||
invalid: daemonPortInvalid,
|
||||
} = useDaemonPort();
|
||||
const [view, setView] = useState<PopupView>("main");
|
||||
const [copiedInstanceId, setCopiedInstanceId] = useState(false);
|
||||
const [purposeDraft, setPurposeDraft] = useState("");
|
||||
@@ -79,6 +86,7 @@ export function App() {
|
||||
}, [copiedTick]);
|
||||
|
||||
const isSkewed = statusState === "version_skew";
|
||||
const isDisconnected = statusState === "disconnected";
|
||||
const connectionLive = statusState === "connected" || isSkewed;
|
||||
const daemonVersion = snapshot.handshake?.version ?? "—";
|
||||
const daemonProtocol = snapshot.handshake?.protocol_version ?? "—";
|
||||
@@ -198,13 +206,21 @@ export function App() {
|
||||
{t(STATE_BADGE_KEYS[statusState])}
|
||||
</Badge>
|
||||
<Switch
|
||||
checked={snapshot.connectionEnabled}
|
||||
checked={snapshot.connectionEnabled && !isDisconnected}
|
||||
onCheckedChange={setConnectionEnabled}
|
||||
aria-label={t("popup.connectionToggleTitle")}
|
||||
data-slot="popup-connection-toggle"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
{isDisconnected && (
|
||||
<p
|
||||
className="mt-2 text-xs leading-snug text-muted-foreground"
|
||||
data-slot="popup-daemon-unreachable"
|
||||
>
|
||||
{t("popup.daemonUnreachable")}
|
||||
</p>
|
||||
)}
|
||||
{isSkewed && (
|
||||
<p
|
||||
className="mt-2 text-xs leading-snug text-muted-foreground"
|
||||
@@ -253,7 +269,66 @@ export function App() {
|
||||
</div>
|
||||
</section>
|
||||
|
||||
{snapshot.lastError && (
|
||||
<section
|
||||
className="rounded-xl border border-border/80 bg-card/60 px-3 py-2.5"
|
||||
data-slot="popup-daemon-port-card"
|
||||
>
|
||||
<div className="flex items-center justify-between gap-2">
|
||||
<span className="flex min-w-0 items-center gap-1">
|
||||
<Label
|
||||
htmlFor="bh-daemon-port"
|
||||
className="truncate text-sm font-medium leading-normal"
|
||||
>
|
||||
{t("popup.daemonPortLabel")}
|
||||
</Label>
|
||||
<span className="group relative inline-flex shrink-0">
|
||||
<button
|
||||
type="button"
|
||||
aria-label={t("popup.daemonPortInfoLabel")}
|
||||
data-slot="popup-daemon-port-info"
|
||||
className="flex size-4 items-center justify-center rounded-full text-muted-foreground/70 transition-colors hover:text-muted-foreground focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring/50"
|
||||
>
|
||||
<RiInformationLine className="size-3.5" aria-hidden />
|
||||
</button>
|
||||
<span
|
||||
role="tooltip"
|
||||
className="pointer-events-none absolute bottom-full left-0 z-10 mb-1.5 w-56 whitespace-normal rounded-md bg-foreground/65 px-2 py-1 text-[10px] font-medium leading-snug text-background opacity-0 shadow-md backdrop-blur-sm transition-opacity group-hover:opacity-100 group-focus-within:opacity-100"
|
||||
>
|
||||
{t("popup.daemonPortHint")}
|
||||
</span>
|
||||
</span>
|
||||
</span>
|
||||
<Input
|
||||
id="bh-daemon-port"
|
||||
type="text"
|
||||
inputMode="numeric"
|
||||
value={daemonPortDraft}
|
||||
onChange={(event: ChangeEvent<HTMLInputElement>) =>
|
||||
setDaemonPortDraft(event.target.value)
|
||||
}
|
||||
onBlur={commitDaemonPort}
|
||||
onKeyDown={(event) => {
|
||||
if (event.key === "Enter") {
|
||||
event.preventDefault();
|
||||
commitDaemonPort();
|
||||
}
|
||||
}}
|
||||
className="mt-0 h-7.5 w-19 shrink-0 rounded-md px-2 py-0 text-center text-sm leading-none shadow-none"
|
||||
aria-invalid={daemonPortInvalid || undefined}
|
||||
data-slot="popup-daemon-port-input"
|
||||
/>
|
||||
</div>
|
||||
{daemonPortInvalid && (
|
||||
<p
|
||||
className="mt-1.5 text-[11px] leading-snug text-destructive"
|
||||
data-slot="popup-daemon-port-error"
|
||||
>
|
||||
{t("popup.daemonPortInvalid")}
|
||||
</p>
|
||||
)}
|
||||
</section>
|
||||
|
||||
{snapshot.lastError && !isDisconnected && (
|
||||
<div
|
||||
className="rounded-lg border border-destructive/25 bg-destructive/10 px-3 py-2 text-xs leading-snug text-destructive"
|
||||
data-slot="popup-error"
|
||||
@@ -267,9 +342,9 @@ export function App() {
|
||||
data-slot="popup-meta"
|
||||
>
|
||||
<div className="flex shrink-0 items-center gap-1">
|
||||
<span title={t("popup.daemonVersionHint")}>CLI {daemonVersion}</span>
|
||||
<span title={t("popup.extensionVersionHint")}>{extensionVersion}</span>
|
||||
<span aria-hidden>/</span>
|
||||
<span title={t("popup.extensionVersionHint")}>Ext {extensionVersion}</span>
|
||||
<span title={t("popup.daemonVersionHint")}>{daemonVersion}</span>
|
||||
</div>
|
||||
<div className="flex min-w-0 items-center justify-end gap-1">
|
||||
<span className="shrink-0">{t("popup.instanceTitle")}</span>
|
||||
|
||||
@@ -40,13 +40,8 @@ const FALLBACK_SNAPSHOT: SnapshotInfo = {
|
||||
/**
|
||||
* Live snapshot of the background's `ConnectionController` for popup UI.
|
||||
*
|
||||
* Posts `set_label` mutations back to the background; the incoming
|
||||
* snapshot stream reflects the canonical state. The `set_port` channel
|
||||
* is defined by the popup-bridge but is not currently routed to the
|
||||
* background — wiring it requires `ConnectionController` to swap its
|
||||
* Transport URL and persist the chosen port. Tracked for a follow-up
|
||||
* milestone; the popup intentionally does not expose a control until
|
||||
* the underlying support lands (review M4/M5 C2).
|
||||
* Posts `set_label` and `set_connection_enabled` mutations back to the
|
||||
* background; the incoming snapshot stream reflects the canonical state.
|
||||
*/
|
||||
export function useConnectionState(): {
|
||||
snapshot: SnapshotInfo;
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
import { useCallback, useEffect, useRef, useState } from "react";
|
||||
import { getDaemonPort, STORAGE_KEYS, setDaemonPort } from "@/lib/instance-id";
|
||||
import { parseDaemonPortInput } from "@/transport/daemon-endpoint";
|
||||
|
||||
/**
|
||||
* Popup-side daemon port preference stored in `chrome.storage.local`.
|
||||
* Commits on blur, Enter, and pagehide.
|
||||
*/
|
||||
export function useDaemonPort(): {
|
||||
draft: string;
|
||||
setDraft: (value: string) => void;
|
||||
commit: () => void;
|
||||
invalid: boolean;
|
||||
} {
|
||||
const [draft, setDraftState] = useState("");
|
||||
const [invalid, setInvalid] = useState(false);
|
||||
const draftRef = useRef(draft);
|
||||
draftRef.current = draft;
|
||||
|
||||
useEffect(() => {
|
||||
if (typeof chrome === "undefined" || !chrome.storage?.local) return undefined;
|
||||
let cancelled = false;
|
||||
getDaemonPort()
|
||||
.then((port) => {
|
||||
if (!cancelled) setDraftState(String(port));
|
||||
})
|
||||
.catch((err) => {
|
||||
console.debug("[browser-skill] daemon port read failed", err);
|
||||
});
|
||||
const onChanged = (changes: Record<string, chrome.storage.StorageChange>, areaName: string) => {
|
||||
if (areaName !== "local") return;
|
||||
const change = changes[STORAGE_KEYS.DAEMON_PORT];
|
||||
if (change && typeof change.newValue === "number") {
|
||||
setDraftState(String(change.newValue));
|
||||
setInvalid(false);
|
||||
}
|
||||
};
|
||||
chrome.storage.onChanged.addListener(onChanged);
|
||||
return () => {
|
||||
cancelled = true;
|
||||
chrome.storage.onChanged.removeListener(onChanged);
|
||||
};
|
||||
}, []);
|
||||
|
||||
const commit = useCallback(() => {
|
||||
const parsed = parseDaemonPortInput(draftRef.current);
|
||||
if (parsed === null) {
|
||||
setInvalid(true);
|
||||
return;
|
||||
}
|
||||
setInvalid(false);
|
||||
setDraftState(String(parsed));
|
||||
if (typeof chrome === "undefined" || !chrome.storage?.local) return;
|
||||
setDaemonPort(parsed).catch((err) => {
|
||||
console.debug("[browser-skill] daemon port write failed", err);
|
||||
});
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
const onPageHide = () => commit();
|
||||
window.addEventListener("pagehide", onPageHide);
|
||||
return () => window.removeEventListener("pagehide", onPageHide);
|
||||
}, [commit]);
|
||||
|
||||
const setDraft = useCallback((value: string) => {
|
||||
setDraftState(value);
|
||||
setInvalid(false);
|
||||
}, []);
|
||||
|
||||
return { draft, setDraft, commit, invalid };
|
||||
}
|
||||
@@ -2,7 +2,7 @@ import { describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
CAPTURE_SUPPRESS,
|
||||
type CaptureSuppressMessage,
|
||||
withExtensionOverlayHidden,
|
||||
withOverlaysHiddenForCapture,
|
||||
} from "../capture-suppress-bridge";
|
||||
|
||||
function recordingSendToTab(events: string[]) {
|
||||
@@ -12,12 +12,12 @@ function recordingSendToTab(events: string[]) {
|
||||
});
|
||||
}
|
||||
|
||||
describe("withExtensionOverlayHidden", () => {
|
||||
describe("withOverlaysHiddenForCapture", () => {
|
||||
it("sends begin → fn → end and returns fn's result", async () => {
|
||||
const events: string[] = [];
|
||||
const sendToTab = recordingSendToTab(events);
|
||||
|
||||
const result = await withExtensionOverlayHidden(
|
||||
const result = await withOverlaysHiddenForCapture(
|
||||
7,
|
||||
async () => {
|
||||
events.push("capture");
|
||||
@@ -37,7 +37,7 @@ describe("withExtensionOverlayHidden", () => {
|
||||
const sendToTab = recordingSendToTab(events);
|
||||
|
||||
await expect(
|
||||
withExtensionOverlayHidden(
|
||||
withOverlaysHiddenForCapture(
|
||||
7,
|
||||
async () => {
|
||||
events.push("capture");
|
||||
@@ -55,7 +55,7 @@ describe("withExtensionOverlayHidden", () => {
|
||||
});
|
||||
const fn = vi.fn(async () => "shot");
|
||||
|
||||
const result = await withExtensionOverlayHidden(7, fn, sendToTab);
|
||||
const result = await withOverlaysHiddenForCapture(7, fn, sendToTab);
|
||||
|
||||
expect(result).toBe("shot");
|
||||
expect(fn).toHaveBeenCalledTimes(1);
|
||||
@@ -69,7 +69,7 @@ describe("withExtensionOverlayHidden", () => {
|
||||
return { type: CAPTURE_SUPPRESS, ok: true };
|
||||
});
|
||||
|
||||
const result = await withExtensionOverlayHidden(7, async () => "shot", sendToTab);
|
||||
const result = await withOverlaysHiddenForCapture(7, async () => "shot", sendToTab);
|
||||
|
||||
expect(result).toBe("shot");
|
||||
expect(sendToTab).toHaveBeenCalledTimes(2);
|
||||
|
||||
@@ -1,12 +1,15 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { DEFAULT_DAEMON_PORT } from "@/transport/daemon-endpoint";
|
||||
import {
|
||||
getConnectionEnabled,
|
||||
getControlHintsHidden,
|
||||
getDaemonPort,
|
||||
getLabel,
|
||||
getOrCreateInstanceId,
|
||||
STORAGE_KEYS,
|
||||
setConnectionEnabled,
|
||||
setControlHintsHidden,
|
||||
setDaemonPort,
|
||||
setLabel,
|
||||
} from "../instance-id";
|
||||
|
||||
@@ -117,4 +120,21 @@ describe("instance-id", () => {
|
||||
expect(store[STORAGE_KEYS.CONTROL_HINTS_HIDDEN]).toBe(true);
|
||||
expect(await getControlHintsHidden(backend)).toBe(true);
|
||||
});
|
||||
|
||||
it("getDaemonPort returns default when storage is empty", async () => {
|
||||
const { backend } = fakeStorage();
|
||||
expect(await getDaemonPort(backend)).toBe(DEFAULT_DAEMON_PORT);
|
||||
});
|
||||
|
||||
it("getDaemonPort normalizes invalid stored values to default", async () => {
|
||||
const { backend } = fakeStorage({ [STORAGE_KEYS.DAEMON_PORT]: 0 });
|
||||
expect(await getDaemonPort(backend)).toBe(DEFAULT_DAEMON_PORT);
|
||||
});
|
||||
|
||||
it("setDaemonPort persists the value retrievable by getDaemonPort", async () => {
|
||||
const { backend, store } = fakeStorage();
|
||||
await setDaemonPort(53200, backend);
|
||||
expect(store[STORAGE_KEYS.DAEMON_PORT]).toBe(53200);
|
||||
expect(await getDaemonPort(backend)).toBe(53200);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,30 +0,0 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { createRandomUuid } from "../random-uuid";
|
||||
|
||||
describe("createRandomUuid", () => {
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
it("uses crypto.randomUUID when it is available", () => {
|
||||
const randomUUID = vi.fn(() => "00000000-0000-4000-8000-000000000000");
|
||||
vi.stubGlobal("crypto", { randomUUID, getRandomValues: vi.fn() });
|
||||
|
||||
expect(createRandomUuid()).toBe("00000000-0000-4000-8000-000000000000");
|
||||
expect(randomUUID).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("falls back to getRandomValues and sets UUID v4 bits", () => {
|
||||
const bytes = new Uint8Array(16).fill(0xff);
|
||||
const getRandomValues = vi.fn((target: Uint8Array) => {
|
||||
target.set(bytes);
|
||||
return target;
|
||||
});
|
||||
vi.stubGlobal("crypto", { getRandomValues });
|
||||
|
||||
const uuid = createRandomUuid();
|
||||
|
||||
expect(uuid).toBe("ffffffff-ffff-4fff-bfff-ffffffffffff");
|
||||
expect(getRandomValues).toHaveBeenCalledOnce();
|
||||
});
|
||||
});
|
||||
@@ -1,9 +1,9 @@
|
||||
/**
|
||||
* Wire protocol and background-side helper for temporarily excluding the
|
||||
* extension's in-page overlay. Screenshots use it to avoid captured chrome;
|
||||
* coordinate-driven native input uses it so Chrome hit-tests the page.
|
||||
* Wire protocol and background-side helper for hiding the extension's
|
||||
* in-page overlay while a screenshot is captured, so captured frames only
|
||||
* contain the page itself.
|
||||
*
|
||||
* The background wraps each bounded operation in `withExtensionOverlayHidden`,
|
||||
* The background wraps every capture in `withOverlaysHiddenForCapture`,
|
||||
* which sends `begin` (the content script hides the overlay host and only
|
||||
* acks once the compositor has produced an overlay-free frame) and always
|
||||
* follows up with `end`, even when the capture throws. Tabs without the
|
||||
@@ -46,7 +46,7 @@ const defaultSendToTab: CaptureSuppressSendToTab = (tabId, message) =>
|
||||
* (or any other `begin` failure) simply skips suppression — there is no
|
||||
* overlay to hide in that tab.
|
||||
*/
|
||||
export async function withExtensionOverlayHidden<T>(
|
||||
export async function withOverlaysHiddenForCapture<T>(
|
||||
tabId: number,
|
||||
fn: () => Promise<T>,
|
||||
sendToTab: CaptureSuppressSendToTab = defaultSendToTab,
|
||||
|
||||
@@ -1,7 +1,10 @@
|
||||
import { normalizeDaemonPort } from "@/transport/daemon-endpoint";
|
||||
|
||||
const STORAGE_KEY = "bsk_instance_id";
|
||||
const LABEL_STORAGE_KEY = "bh_label";
|
||||
const CONNECTION_ENABLED_KEY = "bh_connection_enabled";
|
||||
const CONTROL_HINTS_HIDDEN_KEY = "bsk_control_hints_hidden";
|
||||
const DAEMON_PORT_KEY = "bsk_daemon_port";
|
||||
|
||||
export interface StorageBackend {
|
||||
get(keys: string | string[]): Promise<Record<string, unknown>>;
|
||||
@@ -127,9 +130,23 @@ export async function setControlHintsHidden(
|
||||
await storage.set({ [CONTROL_HINTS_HIDDEN_KEY]: hidden });
|
||||
}
|
||||
|
||||
/** Defaults to {@link DEFAULT_DAEMON_PORT} when unset or invalid. */
|
||||
export async function getDaemonPort(storage: StorageBackend = defaultStorage()): Promise<number> {
|
||||
const items = await storage.get(DAEMON_PORT_KEY);
|
||||
return normalizeDaemonPort(items[DAEMON_PORT_KEY]);
|
||||
}
|
||||
|
||||
export async function setDaemonPort(
|
||||
port: number,
|
||||
storage: StorageBackend = defaultStorage(),
|
||||
): Promise<void> {
|
||||
await storage.set({ [DAEMON_PORT_KEY]: port });
|
||||
}
|
||||
|
||||
export const STORAGE_KEYS = {
|
||||
INSTANCE_ID: STORAGE_KEY,
|
||||
LABEL: LABEL_STORAGE_KEY,
|
||||
CONNECTION_ENABLED: CONNECTION_ENABLED_KEY,
|
||||
CONTROL_HINTS_HIDDEN: CONTROL_HINTS_HIDDEN_KEY,
|
||||
DAEMON_PORT: DAEMON_PORT_KEY,
|
||||
} as const;
|
||||
|
||||
@@ -3,10 +3,8 @@
|
||||
* between the content-script control overlay and the background SW.
|
||||
*
|
||||
* Content script → background:
|
||||
* - `{ kind: "overlay.ready" }` → background decides the per-tab state first
|
||||
* (see overlayStateForTab) and pushes the authoritative overlay state for
|
||||
* that tab. It also proactively pushes on tab create and control-mode
|
||||
* change, so a tab usually receives state before it ever sends ready.
|
||||
* - `{ kind: "overlay.ready" }` → background replies with the authoritative
|
||||
* overlay state for the sender's window.
|
||||
* - `{ kind: "overlay.interrupt", sessionId }` → background asks the
|
||||
* daemon (via a `session.user_interrupt` WS event) to cancel
|
||||
* every inflight + queued tool call for that session with
|
||||
|
||||
@@ -5,19 +5,14 @@ import type { SnapshotInfo } from "./connection-controller";
|
||||
* - Background pushes `{ kind: "snapshot", data: SnapshotInfo }`.
|
||||
* - Popup sends `{ kind: "set_label" }`, `{ kind: "set_connection_enabled" }`, etc.
|
||||
*
|
||||
* NOTE(review M4/M5 C2): the `set_port` variant is defined as a
|
||||
* placeholder so the future custom-port UI does not have to re-design
|
||||
* the bridge, but the popup does NOT render a control that emits it
|
||||
* yet and the background does NOT route it. The wiring requires
|
||||
* ConnectionController to dispose its current Transport and persist
|
||||
* the port in chrome.storage; tracked as a follow-up.
|
||||
* Daemon port preference is persisted via `chrome.storage.local` instead
|
||||
* of this bridge (see `use-daemon-port.ts`).
|
||||
*/
|
||||
|
||||
export const POPUP_PORT_NAME = "popup";
|
||||
|
||||
export type PopupOutbound =
|
||||
| { kind: "set_label"; value: string }
|
||||
| { kind: "set_port"; value: number }
|
||||
| { kind: "set_connection_enabled"; value: boolean };
|
||||
|
||||
export type PopupInbound = { kind: "snapshot"; data: SnapshotInfo };
|
||||
|
||||
@@ -1,26 +0,0 @@
|
||||
/**
|
||||
* Generate a UUID v4 in browser contexts where randomUUID() is unavailable.
|
||||
*
|
||||
* Crypto.randomUUID() is restricted to secure contexts, while
|
||||
* crypto.getRandomValues() is available to content scripts on ordinary HTTP
|
||||
* origins. BrowserSkill records many intranet pages served over HTTP, so the
|
||||
* fallback is required for recording to work on those pages.
|
||||
*/
|
||||
export function createRandomUuid(): string {
|
||||
const cryptoApi = globalThis.crypto;
|
||||
if (typeof cryptoApi.randomUUID === "function") return cryptoApi.randomUUID();
|
||||
|
||||
const bytes = new Uint8Array(16);
|
||||
cryptoApi.getRandomValues(bytes);
|
||||
bytes[6] = (bytes[6]! & 0x0f) | 0x40;
|
||||
bytes[8] = (bytes[8]! & 0x3f) | 0x80;
|
||||
|
||||
const hex = Array.from(bytes, (byte) => byte.toString(16).padStart(2, "0")).join("");
|
||||
return [
|
||||
hex.slice(0, 8),
|
||||
hex.slice(8, 12),
|
||||
hex.slice(12, 16),
|
||||
hex.slice(16, 20),
|
||||
hex.slice(20),
|
||||
].join("-");
|
||||
}
|
||||
@@ -1,4 +1,3 @@
|
||||
import { createRandomUuid } from "../random-uuid";
|
||||
import {
|
||||
isRecordFrameQueryMessage,
|
||||
RECORD_FRAME_PORT,
|
||||
@@ -189,7 +188,7 @@ export class RecordFrameCoordinator {
|
||||
const recording = this.#recordings.get(requestId);
|
||||
if (!recording) return true;
|
||||
recording.finishing = true;
|
||||
const commandId = createRandomUuid();
|
||||
const commandId = crypto.randomUUID();
|
||||
const results = await Promise.all(
|
||||
[...recording.agents.values()].map((agent) => this.#stopAgent(agent, requestId, commandId)),
|
||||
);
|
||||
|
||||
@@ -9,7 +9,7 @@ describe("disconnect session cleanup", () => {
|
||||
const manager = new SessionManager({
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => nextWindowId++),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
remove,
|
||||
},
|
||||
});
|
||||
@@ -41,7 +41,7 @@ describe("disconnect session cleanup", () => {
|
||||
const manager = new SessionManager({
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 100),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
remove: vi.fn(() => removeGate),
|
||||
},
|
||||
});
|
||||
|
||||
@@ -39,7 +39,7 @@ describe("attachSessionEventHandler", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 4242),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
},
|
||||
});
|
||||
await manager.start("aa11");
|
||||
@@ -73,7 +73,7 @@ describe("attachSessionEventHandler", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 4242),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
},
|
||||
});
|
||||
const ctx = await manager.start("aa11");
|
||||
@@ -87,7 +87,6 @@ describe("attachSessionEventHandler", () => {
|
||||
windowEvents: events.api,
|
||||
});
|
||||
|
||||
manager.forgetClosedTab(7, { isWindowClosing: true });
|
||||
events.emit(4242);
|
||||
for (let i = 0; i < 4; i += 1) await Promise.resolve();
|
||||
|
||||
@@ -114,7 +113,7 @@ describe("attachSessionEventHandler", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 1),
|
||||
remove: vi.fn(),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
},
|
||||
});
|
||||
const transport = fakeTransport();
|
||||
@@ -130,7 +129,7 @@ describe("attachSessionEventHandler", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 1),
|
||||
remove: vi.fn(),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
},
|
||||
});
|
||||
const transport = fakeTransport();
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import type { AgentWindowApi, AgentWindowCreateOptions } from "../agent-window";
|
||||
import { isAgentControlledTab, SessionManager } from "../manager";
|
||||
import { SessionManager } from "../manager";
|
||||
|
||||
function fakeAgentWindow(): AgentWindowApi & {
|
||||
createMock: ReturnType<typeof vi.fn>;
|
||||
@@ -13,7 +13,7 @@ function fakeAgentWindow(): AgentWindowApi & {
|
||||
return id;
|
||||
});
|
||||
const removeMock = vi.fn(async (_id: number) => {});
|
||||
const ensureActiveTabMock = vi.fn(async (_windowId: number, _url: string) => 0);
|
||||
const ensureActiveTabMock = vi.fn(async (_windowId: number, _url: string) => {});
|
||||
return {
|
||||
create: createMock,
|
||||
remove: removeMock,
|
||||
@@ -151,76 +151,6 @@ describe("SessionManager", () => {
|
||||
expect(sm.list()).toEqual([]);
|
||||
});
|
||||
|
||||
describe("explicit tab claims", () => {
|
||||
it("claims the exact home tab returned by AgentWindowApi", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow() });
|
||||
const ctx = await sm.start("aa11");
|
||||
expect(ctx.agentCreatedTabs).toEqual(new Set([0]));
|
||||
expect(isAgentControlledTab(ctx, 0)).toBe(true);
|
||||
});
|
||||
|
||||
it("keeps every other tab free without classification state", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow() });
|
||||
const ctx = await sm.start("aa11");
|
||||
expect(isAgentControlledTab(ctx, 99)).toBe(false);
|
||||
expect(ctx).not.toHaveProperty("userTabs");
|
||||
expect(ctx).not.toHaveProperty("pendingAgentTabCount");
|
||||
});
|
||||
|
||||
it("treats a committed borrow as an explicit claim", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow() });
|
||||
const ctx = await sm.start("aa11");
|
||||
const reservation = sm.tryReserveBorrow(42, ctx.sessionId);
|
||||
if ("borrowedBy" in reservation) throw new Error("unexpected borrow conflict");
|
||||
reservation.commit({ tabId: 42, originalWindowId: 7, originalIndex: 3 });
|
||||
expect(isAgentControlledTab(ctx, 42)).toBe(true);
|
||||
});
|
||||
|
||||
it("forgets an agent-created tab after Chrome removes it", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow() });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.agentCreatedTabs.add(42);
|
||||
sm.forgetClosedTab(42);
|
||||
expect(ctx.agentCreatedTabs.has(42)).toBe(false);
|
||||
expect(isAgentControlledTab(ctx, 0)).toBe(true);
|
||||
});
|
||||
|
||||
it("forgets closed borrows idempotently without affecting other tabs or sessions", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow() });
|
||||
const a = await sm.start("aa11");
|
||||
const b = await sm.start("bb22");
|
||||
a.borrowedTabs.set(42, { tabId: 42, originalWindowId: 7, originalIndex: 0 });
|
||||
a.borrowedTabs.set(43, { tabId: 43, originalWindowId: 7, originalIndex: 1 });
|
||||
b.borrowedTabs.set(44, { tabId: 44, originalWindowId: 8, originalIndex: 0 });
|
||||
|
||||
sm.forgetClosedTab(42);
|
||||
sm.forgetClosedTab(42);
|
||||
sm.forgetClosedTab(999);
|
||||
|
||||
expect(isAgentControlledTab(a, 42)).toBe(false);
|
||||
expect(sm.findBorrowingSession(42, null)).toBeNull();
|
||||
expect([...a.borrowedTabs.keys()]).toEqual([43]);
|
||||
expect([...b.borrowedTabs.keys()]).toEqual([44]);
|
||||
expect(sm.list()).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("prevents an in-flight borrow from reclaiming a closed tab", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow() });
|
||||
const ctx = await sm.start("aa11");
|
||||
const reservation = sm.tryReserveBorrow(42, ctx.sessionId);
|
||||
if ("borrowedBy" in reservation) throw new Error("unexpected borrow conflict");
|
||||
|
||||
sm.forgetClosedTab(42);
|
||||
|
||||
expect(() =>
|
||||
reservation.commit({ tabId: 42, originalWindowId: 7, originalIndex: 0 }),
|
||||
).toThrow(/reservation disappeared/);
|
||||
reservation.release();
|
||||
expect(sm.findBorrowingSession(42, null)).toBeNull();
|
||||
expect(ctx.borrowedTabs.has(42)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("findBorrowingSession", () => {
|
||||
it("returns null when no session has borrowed the tab", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow() });
|
||||
|
||||
@@ -14,11 +14,8 @@ export interface AgentWindowApi {
|
||||
* Guarantee the Agent Window has an active, CDP-navigable tab.
|
||||
* `chrome://` pages (including the New Tab page) reject `Page.navigate`,
|
||||
* so sessions bootstrap with `about:blank` instead.
|
||||
*
|
||||
* Resolves with the id of the activated (or newly created) tab, so callers
|
||||
* can track the session's home tab without re-querying Chrome.
|
||||
*/
|
||||
ensureActiveTab(windowId: number, url: string): Promise<number>;
|
||||
ensureActiveTab(windowId: number, url: string): Promise<void>;
|
||||
}
|
||||
|
||||
/** Creation hints for a new Agent Window. */
|
||||
@@ -52,19 +49,15 @@ export const chromeAgentWindowApi: AgentWindowApi = {
|
||||
// cancellation success while the Agent Window remains open.
|
||||
await chrome.windows.remove(windowId);
|
||||
},
|
||||
async ensureActiveTab(windowId: number, url: string): Promise<number> {
|
||||
async ensureActiveTab(windowId: number, url: string): Promise<void> {
|
||||
const tabs = await chrome.tabs.query({ windowId });
|
||||
const first = tabs.find((t) => typeof t.id === "number");
|
||||
if (first?.id !== undefined) {
|
||||
if (first?.id) {
|
||||
if (!first.active) {
|
||||
await chrome.tabs.update(first.id, { active: true });
|
||||
}
|
||||
return first.id;
|
||||
return;
|
||||
}
|
||||
const created = await chrome.tabs.create({ windowId, url, active: true });
|
||||
if (typeof created?.id !== "number") {
|
||||
throw new Error("[bh] chrome.tabs.create returned no tab id");
|
||||
}
|
||||
return created.id;
|
||||
await chrome.tabs.create({ windowId, url, active: true });
|
||||
},
|
||||
};
|
||||
|
||||
@@ -6,20 +6,9 @@ export interface SessionContext {
|
||||
agentWindowId: number;
|
||||
refStore: RefStore;
|
||||
borrowedTabs: Map<number, BorrowedTab>;
|
||||
/**
|
||||
* Tabs explicitly claimed by the agent because it created them. This
|
||||
* includes the Agent Window's home tab and tabs created by `tool.tab_create`.
|
||||
* Tabs opened through Chrome UI never enter this set.
|
||||
*/
|
||||
agentCreatedTabs: Set<number>;
|
||||
createdAtMs: number;
|
||||
}
|
||||
|
||||
/** Whether this session has explicitly claimed control of `tabId`. */
|
||||
export function isAgentControlledTab(ctx: SessionContext, tabId: number): boolean {
|
||||
return ctx.agentCreatedTabs.has(tabId) || ctx.borrowedTabs.has(tabId);
|
||||
}
|
||||
|
||||
export interface BorrowedTab {
|
||||
tabId: number;
|
||||
originalWindowId: number;
|
||||
@@ -117,19 +106,6 @@ export class SessionManager {
|
||||
return Array.from(this.sessions.values());
|
||||
}
|
||||
|
||||
/**
|
||||
* Forget a tab Chrome has removed, including any uncommitted borrow.
|
||||
* Whole-window closures keep committed borrows until the window-removed
|
||||
* handler reports which user tabs could not be returned.
|
||||
*/
|
||||
forgetClosedTab(tabId: number, { isWindowClosing = false } = {}): void {
|
||||
this.borrowReservations.delete(tabId);
|
||||
for (const ctx of this.sessions.values()) {
|
||||
ctx.agentCreatedTabs.delete(tabId);
|
||||
if (!isWindowClosing) ctx.borrowedTabs.delete(tabId);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Look up whether `tabId` is currently borrowed by some *other*
|
||||
* session than the one calling. Used by M8 `tab_borrow` to refuse
|
||||
@@ -203,7 +179,7 @@ export class SessionManager {
|
||||
const { signal: _signal, ...createOptions } = opts;
|
||||
windowId = await this.agentWindow.create(AGENT_WINDOW_HOME, createOptions);
|
||||
throwIfSessionStartAborted(opts.signal);
|
||||
const homeTabId = await this.agentWindow.ensureActiveTab(windowId, AGENT_WINDOW_HOME);
|
||||
await this.agentWindow.ensureActiveTab(windowId, AGENT_WINDOW_HOME);
|
||||
throwIfSessionStartAborted(opts.signal);
|
||||
|
||||
const ctx: SessionContext = {
|
||||
@@ -211,10 +187,6 @@ export class SessionManager {
|
||||
agentWindowId: windowId,
|
||||
refStore: new RefStore(),
|
||||
borrowedTabs: new Map(),
|
||||
// The home tab is the session's first explicit claim. Every other
|
||||
// tab remains free until `tab_create` or `tab_borrow` identifies it
|
||||
// by its concrete Chrome tab id.
|
||||
agentCreatedTabs: new Set([homeTabId]),
|
||||
createdAtMs: this.now(),
|
||||
};
|
||||
this.sessions.set(sessionId, ctx);
|
||||
|
||||
@@ -13,7 +13,7 @@ function fakeAgentWindow(ids: number[]) {
|
||||
return id;
|
||||
}),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -49,54 +49,13 @@ describe("ToolDispatcher", () => {
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
it.each([
|
||||
"tool.snapshot",
|
||||
"tool.evaluate",
|
||||
"tool.get_html",
|
||||
"tool.console",
|
||||
])("annotates Chrome extension-access failures from %s", async (method) => {
|
||||
const tab = { id: 7, windowId: 4242, active: true, url: "https://example.test" };
|
||||
vi.stubGlobal("chrome", {
|
||||
tabs: {
|
||||
get: vi.fn(async () => tab),
|
||||
query: vi.fn(async () => [tab]),
|
||||
},
|
||||
});
|
||||
const { transport, sent, deliver } = fakeTransport();
|
||||
const sessions = new SessionManager({
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 4242),
|
||||
remove: vi.fn(),
|
||||
ensureActiveTab: vi.fn(),
|
||||
},
|
||||
});
|
||||
await sessions.start("test");
|
||||
const error = new Error("Cannot access a chrome-extension:// URL of different extension");
|
||||
const cdp = {
|
||||
send: vi.fn().mockRejectedValue(error),
|
||||
ensureConsoleCapture: vi.fn().mockRejectedValue(error),
|
||||
consoleEntriesSince: vi.fn(),
|
||||
} as unknown as TestDispatcherCdp;
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions, cdp });
|
||||
dispatcher.start();
|
||||
deliver(makeRequest(method, { session_id: "test", expression: "document.title" }));
|
||||
await vi.waitFor(() => expect(sent).toHaveLength(1));
|
||||
expect(sent[0]).toMatchObject({
|
||||
error: {
|
||||
code: "cdp_failed",
|
||||
data: { reason: "cdp_extension_access_denied" },
|
||||
},
|
||||
});
|
||||
dispatcher.stop();
|
||||
});
|
||||
|
||||
it("uses the configured recording runtime for start and stop", async () => {
|
||||
const { transport, sent, deliver } = fakeTransport();
|
||||
const sessions = new SessionManager({
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 4242),
|
||||
remove: vi.fn(),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
},
|
||||
});
|
||||
await sessions.start("aa11");
|
||||
@@ -158,7 +117,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 4242),
|
||||
remove: vi.fn(),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
},
|
||||
});
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions });
|
||||
@@ -181,7 +140,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create,
|
||||
remove: vi.fn(),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
},
|
||||
});
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions });
|
||||
@@ -199,7 +158,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 4242),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
},
|
||||
});
|
||||
await sessions.start("aa11");
|
||||
@@ -213,47 +172,6 @@ describe("ToolDispatcher", () => {
|
||||
expect(sent[0]).toEqual({ id: "r-1", result: {} });
|
||||
});
|
||||
|
||||
it("wires the production tab APIs into tool.session_stop so a surviving user tab releases the window", async () => {
|
||||
// Regression guard for the deps that session_stop reads directly: when
|
||||
// `tabManagement.tabs` / `tabsQuery` are not injected by the dispatcher,
|
||||
// the agent-tab cleanup and the release decision silently no-op and the
|
||||
// Agent Window is closed even though a user tab is still open (issue #57).
|
||||
const removeTab = vi.fn(async () => {});
|
||||
const closeWindow = vi.fn(async () => {});
|
||||
// After cleanup the window still holds one tab (id 99) that is neither
|
||||
// the home tab nor an agent-created tab — i.e. a genuine user tab.
|
||||
vi.stubGlobal("chrome", {
|
||||
tabs: {
|
||||
remove: removeTab,
|
||||
query: vi.fn(async () => [{ id: 99, windowId: 4242, active: true }]),
|
||||
},
|
||||
});
|
||||
const { transport, sent, deliver } = fakeTransport();
|
||||
const sessions = new SessionManager({
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 4242),
|
||||
remove: closeWindow,
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
await sessions.start("aa11");
|
||||
// Register the exact tab id returned by the agent's creation path.
|
||||
sessions.get("aa11")?.agentCreatedTabs.add(7);
|
||||
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions });
|
||||
dispatcher.start();
|
||||
|
||||
deliver(makeRequest("tool.session_stop", { session_id: "aa11" }));
|
||||
await flushMicrotasks();
|
||||
|
||||
// The agent tab is closed through the real chrome.tabs surface...
|
||||
expect(removeTab).toHaveBeenCalledWith(7);
|
||||
// ...and the window is released rather than closed, so the user's tab survives.
|
||||
expect(closeWindow).not.toHaveBeenCalled();
|
||||
expect(sessions.has("aa11")).toBe(false);
|
||||
expect(sent[0]).toEqual({ id: "r-1", result: { window_released: true } });
|
||||
});
|
||||
|
||||
it("routes tool.console through the CDP console buffer", async () => {
|
||||
vi.stubGlobal("chrome", {
|
||||
tabs: {
|
||||
@@ -265,7 +183,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 4242),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
},
|
||||
});
|
||||
await sessions.start("aa11");
|
||||
@@ -328,7 +246,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 4242),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
},
|
||||
});
|
||||
const ctx = await sessions.start("aa11");
|
||||
@@ -421,7 +339,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 4242),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
},
|
||||
});
|
||||
await sessions.start("aa11");
|
||||
@@ -457,7 +375,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 4242),
|
||||
remove,
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
},
|
||||
});
|
||||
await sessions.start("aa11");
|
||||
@@ -505,7 +423,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 1),
|
||||
remove: vi.fn(),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
},
|
||||
});
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions });
|
||||
@@ -525,7 +443,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 1),
|
||||
remove: vi.fn(),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
},
|
||||
});
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions });
|
||||
@@ -549,7 +467,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 1),
|
||||
remove: vi.fn(),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
},
|
||||
});
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions });
|
||||
@@ -566,7 +484,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 1),
|
||||
remove: vi.fn(),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
},
|
||||
});
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions });
|
||||
@@ -583,7 +501,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 1),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
},
|
||||
});
|
||||
const onSessionsChanged = vi.fn();
|
||||
@@ -600,32 +518,16 @@ describe("ToolDispatcher", () => {
|
||||
});
|
||||
|
||||
it("invokes onBrowserControlResumed for browser-control tools but not passive reads", async () => {
|
||||
vi.stubGlobal("chrome", {
|
||||
tabs: {
|
||||
create: vi.fn(async (props: chrome.tabs.CreateProperties) => ({
|
||||
id: 7,
|
||||
windowId: props.windowId,
|
||||
url: props.url,
|
||||
active: props.active,
|
||||
})),
|
||||
},
|
||||
});
|
||||
const { transport, sent, deliver } = fakeTransport();
|
||||
const sessions = new SessionManager({
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 1),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
},
|
||||
});
|
||||
const onBrowserControlResumed = vi.fn();
|
||||
const onAgentTabClaimed = vi.fn();
|
||||
const dispatcher = new ToolDispatcher({
|
||||
transport,
|
||||
sessions,
|
||||
onBrowserControlResumed,
|
||||
onAgentTabClaimed,
|
||||
});
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions, onBrowserControlResumed });
|
||||
dispatcher.start();
|
||||
|
||||
deliver(makeRequest("tool.session_start", { session_id: "aa11" }));
|
||||
@@ -642,7 +544,6 @@ describe("ToolDispatcher", () => {
|
||||
});
|
||||
await flushMicrotasks();
|
||||
expect(onBrowserControlResumed).toHaveBeenCalledWith("aa11");
|
||||
expect(onAgentTabClaimed).toHaveBeenCalledWith(7, 1);
|
||||
});
|
||||
|
||||
it("reasserts remembered hover before follow-up work and releases only after actions", async () => {
|
||||
@@ -656,7 +557,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 1),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
},
|
||||
});
|
||||
const order: string[] = [];
|
||||
@@ -738,7 +639,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 1),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
},
|
||||
});
|
||||
const cdp = {
|
||||
@@ -788,135 +689,6 @@ describe("ToolDispatcher", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ tab_id: 7, code: "not_found" },
|
||||
{ tab_id: undefined, code: "invalid_params" },
|
||||
])("keeps hover when tab_return is rejected with $code", async ({ tab_id, code }) => {
|
||||
const sendMessage = vi.fn(async () => undefined);
|
||||
vi.stubGlobal("chrome", {
|
||||
tabs: {
|
||||
sendMessage,
|
||||
get: vi.fn(async () => ({ id: 7, windowId: 100, active: true })),
|
||||
query: vi.fn(async () => [{ id: 7, windowId: 100, active: true }]),
|
||||
},
|
||||
});
|
||||
const { transport, sent, deliver } = fakeTransport();
|
||||
const sessions = new SessionManager({
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 100),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
await sessions.start("aa11");
|
||||
const cdp = {
|
||||
send: vi.fn(async <T>() => ({}) as T),
|
||||
detachSession: vi.fn(async () => {}),
|
||||
releaseSessionTab: vi.fn(async () => {}),
|
||||
};
|
||||
const dispatcher = new ToolDispatcher({
|
||||
transport,
|
||||
sessions,
|
||||
cdp: cdp as unknown as TestDispatcherCdp,
|
||||
});
|
||||
const helpers = dispatcher as unknown as {
|
||||
rememberHover: (sessionId: string, result: unknown) => unknown;
|
||||
setHoverBypass: (sessionId: string, tabId: number, enabled: boolean) => Promise<void>;
|
||||
withHoverReassert: <T>(
|
||||
params: { session_id: string; tab_id?: number },
|
||||
work: () => Promise<T>,
|
||||
) => Promise<T>;
|
||||
};
|
||||
helpers.rememberHover("aa11", { tab_id: 7, x: 10, y: 20 });
|
||||
await helpers.setHoverBypass("aa11", 7, true);
|
||||
dispatcher.start();
|
||||
deliver(makeRequest("tool.tab_return", { session_id: "aa11", tab_id }));
|
||||
await vi.waitFor(() => expect(sent).toHaveLength(1));
|
||||
|
||||
expect(sent[0]).toMatchObject({ error: { code } });
|
||||
expect(cdp.releaseSessionTab).not.toHaveBeenCalled();
|
||||
expect(sendMessage).not.toHaveBeenCalledWith(7, expect.objectContaining({ enabled: false }));
|
||||
await helpers.withHoverReassert({ session_id: "aa11", tab_id: 7 }, async () => ({}));
|
||||
expect(cdp.send).toHaveBeenCalledExactlyOnceWith(7, "Input.dispatchMouseEvent", {
|
||||
type: "mouseMoved",
|
||||
x: 10,
|
||||
y: 20,
|
||||
});
|
||||
dispatcher.stop();
|
||||
});
|
||||
|
||||
it("releases the returned tab's hover before moving it and leaves other tabs alone", async () => {
|
||||
const sendMessage = vi.fn(async () => undefined);
|
||||
const move = vi.fn(async () => {
|
||||
expect(sendMessage).toHaveBeenCalledWith(7, expect.objectContaining({ enabled: false }));
|
||||
return { id: 7, windowId: 200, index: 4 };
|
||||
});
|
||||
vi.stubGlobal("chrome", {
|
||||
tabs: { sendMessage, move },
|
||||
windows: { get: vi.fn(async () => ({ id: 200 })) },
|
||||
});
|
||||
const { transport, sent, deliver } = fakeTransport();
|
||||
const sessions = new SessionManager({
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 100),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
const ctx = await sessions.start("aa11");
|
||||
ctx.borrowedTabs.set(7, { tabId: 7, originalWindowId: 200, originalIndex: 4 });
|
||||
const cdp = {
|
||||
send: vi.fn(async <T>() => ({}) as T),
|
||||
detachSession: vi.fn(async () => {}),
|
||||
releaseSessionTab: vi.fn(async () => {}),
|
||||
};
|
||||
const dispatcher = new ToolDispatcher({
|
||||
transport,
|
||||
sessions,
|
||||
cdp: cdp as unknown as TestDispatcherCdp,
|
||||
});
|
||||
const helpers = dispatcher as unknown as {
|
||||
rememberHover: (sessionId: string, result: unknown) => unknown;
|
||||
setHoverBypass: (sessionId: string, tabId: number, enabled: boolean) => Promise<void>;
|
||||
withHoverReassert: <T>(
|
||||
params: { session_id: string; tab_id?: number },
|
||||
work: () => Promise<T>,
|
||||
) => Promise<T>;
|
||||
};
|
||||
for (const [sessionId, tabId] of [
|
||||
["aa11", 7],
|
||||
["aa11", 8],
|
||||
["bb22", 9],
|
||||
] as const) {
|
||||
helpers.rememberHover(sessionId, { tab_id: tabId, x: 10, y: 20 });
|
||||
await helpers.setHoverBypass(sessionId, tabId, true);
|
||||
}
|
||||
dispatcher.start();
|
||||
deliver(makeRequest("tool.tab_return", { session_id: "aa11", tab_id: 7 }));
|
||||
await vi.waitFor(() => expect(sent).toHaveLength(1));
|
||||
|
||||
expect(sent[0]).toMatchObject({ result: { tab_id: 7, returned_to_window_id: 200 } });
|
||||
expect(cdp.releaseSessionTab).toHaveBeenCalledExactlyOnceWith("aa11", 7);
|
||||
await helpers.withHoverReassert({ session_id: "aa11", tab_id: 7 }, async () => ({}));
|
||||
expect(cdp.send).not.toHaveBeenCalled();
|
||||
for (const [sessionId, tabId] of [
|
||||
["aa11", 8],
|
||||
["bb22", 9],
|
||||
] as const) {
|
||||
expect(sendMessage).not.toHaveBeenCalledWith(
|
||||
tabId,
|
||||
expect.objectContaining({ enabled: false }),
|
||||
);
|
||||
await helpers.withHoverReassert({ session_id: sessionId, tab_id: tabId }, async () => ({}));
|
||||
expect(cdp.send).toHaveBeenLastCalledWith(tabId, "Input.dispatchMouseEvent", {
|
||||
type: "mouseMoved",
|
||||
x: 10,
|
||||
y: 20,
|
||||
});
|
||||
}
|
||||
dispatcher.stop();
|
||||
});
|
||||
|
||||
it("transfers hover overlay bypass ownership between sessions", async () => {
|
||||
vi.stubGlobal("chrome", {
|
||||
tabs: {
|
||||
@@ -928,7 +700,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 1),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
},
|
||||
});
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions });
|
||||
@@ -965,7 +737,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 4242),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
},
|
||||
});
|
||||
await sessions.start("aa11");
|
||||
@@ -1017,7 +789,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 1),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
},
|
||||
});
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions });
|
||||
@@ -1051,7 +823,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 4242),
|
||||
remove: vi.fn(),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
},
|
||||
});
|
||||
transport.send = () => {
|
||||
@@ -1073,7 +845,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 5555),
|
||||
remove,
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
},
|
||||
});
|
||||
transport.send = () => {
|
||||
@@ -1101,7 +873,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(() => createPromise),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
},
|
||||
});
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions });
|
||||
@@ -1152,7 +924,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 1),
|
||||
remove: vi.fn(),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
},
|
||||
});
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions });
|
||||
@@ -1173,7 +945,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(() => createPromise),
|
||||
remove: vi.fn(),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
},
|
||||
});
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions });
|
||||
|
||||
@@ -20,7 +20,7 @@ function fakeAgentWindow(ids: number[]) {
|
||||
return id;
|
||||
}),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -1,39 +0,0 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { RpcError } from "@/transport/types";
|
||||
import { cdpError, classifyCdpError } from "../errors";
|
||||
|
||||
const denied = "Cannot access a chrome-extension:// URL of different extension";
|
||||
|
||||
describe("CDP extension access errors", () => {
|
||||
it("classifies both Chrome Error instances and string failures", () => {
|
||||
for (const error of [new Error(denied), denied]) {
|
||||
expect(cdpError(error)).toEqual({
|
||||
code: "cdp_failed",
|
||||
message: denied,
|
||||
data: { reason: "cdp_extension_access_denied" },
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
it("preserves transfer outcome and cleanup instructions when CDP was denied", () => {
|
||||
const error: RpcError = {
|
||||
code: "cdp_failed",
|
||||
message: denied,
|
||||
data: {
|
||||
reason: "transfer_outcome_unknown",
|
||||
effect_state: "unknown",
|
||||
cleanup_state: "failed",
|
||||
},
|
||||
};
|
||||
expect(classifyCdpError(error)).toBe(error);
|
||||
});
|
||||
|
||||
it("leaves other error codes and unrelated debugger failures unchanged", () => {
|
||||
for (const error of [
|
||||
{ code: "permission_denied", message: denied },
|
||||
{ code: "cdp_failed", message: "Another debugger is already attached" },
|
||||
] satisfies RpcError[]) {
|
||||
expect(classifyCdpError(error)).toBe(error);
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -12,7 +12,7 @@ function fakeAgentWindow(ids: number[]) {
|
||||
return id;
|
||||
}),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -2,7 +2,6 @@ import { describe, expect, it, vi } from "vitest";
|
||||
import { SessionManager } from "@/session-manager/manager";
|
||||
import { type DownloadsApi, handleDownload } from "../download";
|
||||
import { captureBrowserDownload } from "../download-capture";
|
||||
import { uploadThroughFileDrop } from "../file-drop-transaction";
|
||||
import { uploadThroughActivatedFileInput } from "../file-input-transaction";
|
||||
import type { ResolvedActionTarget } from "../interaction";
|
||||
import type { CdpRunner } from "../shared";
|
||||
@@ -13,7 +12,7 @@ function sessions() {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 100),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -38,20 +37,16 @@ function fakeEvent<T extends (...args: never[]) => unknown>() {
|
||||
};
|
||||
}
|
||||
|
||||
function actionTarget(frameId?: string, sessionId?: string): ResolvedActionTarget {
|
||||
function actionTarget(frameId?: string): ResolvedActionTarget {
|
||||
return {
|
||||
tab: { tabId: 4, windowId: 100, active: true },
|
||||
backendNodeId: 123,
|
||||
cdpTarget: { tabId: 4, ...(sessionId ? { sessionId } : {}) },
|
||||
cdpTarget: { tabId: 4 },
|
||||
...(frameId ? { frameId } : {}),
|
||||
usedRef: "e3",
|
||||
};
|
||||
}
|
||||
|
||||
function withoutOverlay<T>(operation: () => Promise<T>): Promise<T> {
|
||||
return operation();
|
||||
}
|
||||
|
||||
function uploadCdp(
|
||||
options: {
|
||||
inputCount?: number;
|
||||
@@ -64,7 +59,6 @@ function uploadCdp(
|
||||
let cdpEvent: Parameters<NonNullable<CdpRunner["onEvent"]>>[0] | undefined;
|
||||
const send = vi.fn(async (_tabId: number, method: string, params?: object) => {
|
||||
calls.push({ method, params });
|
||||
if (method === "DOM.scrollIntoViewIfNeeded") return {};
|
||||
if (method === "Page.setInterceptFileChooserDialog") return {};
|
||||
if (method === "Page.getLayoutMetrics")
|
||||
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
|
||||
@@ -116,50 +110,6 @@ function uploadCdp(
|
||||
};
|
||||
}
|
||||
|
||||
function dropCdp(
|
||||
options: {
|
||||
targetContainsPoint?: boolean;
|
||||
dispatchError?: (type: string) => Error | Promise<never> | undefined;
|
||||
onCall?: (method: string) => void;
|
||||
} = {},
|
||||
) {
|
||||
const calls: Array<{ method: string; params?: object }> = [];
|
||||
const detach = vi.fn(async () => {});
|
||||
const send = vi.fn(async (_tabId: number, method: string, params?: object) => {
|
||||
calls.push({ method, params });
|
||||
options.onCall?.(method);
|
||||
if (method === "DOM.scrollIntoViewIfNeeded") return {};
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
|
||||
}
|
||||
if (method === "DOM.getContentQuads") {
|
||||
return { quads: [[10, 20, 210, 20, 210, 120, 10, 120]] };
|
||||
}
|
||||
if (method === "DOM.getNodeForLocation") return { backendNodeId: 124 };
|
||||
if (method === "DOM.resolveNode") {
|
||||
const id = (params as { backendNodeId?: number }).backendNodeId;
|
||||
return { object: { objectId: id === 123 ? "drop-target" : "hit-target" } };
|
||||
}
|
||||
if (method === "Runtime.callFunctionOn") {
|
||||
return { result: { value: options.targetContainsPoint ?? true } };
|
||||
}
|
||||
if (method === "Input.dispatchDragEvent") {
|
||||
const type = (params as { type?: string }).type ?? "";
|
||||
const error = options.dispatchError?.(type);
|
||||
if (error instanceof Promise) return error;
|
||||
if (error) throw error;
|
||||
return {};
|
||||
}
|
||||
if (method === "Runtime.releaseObjectGroup") return {};
|
||||
throw new Error(`unexpected drop CDP command ${method}`);
|
||||
});
|
||||
const cdp: CdpRunner = {
|
||||
send: send as unknown as CdpRunner["send"],
|
||||
detach,
|
||||
};
|
||||
return { calls, cdp, detach };
|
||||
}
|
||||
|
||||
describe("file transfer tools", () => {
|
||||
it("captures the file input activated by the requested click and injects only staged paths", async () => {
|
||||
const manager = sessions();
|
||||
@@ -296,6 +246,104 @@ describe("file transfer tools", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("does not dispatch browser work when upload is already cancelled", async () => {
|
||||
const controller = new AbortController();
|
||||
controller.abort();
|
||||
const { cdp, calls } = uploadCdp();
|
||||
|
||||
const result = await uploadThroughActivatedFileInput({
|
||||
cdp,
|
||||
actionTarget: actionTarget(),
|
||||
files: ["/private/stage/one"],
|
||||
timeoutMs: 100,
|
||||
signal: controller.signal,
|
||||
trigger: vi.fn(),
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({
|
||||
code: "cancelled",
|
||||
data: { effect_state: "none", phase: "arm_interception" },
|
||||
});
|
||||
expect(calls).not.toContainEqual({
|
||||
method: "Page.setInterceptFileChooserDialog",
|
||||
params: { enabled: true },
|
||||
});
|
||||
expect(calls.some((call) => call.method === "DOM.setFileInputFiles")).toBe(false);
|
||||
});
|
||||
|
||||
it("stops before file assignment when cancellation interrupts input resolution", async () => {
|
||||
const controller = new AbortController();
|
||||
const fixture = uploadCdp();
|
||||
const originalSend = fixture.cdp.send;
|
||||
let markProbeStarted!: () => void;
|
||||
const probeStarted = new Promise<void>((resolve) => {
|
||||
markProbeStarted = resolve;
|
||||
});
|
||||
fixture.cdp.send = vi.fn((tabId: number, method: string, params?: object) => {
|
||||
const declaration = (params as { functionDeclaration?: string } | undefined)
|
||||
?.functionDeclaration;
|
||||
if (
|
||||
method === "Runtime.callFunctionOn" &&
|
||||
declaration?.includes("count: state.inputs.length")
|
||||
) {
|
||||
markProbeStarted();
|
||||
return new Promise<never>(() => {});
|
||||
}
|
||||
return originalSend(tabId, method, params);
|
||||
}) as CdpRunner["send"];
|
||||
|
||||
const transaction = uploadThroughActivatedFileInput({
|
||||
cdp: fixture.cdp,
|
||||
actionTarget: actionTarget(),
|
||||
files: ["/private/stage/one"],
|
||||
timeoutMs: 1_000,
|
||||
signal: controller.signal,
|
||||
trigger: async () => ({ tab_id: 4, x: 10, y: 10 }),
|
||||
});
|
||||
await probeStarted;
|
||||
controller.abort();
|
||||
const result = await transaction;
|
||||
|
||||
expect(result).toMatchObject({
|
||||
code: "cancelled",
|
||||
data: { effect_state: "none", phase: "resolve_input" },
|
||||
});
|
||||
expect(fixture.calls.some((call) => call.method === "DOM.setFileInputFiles")).toBe(false);
|
||||
});
|
||||
|
||||
it("disarms chooser interception when its enable response times out", async () => {
|
||||
const calls: Array<{ method: string; params?: object }> = [];
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async (_tabId: number, method: string, params?: object) => {
|
||||
calls.push({ method, params });
|
||||
if (
|
||||
method === "Page.setInterceptFileChooserDialog" &&
|
||||
(params as { enabled?: boolean }).enabled
|
||||
) {
|
||||
return new Promise<never>(() => {});
|
||||
}
|
||||
return {};
|
||||
}) as CdpRunner["send"],
|
||||
};
|
||||
|
||||
const result = await uploadThroughActivatedFileInput({
|
||||
cdp,
|
||||
actionTarget: actionTarget(),
|
||||
files: ["/private/stage/one"],
|
||||
timeoutMs: 5,
|
||||
trigger: vi.fn(),
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({
|
||||
code: "timeout",
|
||||
data: { effect_state: "none", phase: "arm_interception" },
|
||||
});
|
||||
expect(calls).toContainEqual({
|
||||
method: "Page.setInterceptFileChooserDialog",
|
||||
params: { enabled: false, cancel: true },
|
||||
});
|
||||
});
|
||||
|
||||
it("marks a timed-out file assignment unknown and detaches browser state", async () => {
|
||||
const fixture = uploadCdp();
|
||||
const originalSend = fixture.cdp.send;
|
||||
@@ -321,236 +369,6 @@ describe("file transfer tools", () => {
|
||||
expect(detach).toHaveBeenCalledWith(4);
|
||||
});
|
||||
|
||||
it("delivers staged files through one explicit native drop without clicking", async () => {
|
||||
const manager = sessions();
|
||||
const ctx = await manager.start("s1");
|
||||
ctx.refStore.set("e3", 123, { tabId: 4 });
|
||||
const lifecycle: string[] = [];
|
||||
const { cdp, calls } = dropCdp({ onCall: (method) => lifecycle.push(method) });
|
||||
const overlayPhases: string[] = [];
|
||||
|
||||
const result = await handleUpload(
|
||||
manager,
|
||||
{
|
||||
session_id: "s1",
|
||||
ref: "@e3",
|
||||
mode: "drop",
|
||||
files: [{ transfer_id: "tr_1", name: "one.png", staged_path: "/stage/one.png" }],
|
||||
},
|
||||
{
|
||||
cdp,
|
||||
tabsApi: tabsApi(),
|
||||
sendToTab: vi.fn(async (_tabId, message) => {
|
||||
overlayPhases.push(message.phase);
|
||||
lifecycle.push(`overlay:${message.phase}`);
|
||||
}),
|
||||
},
|
||||
);
|
||||
|
||||
expect(result).toMatchObject({ tab_id: 4, used_ref: "e3", file_names: ["one.png"] });
|
||||
expect(calls.filter((call) => call.method === "Input.dispatchDragEvent")).toEqual([
|
||||
{
|
||||
method: "Input.dispatchDragEvent",
|
||||
params: {
|
||||
type: "dragEnter",
|
||||
x: 110,
|
||||
y: 70,
|
||||
data: { items: [], files: ["/stage/one.png"], dragOperationsMask: 1 },
|
||||
},
|
||||
},
|
||||
{
|
||||
method: "Input.dispatchDragEvent",
|
||||
params: {
|
||||
type: "dragOver",
|
||||
x: 110,
|
||||
y: 70,
|
||||
data: { items: [], files: ["/stage/one.png"], dragOperationsMask: 1 },
|
||||
},
|
||||
},
|
||||
{
|
||||
method: "Input.dispatchDragEvent",
|
||||
params: {
|
||||
type: "drop",
|
||||
x: 110,
|
||||
y: 70,
|
||||
data: { items: [], files: ["/stage/one.png"], dragOperationsMask: 1 },
|
||||
},
|
||||
},
|
||||
]);
|
||||
expect(calls.some((call) => call.method === "Input.dispatchMouseEvent")).toBe(false);
|
||||
expect(overlayPhases).toEqual(["begin", "end"]);
|
||||
expect(lifecycle.indexOf("DOM.getContentQuads")).toBeLessThan(
|
||||
lifecycle.indexOf("overlay:begin"),
|
||||
);
|
||||
expect(lifecycle.indexOf("overlay:begin")).toBeLessThan(
|
||||
lifecycle.indexOf("DOM.getNodeForLocation"),
|
||||
);
|
||||
expect(lifecycle.lastIndexOf("Input.dispatchDragEvent")).toBeLessThan(
|
||||
lifecycle.indexOf("overlay:end"),
|
||||
);
|
||||
expect(lifecycle.indexOf("overlay:end")).toBeLessThan(
|
||||
lifecycle.indexOf("Runtime.releaseObjectGroup"),
|
||||
);
|
||||
});
|
||||
|
||||
it("fails closed before dragging when another element owns the action point", async () => {
|
||||
const { cdp, calls } = dropCdp({ targetContainsPoint: false });
|
||||
const result = await uploadThroughFileDrop({
|
||||
cdp,
|
||||
actionTarget: actionTarget(),
|
||||
files: ["/stage/one.png"],
|
||||
timeoutMs: 100,
|
||||
withOverlayHidden: withoutOverlay,
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({
|
||||
code: "permission_denied",
|
||||
data: {
|
||||
reason: "file_drop_target_unavailable",
|
||||
mechanism: "drop",
|
||||
effect_state: "none",
|
||||
phase: "resolve_target",
|
||||
},
|
||||
});
|
||||
expect(calls.some((call) => call.method === "Input.dispatchDragEvent")).toBe(false);
|
||||
});
|
||||
|
||||
it("dispatches an OOPIF file drop in the child target's local coordinates", async () => {
|
||||
const rootCalls: Array<{ method: string; params?: object }> = [];
|
||||
const childCalls: Array<{ method: string; params?: object }> = [];
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async (_tabId, method, params) => {
|
||||
rootCalls.push({ method, params });
|
||||
if (method === "DOM.scrollIntoViewIfNeeded") return {};
|
||||
if (method === "DOM.getBoxModel") {
|
||||
return { model: { content: [100, 100, 300, 100, 300, 200, 100, 200] } };
|
||||
}
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 800, clientHeight: 600 } };
|
||||
}
|
||||
throw new Error(`unexpected root command ${method}`);
|
||||
}) as CdpRunner["send"],
|
||||
sendToTarget: vi.fn(async (_target, method, params) => {
|
||||
childCalls.push({ method, params });
|
||||
if (method === "DOM.scrollIntoViewIfNeeded") return {};
|
||||
if (method === "DOM.getContentQuads") {
|
||||
return { quads: [[10, 20, 210, 20, 210, 120, 10, 120]] };
|
||||
}
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 400, clientHeight: 300 } };
|
||||
}
|
||||
if (method === "DOM.getNodeForLocation") {
|
||||
return { backendNodeId: 124, frameId: "child" };
|
||||
}
|
||||
if (method === "DOM.resolveNode") {
|
||||
const id = (params as { backendNodeId?: number }).backendNodeId;
|
||||
return { object: { objectId: id === 123 ? "drop-target" : "hit-target" } };
|
||||
}
|
||||
if (method === "Runtime.callFunctionOn") return { result: { value: true } };
|
||||
if (method === "Input.dispatchDragEvent" || method === "Runtime.releaseObjectGroup") {
|
||||
return {};
|
||||
}
|
||||
throw new Error(`unexpected child command ${method}`);
|
||||
}) as CdpRunner["sendToTarget"],
|
||||
getFrameGraph: vi.fn(async () => ({
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "child",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 10,
|
||||
target: { tabId: 4, sessionId: "child-session" },
|
||||
},
|
||||
],
|
||||
})),
|
||||
};
|
||||
|
||||
const result = await uploadThroughFileDrop({
|
||||
cdp,
|
||||
actionTarget: actionTarget("child", "child-session"),
|
||||
files: ["/stage/one.png"],
|
||||
timeoutMs: 100,
|
||||
withOverlayHidden: withoutOverlay,
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({ x: 110, y: 70 });
|
||||
expect(
|
||||
childCalls
|
||||
.filter((call) => call.method === "Input.dispatchDragEvent")
|
||||
.map((call) => call.params),
|
||||
).toEqual([
|
||||
expect.objectContaining({ type: "dragEnter", x: 110, y: 70 }),
|
||||
expect.objectContaining({ type: "dragOver", x: 110, y: 70 }),
|
||||
expect.objectContaining({ type: "drop", x: 110, y: 70 }),
|
||||
]);
|
||||
expect(rootCalls.some((call) => call.method === "Input.dispatchDragEvent")).toBe(false);
|
||||
});
|
||||
|
||||
it("reports a missing native drag command as unsupported before commit", async () => {
|
||||
const { cdp } = dropCdp({
|
||||
dispatchError: (type) =>
|
||||
type === "dragEnter" ? new Error("'Input.dispatchDragEvent' wasn't found") : undefined,
|
||||
});
|
||||
const result = await uploadThroughFileDrop({
|
||||
cdp,
|
||||
actionTarget: actionTarget(),
|
||||
files: ["/stage/one.png"],
|
||||
timeoutMs: 100,
|
||||
withOverlayHidden: withoutOverlay,
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({
|
||||
code: "unsupported",
|
||||
data: {
|
||||
reason: "upload_mechanism_unsupported",
|
||||
effect_state: "none",
|
||||
phase: "drag_enter",
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("cancels an unresolved drag before returning a pre-commit timeout", async () => {
|
||||
const { cdp, calls } = dropCdp({
|
||||
dispatchError: (type) => (type === "dragEnter" ? new Promise<never>(() => {}) : undefined),
|
||||
});
|
||||
const result = await uploadThroughFileDrop({
|
||||
cdp,
|
||||
actionTarget: actionTarget(),
|
||||
files: ["/stage/one.png"],
|
||||
timeoutMs: 10,
|
||||
withOverlayHidden: withoutOverlay,
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({
|
||||
code: "timeout",
|
||||
data: { reason: "file_drop_failed", effect_state: "none", phase: "drag_enter" },
|
||||
});
|
||||
expect(calls).toContainEqual({
|
||||
method: "Input.dispatchDragEvent",
|
||||
params: expect.objectContaining({ type: "dragCancel" }),
|
||||
});
|
||||
});
|
||||
|
||||
it("marks an unresolved drop unknown and detaches browser state", async () => {
|
||||
const { cdp, detach } = dropCdp({
|
||||
dispatchError: (type) => (type === "drop" ? new Promise<never>(() => {}) : undefined),
|
||||
});
|
||||
const result = await uploadThroughFileDrop({
|
||||
cdp,
|
||||
actionTarget: actionTarget(),
|
||||
files: ["/stage/one.png"],
|
||||
timeoutMs: 10,
|
||||
withOverlayHidden: withoutOverlay,
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({
|
||||
code: "timeout",
|
||||
data: { reason: "file_drop_failed", effect_state: "unknown", phase: "drop" },
|
||||
});
|
||||
expect(detach).toHaveBeenCalledWith(4);
|
||||
});
|
||||
|
||||
it("routes one exact-target download through a browser-relative capability", async () => {
|
||||
const manager = sessions();
|
||||
const ctx = await manager.start("s1");
|
||||
@@ -847,7 +665,10 @@ describe("file transfer tools", () => {
|
||||
expect(downloads.cancel).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("reconciles a download that completes while cancellation is being requested", async () => {
|
||||
it.each([
|
||||
["cancel resolves", false],
|
||||
["cancel reports that completion won", true],
|
||||
])("reconciles a download that completes while %s", async (_label, cancelThrows) => {
|
||||
const onCreated = fakeEvent<(item: chrome.downloads.DownloadItem) => void>();
|
||||
const onChanged = fakeEvent<(delta: chrome.downloads.DownloadDelta) => void>();
|
||||
const onDeterminingFilename =
|
||||
@@ -881,7 +702,7 @@ describe("file transfer tools", () => {
|
||||
onDeterminingFilename,
|
||||
search: vi.fn().mockResolvedValueOnce([initial]).mockResolvedValueOnce([complete]),
|
||||
cancel: vi.fn(async () => {
|
||||
throw new Error("download already complete");
|
||||
if (cancelThrows) throw new Error("download already complete");
|
||||
}),
|
||||
removeFile: vi.fn(async () => {}),
|
||||
};
|
||||
|
||||
@@ -1,573 +0,0 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { SessionManager } from "@/session-manager/manager";
|
||||
import type { CdpRunner } from "@/tools/shared";
|
||||
import { handleFill } from "../interaction";
|
||||
|
||||
interface ScriptParams {
|
||||
functionDeclaration: string;
|
||||
arguments?: Array<{ value: unknown }>;
|
||||
}
|
||||
|
||||
async function setup(markup = '<input value="old">') {
|
||||
document.body.innerHTML = markup;
|
||||
const element = document.body.firstElementChild as HTMLInputElement;
|
||||
const manager = new SessionManager({
|
||||
agentWindow: {
|
||||
create: async () => 100,
|
||||
remove: async () => {},
|
||||
ensureActiveTab: async () => 4,
|
||||
},
|
||||
});
|
||||
const ctx = await manager.start("aa11");
|
||||
ctx.refStore.set("e1", 12, { tabId: 4 });
|
||||
const script = vi.fn(async (params: ScriptParams): Promise<unknown> => {
|
||||
try {
|
||||
const fn = new Function(`return (${params.functionDeclaration})`)();
|
||||
return {
|
||||
result: {
|
||||
value: await fn.apply(
|
||||
element,
|
||||
(params.arguments ?? []).map((arg) => arg.value),
|
||||
),
|
||||
},
|
||||
};
|
||||
} catch {
|
||||
return { exceptionDetails: { text: "test page script failed" } };
|
||||
}
|
||||
});
|
||||
// Number/email controls have a browser caret even though their selection
|
||||
// getters return null. Model that caret instead of assuming it is at the end.
|
||||
const hiddenCaret = { start: 0, end: 0 };
|
||||
const moveToEnd = vi.fn(async () => {
|
||||
if (typeof element.selectionStart === "number") {
|
||||
element.setSelectionRange(element.value.length, element.value.length);
|
||||
} else {
|
||||
hiddenCaret.start = hiddenCaret.end = element.value.length;
|
||||
}
|
||||
});
|
||||
const insert = vi.fn(async (text: string) => {
|
||||
const focused = document.activeElement as HTMLInputElement;
|
||||
if (focused.readOnly || focused.disabled) return;
|
||||
const start = focused.selectionStart ?? hiddenCaret.start;
|
||||
const end = focused.selectionEnd ?? hiddenCaret.end;
|
||||
const value = focused.value.slice(0, start) + text + focused.value.slice(end);
|
||||
focused.value = focused.maxLength >= 0 ? value.slice(0, focused.maxLength) : value;
|
||||
hiddenCaret.start = hiddenCaret.end = Math.min(start + text.length, focused.value.length);
|
||||
focused.dispatchEvent(new Event("input", { bubbles: true }));
|
||||
});
|
||||
const release = vi.fn(async () => ({}));
|
||||
const send = vi.fn(async (_tabId: number, method: string, params?: object) => {
|
||||
switch (method) {
|
||||
case "DOM.describeNode":
|
||||
return {
|
||||
node: {
|
||||
nodeName: element.tagName,
|
||||
attributes: Array.from(element.attributes).flatMap((attr) => [attr.name, attr.value]),
|
||||
},
|
||||
};
|
||||
case "DOM.scrollIntoViewIfNeeded":
|
||||
return {};
|
||||
case "DOM.focus":
|
||||
element.focus();
|
||||
return {};
|
||||
case "DOM.resolveNode":
|
||||
return { object: { objectId: "fill-target" } };
|
||||
case "Runtime.callFunctionOn":
|
||||
return script(params as ScriptParams);
|
||||
case "Input.insertText":
|
||||
await insert((params as { text: string }).text);
|
||||
return {};
|
||||
case "Input.dispatchKeyEvent":
|
||||
if ((params as { commands?: string[] }).commands?.includes("moveToEndOfDocument")) {
|
||||
await moveToEnd();
|
||||
}
|
||||
return {};
|
||||
case "Runtime.releaseObject":
|
||||
return release();
|
||||
default:
|
||||
throw new Error(`unexpected ${method}`);
|
||||
}
|
||||
});
|
||||
return {
|
||||
element,
|
||||
hiddenCaret,
|
||||
moveToEnd,
|
||||
script,
|
||||
insert,
|
||||
release,
|
||||
send,
|
||||
fill: (value = "hello", clearBefore = true, signal?: AbortSignal) =>
|
||||
handleFill(
|
||||
manager,
|
||||
{ session_id: "aa11", ref: "e1", value, clear_before: clearBefore },
|
||||
{
|
||||
cdp: { send: send as CdpRunner["send"] },
|
||||
tabsApi: {
|
||||
get: async (id) => ({ id, windowId: 100, active: true }) as chrome.tabs.Tab,
|
||||
query: async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab],
|
||||
},
|
||||
signal,
|
||||
},
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
document.body.innerHTML = "";
|
||||
});
|
||||
|
||||
describe("fill result verification", () => {
|
||||
it.each([
|
||||
"readonly",
|
||||
"disabled",
|
||||
])("preserves a %s field before rejecting it", async (attribute) => {
|
||||
const h = await setup(`<input ${attribute} value="old">`);
|
||||
expect(await h.fill()).toMatchObject({
|
||||
code: "invalid_params",
|
||||
data: { reason: "target_not_fillable" },
|
||||
});
|
||||
expect(h.element.value).toBe("old");
|
||||
expect(h.insert).not.toHaveBeenCalled();
|
||||
expect(h.release).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it.each([
|
||||
"checkbox",
|
||||
"radio",
|
||||
"range",
|
||||
"date",
|
||||
"file",
|
||||
])("rejects an unsupported %s input before clearing", async (type) => {
|
||||
const h = await setup(`<input type="${type}">`);
|
||||
const before = h.element.value;
|
||||
expect(await h.fill()).toMatchObject({
|
||||
code: "invalid_params",
|
||||
data: { reason: "target_not_fillable" },
|
||||
});
|
||||
expect(h.element.value).toBe(before);
|
||||
expect(h.insert).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("rejects an overlong value before clearing", async () => {
|
||||
const h = await setup('<input maxlength="3" value="old">');
|
||||
expect(await h.fill()).toMatchObject({
|
||||
code: "invalid_params",
|
||||
message: /maxlength/,
|
||||
data: { reason: "fill_value_invalid" },
|
||||
});
|
||||
expect(h.element.value).toBe("old");
|
||||
expect(h.insert).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each([
|
||||
true,
|
||||
false,
|
||||
])("classifies a DOM.focus rejection on a disabled=%s target", async (disabled) => {
|
||||
const h = await setup(`<input ${disabled ? "disabled" : ""} value="old">`);
|
||||
const send = h.send.getMockImplementation()!;
|
||||
h.send.mockImplementation(async (tabId, method, params) => {
|
||||
if (method === "DOM.focus") throw new Error("Element is not focusable");
|
||||
return send(tabId, method, params);
|
||||
});
|
||||
expect(await h.fill()).toMatchObject({
|
||||
code: disabled ? "invalid_params" : "cdp_failed",
|
||||
data: { reason: disabled ? "target_not_fillable" : "fill_failed" },
|
||||
});
|
||||
expect(h.element.value).toBe("old");
|
||||
expect(h.insert).not.toHaveBeenCalled();
|
||||
expect(h.release).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("fills a background document without requiring foreground browser focus", async () => {
|
||||
const h = await setup();
|
||||
vi.spyOn(document, "hasFocus").mockReturnValue(false);
|
||||
expect(await h.fill()).toMatchObject({ value_length: 5 });
|
||||
expect(h.element.value).toBe("hello");
|
||||
});
|
||||
|
||||
it("restores focus once after clearing without typing into another field", async () => {
|
||||
const h = await setup();
|
||||
const other = document.createElement("input");
|
||||
document.body.append(other);
|
||||
h.element.addEventListener("input", () => other.focus());
|
||||
expect(await h.fill()).toMatchObject({ value_length: 5 });
|
||||
expect(h.element.value).toBe("hello");
|
||||
expect(other.value).toBe("");
|
||||
expect(h.send.mock.calls.filter(([, method]) => method === "DOM.focus")).toHaveLength(2);
|
||||
});
|
||||
|
||||
it.each([
|
||||
"focus",
|
||||
"value",
|
||||
])("rechecks %s after restoring focus, without retrying again", async (action) => {
|
||||
const h = await setup();
|
||||
const other = document.createElement("input");
|
||||
document.body.append(other);
|
||||
h.element.addEventListener(
|
||||
"input",
|
||||
() => {
|
||||
other.focus();
|
||||
h.element.addEventListener("focus", () => {
|
||||
if (action === "focus") queueMicrotask(() => other.focus());
|
||||
else h.element.value = "changed";
|
||||
});
|
||||
},
|
||||
{ once: true },
|
||||
);
|
||||
expect(await h.fill()).toMatchObject({
|
||||
code: "cdp_failed",
|
||||
data: { reason: action === "focus" ? "fill_focus_lost" : "fill_target_changed" },
|
||||
});
|
||||
expect(other.value).toBe("");
|
||||
expect(h.insert).not.toHaveBeenCalled();
|
||||
expect(h.send.mock.calls.filter(([, method]) => method === "DOM.focus")).toHaveLength(2);
|
||||
});
|
||||
|
||||
it.each([
|
||||
"remove",
|
||||
"readonly",
|
||||
"restore",
|
||||
"microtask-restore",
|
||||
])("stops when clearing causes the page to %s the target", async (action) => {
|
||||
const h = await setup();
|
||||
h.element.addEventListener("input", () => {
|
||||
if (action === "remove") h.element.remove();
|
||||
if (action === "readonly") h.element.readOnly = true;
|
||||
if (action === "restore") h.element.value = "restored";
|
||||
if (action === "microtask-restore")
|
||||
queueMicrotask(() => {
|
||||
h.element.value = "restored";
|
||||
});
|
||||
});
|
||||
expect(await h.fill()).toMatchObject({
|
||||
code: "cdp_failed",
|
||||
data: { reason: "fill_target_changed" },
|
||||
});
|
||||
expect(h.insert).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("fails when insertText is acknowledged without changing the value", async () => {
|
||||
const h = await setup();
|
||||
h.insert.mockImplementation(async () => {});
|
||||
expect(await h.fill()).toMatchObject({
|
||||
code: "cdp_failed",
|
||||
data: { reason: "fill_value_mismatch" },
|
||||
});
|
||||
expect(h.release).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("detects truncation when maxlength changes after preparation", async () => {
|
||||
const h = await setup();
|
||||
h.element.addEventListener(
|
||||
"input",
|
||||
() => {
|
||||
h.element.maxLength = 3;
|
||||
},
|
||||
{ once: true },
|
||||
);
|
||||
expect(await h.fill()).toMatchObject({
|
||||
code: "cdp_failed",
|
||||
data: { reason: "fill_value_mismatch" },
|
||||
});
|
||||
expect(h.element.value).toBe("hel");
|
||||
});
|
||||
|
||||
it.each([
|
||||
"input",
|
||||
"change",
|
||||
"microtask-change",
|
||||
])("detects a value reverted by the %s handler", async (event) => {
|
||||
const h = await setup();
|
||||
h.element.addEventListener(event === "microtask-change" ? "change" : event, () => {
|
||||
if (event === "microtask-change")
|
||||
queueMicrotask(() => {
|
||||
h.element.value = "";
|
||||
});
|
||||
else h.element.value = "";
|
||||
});
|
||||
expect(await h.fill()).toMatchObject({ code: "cdp_failed", message: /expected value/ });
|
||||
});
|
||||
|
||||
it("compares content rather than length without including field values in the error", async () => {
|
||||
const h = await setup('<input type="password">');
|
||||
h.element.addEventListener("change", () => {
|
||||
h.element.value = "other-secret";
|
||||
});
|
||||
const result = await h.fill("given-secret");
|
||||
expect(result).toMatchObject({ code: "cdp_failed", message: /expected value/ });
|
||||
expect(JSON.stringify(result)).not.toMatch(/given-secret|other-secret/);
|
||||
});
|
||||
|
||||
it("does not report success for a target replaced by a change handler", async () => {
|
||||
const h = await setup();
|
||||
h.element.addEventListener("change", () => h.element.replaceWith(h.element.cloneNode()));
|
||||
expect(await h.fill()).toMatchObject({
|
||||
code: "cdp_failed",
|
||||
data: { reason: "fill_target_changed" },
|
||||
});
|
||||
});
|
||||
|
||||
it.each([
|
||||
1, 2, 3, 4, 5,
|
||||
])("rejects script exceptions at phase %s and releases the target", async (phase) => {
|
||||
const h = await setup();
|
||||
vi.spyOn(document, "hasFocus").mockReturnValue(false);
|
||||
const run = h.script.getMockImplementation()!;
|
||||
let count = 0;
|
||||
h.script.mockImplementation(async (params) =>
|
||||
++count === phase ? { exceptionDetails: { text: "page secret" } } : run(params),
|
||||
);
|
||||
const result = await h.fill();
|
||||
expect(result).toMatchObject({ code: "cdp_failed", data: { reason: "fill_failed" } });
|
||||
expect(JSON.stringify(result)).not.toContain("page secret");
|
||||
if (phase <= 3) expect(h.insert).not.toHaveBeenCalled();
|
||||
expect(h.release).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it.each([1, 2, 3, 5])("rejects missing results at phase %s", async (phase) => {
|
||||
const h = await setup();
|
||||
vi.spyOn(document, "hasFocus").mockReturnValue(false);
|
||||
const run = h.script.getMockImplementation()!;
|
||||
let count = 0;
|
||||
h.script.mockImplementation(async (params) =>
|
||||
++count === phase ? { result: { type: "undefined" } } : run(params),
|
||||
);
|
||||
expect(await h.fill()).toMatchObject({ code: "cdp_failed" });
|
||||
if (phase <= 3) expect(h.insert).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each(["input", "change"])("checks state after nested microtasks from %s", async (event) => {
|
||||
const h = await setup();
|
||||
h.element.addEventListener(event, () => {
|
||||
queueMicrotask(() =>
|
||||
queueMicrotask(() => {
|
||||
h.element.value = "reverted";
|
||||
}),
|
||||
);
|
||||
});
|
||||
expect(await h.fill()).toMatchObject({ code: "cdp_failed" });
|
||||
if (event === "input") expect(h.insert).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each([
|
||||
"<input>",
|
||||
'<input type="password">',
|
||||
"<textarea></textarea>",
|
||||
])("verifies successful text in %s", async (markup) => {
|
||||
const h = await setup(markup);
|
||||
expect(await h.fill("你好🙂")).toMatchObject({ value_length: 4 });
|
||||
expect(h.element.value).toBe("你好🙂");
|
||||
expect(h.release).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("verifies append against the old value plus the requested text", async () => {
|
||||
const h = await setup();
|
||||
h.element.focus();
|
||||
h.element.setSelectionRange(3, 3);
|
||||
expect(await h.fill("!", false)).toMatchObject({ value_length: 4 });
|
||||
expect(h.element.value).toBe("old!");
|
||||
});
|
||||
|
||||
it.each([
|
||||
["<textarea></textarea>", "a\r\nb", "a\nb"],
|
||||
["<input>", "a\nb", "a b"],
|
||||
["<input>", "a\r\nb", "a b"],
|
||||
[
|
||||
'<input type="email" multiple>',
|
||||
" a@example.com , b@example.com ",
|
||||
"a@example.com,b@example.com",
|
||||
],
|
||||
])("accepts native normalization for %s", async (markup, requested, normalized) => {
|
||||
if (markup.startsWith("<textarea")) {
|
||||
// Happy DOM does not implement Chrome's textarea line-ending normalization.
|
||||
const setter = Object.getOwnPropertyDescriptor(HTMLTextAreaElement.prototype, "value")!.set!;
|
||||
vi.spyOn(HTMLTextAreaElement.prototype, "value", "set").mockImplementation(function (
|
||||
this: HTMLTextAreaElement,
|
||||
value,
|
||||
) {
|
||||
setter.call(this, value.replace(/\r\n?/g, "\n"));
|
||||
});
|
||||
}
|
||||
const h = await setup(markup);
|
||||
h.insert.mockImplementation(async () => {
|
||||
h.element.value = normalized;
|
||||
});
|
||||
expect(await h.fill(requested)).toMatchObject({ value_length: normalized.length });
|
||||
});
|
||||
|
||||
it("does not count an editable padding break as an extra typed character", async () => {
|
||||
const h = await setup('<div contenteditable="true" tabindex="0"></div>');
|
||||
// These two layout/editability properties are missing in Happy DOM.
|
||||
let innerText = "";
|
||||
Object.defineProperty(h.element, "isContentEditable", { get: () => true });
|
||||
Object.defineProperty(h.element, "innerText", { get: () => innerText });
|
||||
h.insert.mockImplementation(async () => {
|
||||
h.element.innerHTML = "one<div><br></div>";
|
||||
innerText = "one\n\n";
|
||||
});
|
||||
expect(await h.fill("one\n")).toMatchObject({ value_length: 4 });
|
||||
});
|
||||
|
||||
it.each([
|
||||
[0, 0],
|
||||
[1, 1],
|
||||
[0, 3],
|
||||
])("appends after moving selection %s:%s to the end", async (start, end) => {
|
||||
const h = await setup();
|
||||
h.element.focus();
|
||||
h.element.setSelectionRange(start, end);
|
||||
expect(await h.fill("🙂", false)).toMatchObject({ value_length: 5 });
|
||||
expect(h.element.value).toBe("old🙂");
|
||||
});
|
||||
|
||||
it.each(
|
||||
["number", "email"].flatMap((type) =>
|
||||
[
|
||||
[0, 0],
|
||||
[1, 1],
|
||||
[0, 2],
|
||||
].map(([start, end]) => ({ type, start, end })),
|
||||
),
|
||||
)("appends to $type from selection $start:$end without using its unsupported API", async ({
|
||||
type,
|
||||
start,
|
||||
end,
|
||||
}) => {
|
||||
const h = await setup(`<input type="${type}" value="12">`);
|
||||
Object.assign(h.hiddenCaret, { start, end });
|
||||
vi.spyOn(h.element, "selectionStart", "get").mockReturnValue(null);
|
||||
vi.spyOn(h.element, "selectionEnd", "get").mockReturnValue(null);
|
||||
const selection = vi.spyOn(h.element, "setSelectionRange").mockImplementation(() => {
|
||||
throw new DOMException("unsupported", "InvalidStateError");
|
||||
});
|
||||
expect(await h.fill("3", false)).toMatchObject({ value_length: 3 });
|
||||
expect(h.element.value).toBe("123");
|
||||
expect(selection).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each([
|
||||
"number",
|
||||
"email",
|
||||
])("does not report successful %s append when the caret command is ignored", async (type) => {
|
||||
const h = await setup(`<input type="${type}" value="12">`);
|
||||
vi.spyOn(h.element, "selectionStart", "get").mockReturnValue(null);
|
||||
vi.spyOn(h.element, "selectionEnd", "get").mockReturnValue(null);
|
||||
h.moveToEnd.mockResolvedValueOnce(undefined);
|
||||
expect(await h.fill("3", false)).toMatchObject({
|
||||
code: "cdp_failed",
|
||||
data: { reason: "fill_value_mismatch" },
|
||||
});
|
||||
expect(h.element.value).toBe("312");
|
||||
expect(h.insert).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("rejects invalid number text before clearing", async () => {
|
||||
const h = await setup('<input type="number" value="12">');
|
||||
expect(await h.fill("abc")).toMatchObject({
|
||||
code: "invalid_params",
|
||||
data: { reason: "fill_value_invalid" },
|
||||
});
|
||||
expect(h.element.value).toBe("12");
|
||||
expect(h.insert).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("reports page formatting as unconfirmed without retrying the write", async () => {
|
||||
const h = await setup();
|
||||
h.element.addEventListener("change", () => {
|
||||
h.element.value = h.element.value.toUpperCase();
|
||||
});
|
||||
expect(await h.fill()).toMatchObject({
|
||||
code: "cdp_failed",
|
||||
data: { reason: "fill_value_mismatch" },
|
||||
});
|
||||
expect(h.element.value).toBe("HELLO");
|
||||
expect(h.insert).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it.each([
|
||||
["focus", false],
|
||||
["focus", true],
|
||||
["value", false],
|
||||
["value", true],
|
||||
["cancel", false],
|
||||
["cancel", true],
|
||||
["throw", false],
|
||||
["throw", true],
|
||||
] as const)("stops when positioning triggers %s (clear=%s) and releases the key", async (action, clearBefore) => {
|
||||
const h = await setup();
|
||||
vi.spyOn(document, "hasFocus").mockReturnValue(false);
|
||||
const controller = new AbortController();
|
||||
const other = document.createElement("input");
|
||||
document.body.append(other);
|
||||
const send = h.send.getMockImplementation()!;
|
||||
h.send.mockImplementation(async (tabId, method, params) => {
|
||||
const reply = await send(tabId, method, params);
|
||||
if (
|
||||
method === "Input.dispatchKeyEvent" &&
|
||||
(params as { type: string }).type === "rawKeyDown"
|
||||
) {
|
||||
if (action === "focus") other.focus();
|
||||
if (action === "value") h.element.value = "changed";
|
||||
if (action === "cancel") controller.abort();
|
||||
if (action === "throw") throw new Error("dispatch failed");
|
||||
}
|
||||
return reply;
|
||||
});
|
||||
const result = await h.fill("!", clearBefore, controller.signal);
|
||||
expect(result).toMatchObject({ code: action === "cancel" ? "cancelled" : "cdp_failed" });
|
||||
expect(h.insert).not.toHaveBeenCalled();
|
||||
expect(other.value).toBe("");
|
||||
expect(h.send).toHaveBeenCalledWith(
|
||||
4,
|
||||
"Input.dispatchKeyEvent",
|
||||
expect.objectContaining({ type: "keyUp" }),
|
||||
);
|
||||
expect(h.release).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it.each([true, false])("verifies an empty request with clear_before=%s", async (clearBefore) => {
|
||||
const h = await setup();
|
||||
expect(await h.fill("", clearBefore)).toMatchObject({ value_length: clearBefore ? 0 : 3 });
|
||||
expect(h.element.value).toBe(clearBefore ? "" : "old");
|
||||
expect(h.insert).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("cancels after insertion without sending change events", async () => {
|
||||
const h = await setup();
|
||||
const controller = new AbortController();
|
||||
h.insert.mockImplementation(async () => controller.abort());
|
||||
expect(await h.fill("hello", true, controller.signal)).toMatchObject({ code: "cancelled" });
|
||||
expect(h.script).toHaveBeenCalledTimes(2);
|
||||
expect(h.release).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("cancels after resolving the remote object without clearing the field", async () => {
|
||||
const h = await setup();
|
||||
const controller = new AbortController();
|
||||
const send = h.send.getMockImplementation()!;
|
||||
h.send.mockImplementation(async (tabId, method, params) => {
|
||||
const result = await send(tabId, method, params);
|
||||
if (method === "DOM.resolveNode") controller.abort();
|
||||
return result;
|
||||
});
|
||||
expect(await h.fill("hello", true, controller.signal)).toMatchObject({ code: "cancelled" });
|
||||
expect(h.element.value).toBe("old");
|
||||
expect(h.script).not.toHaveBeenCalled();
|
||||
expect(h.release).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("honors cancellation during verification", async () => {
|
||||
const h = await setup();
|
||||
const controller = new AbortController();
|
||||
h.element.addEventListener("change", () => controller.abort());
|
||||
expect(await h.fill("hello", true, controller.signal)).toMatchObject({ code: "cancelled" });
|
||||
expect(h.release).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("keeps the verified outcome when releasing the remote object fails", async () => {
|
||||
const h = await setup();
|
||||
h.release.mockRejectedValueOnce(new Error("target closed"));
|
||||
expect(await h.fill()).toMatchObject({ value_length: 5 });
|
||||
});
|
||||
});
|
||||
@@ -1,5 +1,5 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { resolveNodeGeometry } from "../frame-geometry";
|
||||
import { resolveFrameProjection, resolveNodeGeometry } from "../frame-geometry";
|
||||
import {
|
||||
clipPolygon,
|
||||
polygonArea,
|
||||
@@ -10,7 +10,6 @@ import {
|
||||
rectPolygon,
|
||||
regionBounds,
|
||||
} from "../geometry";
|
||||
import { GeometryContext } from "../geometry/frame-context";
|
||||
import type { CdpRunner } from "../shared";
|
||||
|
||||
describe("frame geometry projection", () => {
|
||||
@@ -83,24 +82,28 @@ describe("frame geometry projection", () => {
|
||||
throw new Error(`unexpected child command ${method}`);
|
||||
}) as CdpRunner["sendToTarget"],
|
||||
};
|
||||
const projection = await new GeometryContext(cdp, 4, {
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "same-process-parent",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 10,
|
||||
target: { tabId: 4 },
|
||||
},
|
||||
{
|
||||
frameId: "oopif",
|
||||
parentFrameId: "same-process-parent",
|
||||
ownerBackendNodeId: 20,
|
||||
target: { tabId: 4, sessionId: "oopif-session" },
|
||||
},
|
||||
],
|
||||
}).targetProjection("oopif");
|
||||
const projection = await resolveFrameProjection(
|
||||
cdp,
|
||||
{
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "same-process-parent",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 10,
|
||||
target: { tabId: 4 },
|
||||
},
|
||||
{
|
||||
frameId: "oopif",
|
||||
parentFrameId: "same-process-parent",
|
||||
ownerBackendNodeId: 20,
|
||||
target: { tabId: 4, sessionId: "oopif-session" },
|
||||
},
|
||||
],
|
||||
},
|
||||
"oopif",
|
||||
);
|
||||
|
||||
expect(projection?.edges).toEqual([
|
||||
{
|
||||
@@ -166,7 +169,6 @@ describe("frame geometry projection", () => {
|
||||
expect(geometry).toMatchObject({
|
||||
topBounds: { x: 100, y: 100, width: 200, height: 100 },
|
||||
actionPoint: { x: 200, y: 150 },
|
||||
targetActionPoint: { x: 100, y: 50 },
|
||||
});
|
||||
});
|
||||
|
||||
@@ -259,7 +261,6 @@ describe("frame geometry projection", () => {
|
||||
expect(geometry).toMatchObject({
|
||||
topBounds: { x: 100, y: 100, width: 200, height: 100 },
|
||||
actionPoint: { x: 200, y: 150 },
|
||||
targetActionPoint: { x: 200, y: 150 },
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -1,380 +0,0 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import type { CdpFrameGraph } from "@/browser-driver/frame-graph";
|
||||
import { resolveNodeGeometry } from "../frame-geometry";
|
||||
import { rectPolygon } from "../geometry";
|
||||
import {
|
||||
projectSnapshotRect,
|
||||
screenshotPageRect,
|
||||
snapshotViewportRect,
|
||||
} from "../geometry/coordinate-types";
|
||||
import { GeometryContext } from "../geometry/frame-context";
|
||||
import type { CdpRunner } from "../shared";
|
||||
|
||||
const target = { tabId: 4 };
|
||||
const graph: CdpFrameGraph = {
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target },
|
||||
{ frameId: "child", parentFrameId: "main", ownerBackendNodeId: 10, target },
|
||||
],
|
||||
};
|
||||
|
||||
function driver() {
|
||||
const send = vi.fn(async (_tab: number, method: string): Promise<object> => {
|
||||
if (method === "Page.getLayoutMetrics")
|
||||
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 1000 } };
|
||||
if (method === "DOM.getBoxModel")
|
||||
return { model: { content: [52.5, 612.5, 427.5, 612.5, 427.5, 862.5, 52.5, 862.5] } };
|
||||
if (method === "DOM.resolveNode") return { object: { objectId: "owner" } };
|
||||
if (method === "Runtime.callFunctionOn")
|
||||
return { result: { value: { width: 300, height: 200 } } };
|
||||
if (method === "Runtime.releaseObject") return {};
|
||||
throw new Error(method);
|
||||
});
|
||||
return { send: send as CdpRunner["send"], getFrameGraph: vi.fn(async () => graph), calls: send };
|
||||
}
|
||||
|
||||
describe("measurement geometry context", () => {
|
||||
it("shares in-flight frame measurements, but never shares them with the next context", async () => {
|
||||
const cdp = driver();
|
||||
const context = new GeometryContext(cdp, 4);
|
||||
const results = await Promise.all(
|
||||
Array.from({ length: 20 }, () => context.targetProjection("child")),
|
||||
);
|
||||
expect(results.every((value) => value === results[0])).toBe(true);
|
||||
expect(cdp.getFrameGraph).toHaveBeenCalledTimes(1);
|
||||
expect(cdp.calls.mock.calls.map(([, method]) => method)).toEqual([
|
||||
"Page.getLayoutMetrics",
|
||||
"DOM.getBoxModel",
|
||||
]);
|
||||
await new GeometryContext(cdp, 4).targetProjection("child");
|
||||
expect(cdp.getFrameGraph).toHaveBeenCalledTimes(2);
|
||||
expect(cdp.calls.mock.calls).toHaveLength(4);
|
||||
});
|
||||
|
||||
it("projects frame-local snapshot CSS bounds through the scaled owner content quad", async () => {
|
||||
const cdp = driver();
|
||||
const context = new GeometryContext(cdp, 4);
|
||||
const source = { target, frameId: "child" };
|
||||
const projection = await context.snapshotProjection(source, 10, [], {
|
||||
width: 1000,
|
||||
height: 1000,
|
||||
});
|
||||
if (projection.status !== "available") throw new Error("expected available projection");
|
||||
const input = snapshotViewportRect([17, 23, 120, 40], source, { x: 0, y: 0 });
|
||||
expect(projectSnapshotRect(input!, projection.projection)).toEqual({
|
||||
x: 73.75,
|
||||
y: 641.25,
|
||||
width: 150,
|
||||
height: 50,
|
||||
});
|
||||
const scrolled = snapshotViewportRect([17, 23, 120, 40], source, { x: 0, y: 10 });
|
||||
expect(projectSnapshotRect(scrolled!, projection.projection)?.y).toBe(628.75);
|
||||
await context.snapshotProjection(source, 10, [], { width: 1000, height: 1000 });
|
||||
expect(cdp.calls.mock.calls.filter(([, method]) => method === "DOM.resolveNode")).toHaveLength(
|
||||
1,
|
||||
);
|
||||
expect(
|
||||
cdp.calls.mock.calls.filter(([, method]) => method === "Runtime.releaseObject"),
|
||||
).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("falls back only for missing or invalid batch entries and does not reuse sizes across contexts", async () => {
|
||||
const cdp = driver();
|
||||
const original = cdp.send;
|
||||
const calls: string[] = [];
|
||||
cdp.send = async (tabId, method, params) => {
|
||||
calls.push(method);
|
||||
if (method === "Runtime.evaluate")
|
||||
return {
|
||||
result: {
|
||||
deepSerializedValue: {
|
||||
type: "array",
|
||||
value: [
|
||||
{
|
||||
type: "array",
|
||||
value: [
|
||||
{ type: "node", value: { backendNodeId: 10 } },
|
||||
{ type: "string", value: '{"width":300,"height":200}' },
|
||||
],
|
||||
},
|
||||
{
|
||||
type: "array",
|
||||
value: [
|
||||
{ type: "node", value: { backendNodeId: 11 } },
|
||||
{ type: "string", value: '{"width":0,"height":200}' },
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
} as never;
|
||||
if (method === "Runtime.releaseObjectGroup") return {} as never;
|
||||
return original(tabId, method, params);
|
||||
};
|
||||
const context = new GeometryContext(cdp, 4);
|
||||
context.registerSnapshotOwners(target, new Set([10, 11]));
|
||||
const values = await Promise.all(
|
||||
[10, 11].map((id) =>
|
||||
context.snapshotProjection({ target, frameId: String(id) }, id, [], {
|
||||
width: 1000,
|
||||
height: 1000,
|
||||
}),
|
||||
),
|
||||
);
|
||||
expect(values.every((value) => value.status === "available")).toBe(true);
|
||||
expect(calls.filter((method) => method === "DOM.resolveNode")).toHaveLength(1);
|
||||
expect(calls.filter((method) => method === "Runtime.evaluate")).toHaveLength(1);
|
||||
await new GeometryContext(cdp, 4).snapshotProjection({ target, frameId: "child" }, 10, [], {
|
||||
width: 1000,
|
||||
height: 1000,
|
||||
});
|
||||
expect(calls.filter((method) => method === "DOM.resolveNode")).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("waits for batch object cleanup and propagates cancellation", async () => {
|
||||
const cdp = driver();
|
||||
const original = cdp.send;
|
||||
const controller = new AbortController();
|
||||
let release!: () => void;
|
||||
let cleanup!: () => void;
|
||||
const started = new Promise<void>((resolve) => {
|
||||
cleanup = resolve;
|
||||
});
|
||||
cdp.send = async (tabId, method, params) => {
|
||||
if (method === "Runtime.evaluate") return {} as never;
|
||||
if (method === "Runtime.releaseObjectGroup") {
|
||||
cleanup();
|
||||
await new Promise<void>((resolve) => {
|
||||
release = resolve;
|
||||
});
|
||||
return {} as never;
|
||||
}
|
||||
return original(tabId, method, params);
|
||||
};
|
||||
const context = new GeometryContext(cdp, 4, undefined, controller.signal);
|
||||
context.registerSnapshotOwners(target, new Set([10, 11]));
|
||||
let settled = false;
|
||||
const operation = context
|
||||
.snapshotProjection({ target, frameId: "child" }, 10, [], { width: 1000, height: 1000 })
|
||||
.finally(() => {
|
||||
settled = true;
|
||||
});
|
||||
const rejected = expect(operation).rejects.toMatchObject({ name: "AbortError" });
|
||||
await started;
|
||||
controller.abort();
|
||||
expect(settled).toBe(false);
|
||||
release();
|
||||
await rejected;
|
||||
expect(cdp.calls.mock.calls.some(([, method]) => method === "DOM.resolveNode")).toBe(false);
|
||||
});
|
||||
|
||||
it("rejects owner mismatches and invalid numeric bounds", () => {
|
||||
const input = snapshotViewportRect(
|
||||
[30, 50, 120, 40],
|
||||
{ target, frameId: "main" },
|
||||
{ x: 10, y: 20 },
|
||||
)!;
|
||||
const projection = {
|
||||
source: { target, frameId: "main" },
|
||||
geometry: { sourceClips: [], edges: [], topViewport: { width: 800, height: 600 } },
|
||||
};
|
||||
expect(projectSnapshotRect(input, projection)).toEqual({
|
||||
x: 20,
|
||||
y: 30,
|
||||
width: 120,
|
||||
height: 40,
|
||||
});
|
||||
expect(
|
||||
projectSnapshotRect(input, { ...projection, source: { target, frameId: "other" } }),
|
||||
).toBeNull();
|
||||
expect(
|
||||
projectSnapshotRect(input, {
|
||||
...projection,
|
||||
source: { target: { tabId: 5 }, frameId: "main" },
|
||||
}),
|
||||
).toBeNull();
|
||||
expect(snapshotViewportRect([0, 0, NaN, 40], { target }, { x: 0, y: 0 })).toBeNull();
|
||||
expect(snapshotViewportRect([0, 0, 10, 40], { target }, { x: Infinity, y: 0 })).toBeNull();
|
||||
});
|
||||
|
||||
it("does not turn a border quad or a failed owner read into a content projection", async () => {
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async () => ({
|
||||
model: { border: [0, 0, 100, 0, 100, 100, 0, 100] },
|
||||
})) as CdpRunner["send"],
|
||||
};
|
||||
expect(
|
||||
await new GeometryContext(cdp, 4).snapshotProjection({ target, frameId: "child" }, 10, [], {
|
||||
width: 100,
|
||||
height: 100,
|
||||
}),
|
||||
).toEqual({
|
||||
status: "unavailable",
|
||||
source: { target, frameId: "child" },
|
||||
ownerBackendNodeId: 10,
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps ancestor clips in target coordinates instead of applying nested offsets twice", async () => {
|
||||
const cdp = driver();
|
||||
const projection = await new GeometryContext(cdp, 4).snapshotProjection(
|
||||
{ target, frameId: "nested" },
|
||||
10,
|
||||
[rectPolygon({ x: 60, y: 620, w: 100, h: 100 })],
|
||||
{ width: 1000, height: 1000 },
|
||||
);
|
||||
if (projection.status !== "available") throw new Error("expected available projection");
|
||||
const input = snapshotViewportRect(
|
||||
[0, 0, 300, 200],
|
||||
{ target, frameId: "nested" },
|
||||
{ x: 0, y: 0 },
|
||||
);
|
||||
expect(projectSnapshotRect(input!, projection.projection)).toEqual({
|
||||
x: 60,
|
||||
y: 620,
|
||||
width: 100,
|
||||
height: 100,
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects malformed frame ancestry without looping", async () => {
|
||||
const cdp = driver();
|
||||
const cycle: CdpFrameGraph = {
|
||||
rootFrameId: "a",
|
||||
frames: [
|
||||
{ frameId: "a", parentFrameId: "b", target },
|
||||
{ frameId: "b", parentFrameId: "a", target },
|
||||
],
|
||||
};
|
||||
expect(await new GeometryContext(cdp, 4, cycle).targetProjection("a")).toBeNull();
|
||||
expect(cdp.calls).not.toHaveBeenCalled();
|
||||
const orphan: CdpFrameGraph = {
|
||||
rootFrameId: "a",
|
||||
frames: [{ frameId: "a", parentFrameId: "missing", target }],
|
||||
};
|
||||
expect(await new GeometryContext(cdp, 4, orphan).targetProjection("a")).toBeNull();
|
||||
});
|
||||
|
||||
it("rejects a live node whose frame belongs to another target before any DOM input", async () => {
|
||||
const cdp = driver();
|
||||
expect(
|
||||
await resolveNodeGeometry(
|
||||
cdp,
|
||||
4,
|
||||
{ target: { tabId: 4, sessionId: "wrong" }, frameId: "child", backendNodeId: 20 },
|
||||
{ scrollIntoView: true },
|
||||
),
|
||||
).toMatchObject({ code: "cdp_failed" });
|
||||
expect(cdp.calls).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("bounds concurrent measurements and does not dispatch queued reads after cancellation", async () => {
|
||||
let active = 0;
|
||||
let peak = 0;
|
||||
const release: Array<() => void> = [];
|
||||
const send = vi.fn(async () => {
|
||||
active++;
|
||||
peak = Math.max(peak, active);
|
||||
await new Promise<void>((resolve) => release.push(resolve));
|
||||
active--;
|
||||
return { cssLayoutViewport: { clientWidth: 100, clientHeight: 100 } };
|
||||
});
|
||||
const controller = new AbortController();
|
||||
const context = new GeometryContext(
|
||||
{ send: send as CdpRunner["send"] },
|
||||
4,
|
||||
undefined,
|
||||
controller.signal,
|
||||
);
|
||||
const tasks = Array.from({ length: 8 }, (_, index) =>
|
||||
context.viewport({ tabId: 4, sessionId: String(index) }),
|
||||
);
|
||||
const settled = Promise.allSettled(tasks);
|
||||
expect(send).toHaveBeenCalledTimes(4);
|
||||
controller.abort();
|
||||
for (const done of release) done();
|
||||
const results = await settled;
|
||||
expect(peak).toBe(4);
|
||||
expect(send).toHaveBeenCalledTimes(4);
|
||||
expect(results.filter((item) => item.status === "rejected")).toHaveLength(4);
|
||||
});
|
||||
|
||||
it("releases an owner object even when cancellation arrives before its viewport read", async () => {
|
||||
const controller = new AbortController();
|
||||
const cdp = driver();
|
||||
const original = cdp.calls.getMockImplementation()!;
|
||||
cdp.calls.mockImplementation(async (tab, method) => {
|
||||
const result = await original(tab, method);
|
||||
if (method === "DOM.resolveNode") controller.abort();
|
||||
return result;
|
||||
});
|
||||
const context = new GeometryContext(cdp, 4, undefined, controller.signal);
|
||||
await expect(
|
||||
context.snapshotProjection({ target, frameId: "child" }, 10, [], {
|
||||
width: 1000,
|
||||
height: 1000,
|
||||
}),
|
||||
).rejects.toMatchObject({ name: "AbortError" });
|
||||
expect(cdp.calls.mock.calls.map(([, method]) => method)).toEqual([
|
||||
"DOM.getBoxModel",
|
||||
"DOM.resolveNode",
|
||||
"Runtime.releaseObject",
|
||||
]);
|
||||
});
|
||||
|
||||
it("reads live owner geometry only after scrolling", async () => {
|
||||
const cdp = driver();
|
||||
const original = cdp.calls.getMockImplementation()!;
|
||||
let scrolled = false;
|
||||
cdp.calls.mockImplementation(async (tab, method) => {
|
||||
if (method === "DOM.scrollIntoViewIfNeeded") {
|
||||
scrolled = true;
|
||||
return {};
|
||||
}
|
||||
if (method === "DOM.getContentQuads") {
|
||||
expect(scrolled).toBe(true);
|
||||
return { quads: [[60, 620, 100, 620, 100, 640, 60, 640]] };
|
||||
}
|
||||
if (method === "DOM.getBoxModel") expect(scrolled).toBe(true);
|
||||
return original(tab, method);
|
||||
});
|
||||
expect(
|
||||
await resolveNodeGeometry(
|
||||
cdp,
|
||||
4,
|
||||
{ target, frameId: "child", backendNodeId: 20 },
|
||||
{ scrollIntoView: true },
|
||||
),
|
||||
).toMatchObject({ topBounds: { x: 60, y: 620, width: 40, height: 20 } });
|
||||
expect(cdp.getFrameGraph).toHaveBeenCalledTimes(1);
|
||||
expect(cdp.calls.mock.calls.filter(([, method]) => method === "DOM.getBoxModel")).toHaveLength(
|
||||
1,
|
||||
);
|
||||
});
|
||||
|
||||
it("adapts viewport CSS bounds to page DIP with scroll and browser zoom, never raster DPR", () => {
|
||||
const viewport = { width: 1600, height: 1000, scrollX: 10, scrollY: 100, cssToDip: 0.9 };
|
||||
const clip = screenshotPageRect({ x: 20, y: 30, width: 120, height: 40 }, viewport);
|
||||
expect(clip).toEqual({ space: "page-dip", rect: { x: 27, y: 117, width: 108, height: 36 } });
|
||||
expect(
|
||||
screenshotPageRect({ x: 20, y: 30, width: 120, height: 40 }, { ...viewport, cssToDip: NaN }),
|
||||
).toBeNull();
|
||||
expect(
|
||||
screenshotPageRect({ x: 20, y: 30, width: 120, height: 40 }, { ...viewport, cssToDip: 0 }),
|
||||
).toBeNull();
|
||||
});
|
||||
|
||||
it("retries a rejected measurement only in a new operation", async () => {
|
||||
const cdp = driver();
|
||||
cdp.calls.mockRejectedValueOnce(new Error("unavailable"));
|
||||
const context = new GeometryContext(cdp, 4);
|
||||
await expect(context.viewport(target)).rejects.toThrow("unavailable");
|
||||
await expect(context.viewport(target)).rejects.toThrow("unavailable");
|
||||
expect(cdp.calls).toHaveBeenCalledTimes(1);
|
||||
expect(await new GeometryContext(cdp, 4).viewport(target)).toEqual({
|
||||
width: 1000,
|
||||
height: 1000,
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -21,7 +21,7 @@ function fakeAgentWindow(ids: number[]) {
|
||||
return id;
|
||||
}),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -540,21 +540,6 @@ describe("handleHover", () => {
|
||||
});
|
||||
});
|
||||
|
||||
function successfulFillScript(params: unknown) {
|
||||
const script = params as { arguments?: Array<{ value: unknown }>; functionDeclaration: string };
|
||||
const args = script.arguments ?? [];
|
||||
return {
|
||||
result: {
|
||||
value:
|
||||
args.length === 2
|
||||
? { before: "", expected: args[0].value }
|
||||
: script.functionDeclaration.startsWith("function(expected)")
|
||||
? { connected: true, matches: true, valueLength: String(args[0].value).length }
|
||||
: "ready",
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe("handleFill", () => {
|
||||
it("returns not_found for unknown ref", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
@@ -601,8 +586,7 @@ describe("handleFill", () => {
|
||||
"DOM.scrollIntoViewIfNeeded": () => ({}),
|
||||
"DOM.focus": () => ({}),
|
||||
"DOM.resolveNode": () => ({ object: { objectId: "obj-1" } }),
|
||||
"Runtime.callFunctionOn": successfulFillScript,
|
||||
"Runtime.releaseObject": () => ({}),
|
||||
"Runtime.callFunctionOn": () => ({ result: { type: "undefined" } }),
|
||||
"Input.insertText": () => ({}),
|
||||
});
|
||||
const res = await handleFill(
|
||||
@@ -616,12 +600,13 @@ describe("handleFill", () => {
|
||||
expect(res.used_ref).toBe("e1");
|
||||
const insert = fake.sent.find((c) => c.method === "Input.insertText");
|
||||
expect(insert?.params).toEqual({ text: "hello" });
|
||||
// Foreground replacement needs no extra caret-positioning round trip.
|
||||
// clear_before defaults to true → callFunctionOn invoked once to
|
||||
// clear, once to fire input/change after typing.
|
||||
const callFns = fake.sent.filter((c) => c.method === "Runtime.callFunctionOn");
|
||||
expect(callFns).toHaveLength(4);
|
||||
expect(callFns.length).toBeGreaterThanOrEqual(2);
|
||||
});
|
||||
|
||||
it("passes clear_before=false to preparation and verifies the result", async () => {
|
||||
it("clear_before=false skips the wipe call", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.refStore.set("e1", 1, { tabId: 4 });
|
||||
@@ -633,12 +618,12 @@ describe("handleFill", () => {
|
||||
"DOM.focus": () => ({}),
|
||||
"DOM.resolveNode": () => ({ object: { objectId: "obj-2" } }),
|
||||
"Runtime.callFunctionOn": (p) => {
|
||||
const args = (p as { arguments?: Array<{ value: unknown }> }).arguments ?? [];
|
||||
if (args.length === 2) expect(args[1].value).toBe(false);
|
||||
return successfulFillScript(p);
|
||||
const fn = (p as { functionDeclaration?: string }).functionDeclaration ?? "";
|
||||
// No "this.value = ''" clearing on a no-clear path; only the
|
||||
// post-input dispatchEvent.
|
||||
expect(fn).not.toMatch(/this\.value\s*=\s*''/);
|
||||
return { result: { type: "undefined" } };
|
||||
},
|
||||
"Input.dispatchKeyEvent": () => ({}),
|
||||
"Runtime.releaseObject": () => ({}),
|
||||
"Input.insertText": () => ({}),
|
||||
});
|
||||
const res = await handleFill(
|
||||
@@ -665,8 +650,7 @@ describe("handleFill", () => {
|
||||
"DOM.scrollIntoViewIfNeeded": () => ({}),
|
||||
"DOM.focus": () => ({}),
|
||||
"DOM.resolveNode": () => ({ object: { objectId: "obj-3" } }),
|
||||
"Runtime.callFunctionOn": successfulFillScript,
|
||||
"Runtime.releaseObject": () => ({}),
|
||||
"Runtime.callFunctionOn": () => ({ result: { type: "undefined" } }),
|
||||
"Input.insertText": () => ({}),
|
||||
});
|
||||
const res = await handleFill(
|
||||
|
||||
@@ -1,266 +0,0 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { SessionManager } from "@/session-manager/manager";
|
||||
import type { BrowserNavigationApi } from "../browser-navigation";
|
||||
import { handleNavigate, handleReload } from "../navigation";
|
||||
import type { CdpRunner } from "../shared";
|
||||
|
||||
const denied = "Cannot access a chrome-extension:// URL of different extension";
|
||||
const url = "https://example.test/recovered";
|
||||
|
||||
function event<T extends (...args: never[]) => void>() {
|
||||
const listeners = new Set<T>();
|
||||
return {
|
||||
listeners,
|
||||
addListener: (listener: T) => listeners.add(listener),
|
||||
removeListener: (listener: T) => listeners.delete(listener),
|
||||
fire: (...args: Parameters<T>) => {
|
||||
for (const listener of [...listeners]) listener(...args);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function fixture() {
|
||||
const manager = new SessionManager({
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 100),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 4),
|
||||
},
|
||||
});
|
||||
await manager.start("test");
|
||||
type Listener = Parameters<BrowserNavigationApi["onCommitted"]["addListener"]>[0];
|
||||
const events = {
|
||||
onCommitted: event<Listener>(),
|
||||
onDOMContentLoaded: event<Listener>(),
|
||||
onCompleted: event<Listener>(),
|
||||
onErrorOccurred: event<Listener>(),
|
||||
};
|
||||
const cdpEvents = event<Parameters<NonNullable<CdpRunner["onEvent"]>>[0]>();
|
||||
const state = { blocked: true, destinationBlocked: false, emitIdle: true };
|
||||
const send = vi.fn(async (_tabId: number, method: string) => {
|
||||
if (state.blocked) throw new Error(denied);
|
||||
if (method === "Page.getFrameTree") {
|
||||
return { frameTree: { frame: { id: "main", loaderId: "new-loader" } } };
|
||||
}
|
||||
if (method === "Runtime.evaluate") return { result: { value: "complete" } };
|
||||
if (method === "Page.setLifecycleEventsEnabled" && state.emitIdle) {
|
||||
cdpEvents.fire({ tabId: 4 }, "Page.lifecycleEvent", {
|
||||
frameId: "main",
|
||||
loaderId: "new-loader",
|
||||
name: "networkIdle",
|
||||
});
|
||||
}
|
||||
if (method === "Page.navigate") return { frameId: "main", loaderId: "new-loader" };
|
||||
return {};
|
||||
});
|
||||
const cdp: CdpRunner = {
|
||||
send: send as CdpRunner["send"],
|
||||
onEvent: (listener) => {
|
||||
cdpEvents.addListener(listener);
|
||||
return { dispose: () => cdpEvents.removeListener(listener) };
|
||||
},
|
||||
};
|
||||
const main = { tabId: 4, frameId: 0, documentId: "new-document" };
|
||||
const navigate = async () => {
|
||||
state.blocked = state.destinationBlocked;
|
||||
events.onCommitted.fire(main);
|
||||
events.onDOMContentLoaded.fire(main);
|
||||
events.onCompleted.fire(main);
|
||||
};
|
||||
const browserNavigation = { ...events, update: vi.fn(navigate), reload: vi.fn(navigate) };
|
||||
const tab = { id: 4, windowId: 100, active: true, url } as chrome.tabs.Tab;
|
||||
const tabsApi = { get: vi.fn(async () => tab), query: vi.fn(async () => [tab]) };
|
||||
const deps = { cdp, tabsApi, browserNavigation, defaultTimeoutMs: 1000 };
|
||||
const expectCleanedUp = () => {
|
||||
for (const entry of Object.values(events)) expect(entry.listeners.size).toBe(0);
|
||||
expect(cdpEvents.listeners.size).toBe(0);
|
||||
};
|
||||
return { manager, deps, state, send, events, main, cdpEvents, expectCleanedUp };
|
||||
}
|
||||
|
||||
describe("navigation after Chrome denies extension-frame access", () => {
|
||||
it.each([
|
||||
"load",
|
||||
"commit",
|
||||
"domcontentloaded",
|
||||
"networkidle",
|
||||
] as const)("recovers the same tab with the requested %s checkpoint", async (waitUntil) => {
|
||||
const f = await fixture();
|
||||
const result = await handleNavigate(
|
||||
f.manager,
|
||||
{
|
||||
session_id: "test",
|
||||
url,
|
||||
wait_until: waitUntil,
|
||||
},
|
||||
f.deps,
|
||||
);
|
||||
|
||||
expect(result).toMatchObject({ tab_id: 4, final_url: url, reached: waitUntil });
|
||||
expect(f.deps.browserNavigation.update).toHaveBeenCalledExactlyOnceWith(4, { url });
|
||||
expect(f.send.mock.calls.some(([, method]) => method === "Page.navigate")).toBe(false);
|
||||
expect(f.send).toHaveBeenCalledWith(4, "Page.enable", {});
|
||||
f.expectCleanedUp();
|
||||
});
|
||||
|
||||
it("recovers reload through tabs.reload and preserves hard reload", async () => {
|
||||
const f = await fixture();
|
||||
const result = await handleReload(f.manager, { session_id: "test", hard: true }, f.deps);
|
||||
expect(result).toMatchObject({ tab_id: 4, reached: "load" });
|
||||
expect(f.deps.browserNavigation.reload).toHaveBeenCalledExactlyOnceWith(4, {
|
||||
bypassCache: true,
|
||||
});
|
||||
expect(f.deps.browserNavigation.update).not.toHaveBeenCalled();
|
||||
f.expectCleanedUp();
|
||||
});
|
||||
|
||||
it("reports the restriction when reload leaves the conflicting frame present", async () => {
|
||||
const f = await fixture();
|
||||
f.state.destinationBlocked = true;
|
||||
const result = await handleReload(f.manager, { session_id: "test" }, f.deps);
|
||||
expect(result).toMatchObject({
|
||||
code: "cdp_failed",
|
||||
data: { reason: "cdp_extension_access_denied" },
|
||||
});
|
||||
expect(f.deps.browserNavigation.reload).toHaveBeenCalledOnce();
|
||||
f.expectCleanedUp();
|
||||
});
|
||||
|
||||
it("does not fall back for unrelated debugger failures", async () => {
|
||||
const f = await fixture();
|
||||
f.send.mockRejectedValue(new Error("Another debugger is already attached"));
|
||||
const result = await handleNavigate(f.manager, { session_id: "test", url }, f.deps);
|
||||
expect(result).toMatchObject({
|
||||
code: "cdp_failed",
|
||||
message: "Another debugger is already attached",
|
||||
});
|
||||
expect(f.deps.browserNavigation.update).not.toHaveBeenCalled();
|
||||
f.expectCleanedUp();
|
||||
});
|
||||
|
||||
it("does not replay navigation when a dispatched CDP command fails", async () => {
|
||||
const f = await fixture();
|
||||
f.state.blocked = false;
|
||||
const send = f.send.getMockImplementation()!;
|
||||
f.send.mockImplementation(async (tabId, method) => {
|
||||
if (method === "Page.navigate") throw new Error(denied);
|
||||
return send(tabId, method);
|
||||
});
|
||||
const result = await handleNavigate(f.manager, { session_id: "test", url }, f.deps);
|
||||
expect(result).toMatchObject({
|
||||
code: "cdp_failed",
|
||||
data: { reason: "cdp_extension_access_denied" },
|
||||
});
|
||||
expect(f.deps.browserNavigation.update).not.toHaveBeenCalled();
|
||||
f.expectCleanedUp();
|
||||
});
|
||||
|
||||
it("keeps the Agent Window guard in front of the recovery", async () => {
|
||||
const f = await fixture();
|
||||
f.deps.tabsApi.get.mockResolvedValue({
|
||||
id: 4,
|
||||
windowId: 200,
|
||||
active: true,
|
||||
url,
|
||||
} as chrome.tabs.Tab);
|
||||
const result = await handleNavigate(f.manager, { session_id: "test", tab_id: 4, url }, f.deps);
|
||||
expect(result).toMatchObject({ code: "permission_denied" });
|
||||
expect(f.send).not.toHaveBeenCalled();
|
||||
expect(f.deps.browserNavigation.update).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("ignores old-document, subframe, and other-tab completion events", async () => {
|
||||
const f = await fixture();
|
||||
f.deps.browserNavigation.update.mockImplementation(async () => {});
|
||||
let settled = false;
|
||||
const work = handleNavigate(f.manager, { session_id: "test", url }, f.deps).then((result) => {
|
||||
settled = true;
|
||||
return result;
|
||||
});
|
||||
await vi.waitFor(() => expect(f.deps.browserNavigation.update).toHaveBeenCalledOnce());
|
||||
f.events.onCompleted.fire(f.main);
|
||||
f.events.onCommitted.fire({ ...f.main, tabId: 9 });
|
||||
f.events.onCommitted.fire({ ...f.main, frameId: 1 });
|
||||
f.events.onCompleted.fire(f.main);
|
||||
await Promise.resolve();
|
||||
expect(settled).toBe(false);
|
||||
f.events.onCommitted.fire(f.main);
|
||||
f.events.onCompleted.fire({ ...f.main, documentId: "old-document" });
|
||||
await Promise.resolve();
|
||||
expect(settled).toBe(false);
|
||||
f.state.blocked = false;
|
||||
f.events.onCompleted.fire(f.main);
|
||||
expect(await work).toMatchObject({ reached: "load" });
|
||||
f.expectCleanedUp();
|
||||
});
|
||||
|
||||
it("does not infer network idle from a browser load event", async () => {
|
||||
const f = await fixture();
|
||||
f.state.emitIdle = false;
|
||||
const result = await handleNavigate(
|
||||
f.manager,
|
||||
{
|
||||
session_id: "test",
|
||||
url,
|
||||
wait_until: "networkidle",
|
||||
timeout_ms: 10,
|
||||
},
|
||||
f.deps,
|
||||
);
|
||||
expect(result).toMatchObject({ reached: "timeout" });
|
||||
f.expectCleanedUp();
|
||||
});
|
||||
|
||||
it("cleans up browser listeners on timeout", async () => {
|
||||
const f = await fixture();
|
||||
f.deps.browserNavigation.update.mockImplementation(async () => {});
|
||||
const result = await handleNavigate(
|
||||
f.manager,
|
||||
{ session_id: "test", url, timeout_ms: 10 },
|
||||
f.deps,
|
||||
);
|
||||
expect(result).toMatchObject({ reached: "timeout" });
|
||||
f.expectCleanedUp();
|
||||
});
|
||||
|
||||
it("does not navigate an already-cancelled request", async () => {
|
||||
const f = await fixture();
|
||||
const result = await handleNavigate(
|
||||
f.manager,
|
||||
{ session_id: "test", url },
|
||||
{
|
||||
...f.deps,
|
||||
signal: AbortSignal.abort(),
|
||||
},
|
||||
);
|
||||
expect(result).toMatchObject({ code: "cancelled" });
|
||||
expect(f.deps.browserNavigation.update).not.toHaveBeenCalled();
|
||||
f.expectCleanedUp();
|
||||
});
|
||||
|
||||
it("cleans up browser listeners when cancelled after dispatch", async () => {
|
||||
const f = await fixture();
|
||||
const ac = new AbortController();
|
||||
f.deps.browserNavigation.update.mockImplementation(async () => {
|
||||
ac.abort();
|
||||
});
|
||||
const result = await handleNavigate(
|
||||
f.manager,
|
||||
{ session_id: "test", url },
|
||||
{ ...f.deps, signal: ac.signal },
|
||||
);
|
||||
expect(result).toMatchObject({ code: "cancelled" });
|
||||
expect(f.deps.browserNavigation.update).toHaveBeenCalledOnce();
|
||||
f.expectCleanedUp();
|
||||
});
|
||||
|
||||
it("reports browser navigation errors and releases listeners", async () => {
|
||||
const f = await fixture();
|
||||
f.deps.browserNavigation.update.mockImplementation(async () => {
|
||||
f.events.onErrorOccurred.fire({ ...f.main, error: "net::ERR_NAME_NOT_RESOLVED" });
|
||||
});
|
||||
const result = await handleNavigate(f.manager, { session_id: "test", url }, f.deps);
|
||||
expect(result).toMatchObject({ code: "cdp_failed", message: "net::ERR_NAME_NOT_RESOLVED" });
|
||||
f.expectCleanedUp();
|
||||
});
|
||||
});
|
||||
@@ -21,7 +21,7 @@ function fakeAgentWindow(ids: number[]) {
|
||||
return id;
|
||||
}),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -12,7 +12,7 @@ function fakeAgentWindow(ids: number[]) {
|
||||
return id;
|
||||
}),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,26 +1,8 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { SessionManager } from "@/session-manager/manager";
|
||||
import { handleSessionStop } from "../session";
|
||||
import type { ChromeTabsApi } from "../shared";
|
||||
import { type AgentOverlayResetApi, type ChromeWindowsApi, type TabMutationApi } from "../tabs";
|
||||
|
||||
/** Build a read-only query api over a FakeState. */
|
||||
function makeQuery(state: FakeState): ChromeTabsApi {
|
||||
return {
|
||||
get: vi.fn(async (id) => {
|
||||
const t = state.tabs.get(id);
|
||||
if (!t) throw new Error(`tab ${id} not found`);
|
||||
return t;
|
||||
}),
|
||||
query: vi.fn(async (q: chrome.tabs.QueryInfo) => {
|
||||
const w = q.windowId;
|
||||
return Array.from(state.tabs.values()).filter(
|
||||
(t) => typeof w !== "number" || t.windowId === w,
|
||||
);
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
function fakeAgentWindow(ids: number[]) {
|
||||
let i = 0;
|
||||
const create = vi.fn(async () => {
|
||||
@@ -29,7 +11,7 @@ function fakeAgentWindow(ids: number[]) {
|
||||
return id;
|
||||
});
|
||||
const remove = vi.fn(async () => {});
|
||||
const ensureActiveTab = vi.fn(async () => 0);
|
||||
const ensureActiveTab = vi.fn(async () => {});
|
||||
return { create, remove, ensureActiveTab };
|
||||
}
|
||||
|
||||
@@ -48,9 +30,7 @@ function makeApis(
|
||||
} {
|
||||
const tabs: TabMutationApi = {
|
||||
create: vi.fn(),
|
||||
remove: vi.fn(async (id: number) => {
|
||||
state.tabs.delete(id);
|
||||
}),
|
||||
remove: vi.fn(async () => {}),
|
||||
update: vi.fn(async (_id, _p) => undefined),
|
||||
get: vi.fn(async (id) => {
|
||||
const t = state.tabs.get(id);
|
||||
@@ -257,15 +237,11 @@ describe("handleSessionStop with auto-return", () => {
|
||||
moves: [],
|
||||
};
|
||||
const { tabs, windows } = makeApis(state, { moveThrowsFor: new Set([1]) });
|
||||
const cdp = {
|
||||
detachSession: vi.fn(async () => {}),
|
||||
releaseSessionTab: vi.fn(async () => {}),
|
||||
};
|
||||
|
||||
const res = await handleSessionStop(
|
||||
sm,
|
||||
{ session_id: "aa11" },
|
||||
{ cdp, tabManagement: { tabs, windows } },
|
||||
{ tabManagement: { tabs, windows } },
|
||||
);
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
expect(res.return_failures?.map((f) => f.tab_id)).toEqual([1]);
|
||||
@@ -273,8 +249,6 @@ describe("handleSessionStop with auto-return", () => {
|
||||
expect(sm.has("aa11")).toBe(true);
|
||||
expect(ctx.borrowedTabs.has(1)).toBe(true);
|
||||
expect(ctx.borrowedTabs.has(2)).toBe(false);
|
||||
expect(cdp.releaseSessionTab).toHaveBeenCalledExactlyOnceWith("aa11", 2);
|
||||
expect(cdp.detachSession).not.toHaveBeenCalled();
|
||||
expect(aw.remove).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
@@ -293,275 +267,4 @@ describe("handleSessionStop with auto-return", () => {
|
||||
await handleSessionStop(sm, { session_id: "aa11" }, { tabManagement: { tabs, windows } });
|
||||
expect(ctx.refStore.isEmpty()).toBe(true);
|
||||
});
|
||||
|
||||
it.each([
|
||||
"before teardown",
|
||||
"during its return",
|
||||
"after its return failed",
|
||||
"before its turn in the return loop",
|
||||
])("finishes stopping when a borrowed tab closes %s", async (timing) => {
|
||||
const aw = fakeAgentWindow([100]);
|
||||
const sm = new SessionManager({ agentWindow: aw });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.borrowedTabs.set(1, { tabId: 1, originalWindowId: 200, originalIndex: 0 });
|
||||
ctx.borrowedTabs.set(2, { tabId: 2, originalWindowId: 200, originalIndex: 1 });
|
||||
const state: FakeState = {
|
||||
tabs: new Map([
|
||||
[1, { id: 1, windowId: 100 } as chrome.tabs.Tab],
|
||||
[2, { id: 2, windowId: 100 } as chrome.tabs.Tab],
|
||||
]),
|
||||
windowsClosed: new Set(),
|
||||
moves: [],
|
||||
};
|
||||
const { tabs, windows } = makeApis(state);
|
||||
const cdp = { detachSession: vi.fn(async () => {}) };
|
||||
const closeTab = (tabId: number) => {
|
||||
state.tabs.delete(tabId);
|
||||
sm.forgetClosedTab(tabId);
|
||||
};
|
||||
const baseMove = tabs.move;
|
||||
tabs.move = vi.fn(async (tabId, props) => {
|
||||
if (tabId === 1 && timing === "during its return") closeTab(1);
|
||||
if (tabId === 1 && timing === "after its return failed") {
|
||||
throw new Error("simulated move failure");
|
||||
}
|
||||
if (tabId === 2 && timing === "after its return failed") closeTab(1);
|
||||
if (tabId === 1 && timing === "before its turn in the return loop") closeTab(2);
|
||||
if (!state.tabs.has(tabId)) throw new Error(`tab ${tabId} not found`);
|
||||
return baseMove(tabId, props);
|
||||
});
|
||||
if (timing === "before teardown") closeTab(1);
|
||||
|
||||
const res = await handleSessionStop(
|
||||
sm,
|
||||
{ session_id: "aa11" },
|
||||
{
|
||||
cdp,
|
||||
tabManagement: {
|
||||
tabs,
|
||||
windows,
|
||||
agentOverlayReset: { resetAgentOverlays: vi.fn(async () => {}) },
|
||||
},
|
||||
tabsQuery: makeQuery(state),
|
||||
},
|
||||
);
|
||||
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
const closedTabId = timing === "before its turn in the return loop" ? 2 : 1;
|
||||
expect(res.returned_tab_ids).toEqual([closedTabId === 1 ? 2 : 1]);
|
||||
expect(res.return_failures).toBeUndefined();
|
||||
expect(ctx.borrowedTabs.size).toBe(0);
|
||||
expect(sm.has("aa11")).toBe(false);
|
||||
expect(cdp.detachSession).toHaveBeenCalledWith("aa11");
|
||||
expect(aw.remove).toHaveBeenCalledWith(100);
|
||||
if (timing === "before teardown" || timing === "before its turn in the return loop") {
|
||||
expect(tabs.move).not.toHaveBeenCalledWith(closedTabId, expect.anything());
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("handleSessionStop window release (issue #57)", () => {
|
||||
const agentWindowId = 100;
|
||||
|
||||
it("releases the window when user-created tabs remain", async () => {
|
||||
const aw = fakeAgentWindow([agentWindowId]);
|
||||
// ensureActiveTab returns the home tab id (10). We override the fake to
|
||||
// return a known id.
|
||||
aw.ensureActiveTab = vi.fn(async () => 10);
|
||||
const sm = new SessionManager({ agentWindow: aw });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.agentCreatedTabs.add(11);
|
||||
ctx.agentCreatedTabs.add(12);
|
||||
// user tab 99 (created via Chrome UI, NOT in agentCreatedTabs)
|
||||
const state: FakeState = {
|
||||
tabs: new Map([
|
||||
[10, { id: 10, windowId: agentWindowId } as chrome.tabs.Tab],
|
||||
[11, { id: 11, windowId: agentWindowId } as chrome.tabs.Tab],
|
||||
[12, { id: 12, windowId: agentWindowId } as chrome.tabs.Tab],
|
||||
[99, { id: 99, windowId: agentWindowId } as chrome.tabs.Tab],
|
||||
]),
|
||||
windowsClosed: new Set(),
|
||||
moves: [],
|
||||
};
|
||||
const { tabs, windows } = makeApis(state);
|
||||
const query = makeQuery(state);
|
||||
|
||||
const res = await handleSessionStop(
|
||||
sm,
|
||||
{ session_id: "aa11" },
|
||||
{ tabManagement: { tabs, windows }, tabsQuery: query },
|
||||
);
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
expect(res.window_released).toBe(true);
|
||||
// agent tabs + home removed, user tab 99 kept
|
||||
expect((tabs.remove as ReturnType<typeof vi.fn>).mock.calls.map((c) => c[0]).sort()).toEqual([
|
||||
10, 11, 12,
|
||||
]);
|
||||
expect(state.tabs.has(99)).toBe(true);
|
||||
// window released (dropOnly), not closed
|
||||
expect(aw.remove).not.toHaveBeenCalled();
|
||||
expect(sm.has("aa11")).toBe(false);
|
||||
});
|
||||
|
||||
it("closes the window (not release) when no user tabs remain", async () => {
|
||||
const aw = fakeAgentWindow([agentWindowId]);
|
||||
aw.ensureActiveTab = vi.fn(async () => 10);
|
||||
const sm = new SessionManager({ agentWindow: aw });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.agentCreatedTabs.add(11);
|
||||
const state: FakeState = {
|
||||
tabs: new Map([
|
||||
[10, { id: 10, windowId: agentWindowId } as chrome.tabs.Tab],
|
||||
[11, { id: 11, windowId: agentWindowId } as chrome.tabs.Tab],
|
||||
]),
|
||||
windowsClosed: new Set(),
|
||||
moves: [],
|
||||
};
|
||||
const { tabs, windows } = makeApis(state);
|
||||
const query = makeQuery(state);
|
||||
|
||||
const res = await handleSessionStop(
|
||||
sm,
|
||||
{ session_id: "aa11" },
|
||||
{ tabManagement: { tabs, windows }, tabsQuery: query },
|
||||
);
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
expect(res.window_released).toBeFalsy();
|
||||
expect(aw.remove).toHaveBeenCalledWith(agentWindowId);
|
||||
expect(sm.has("aa11")).toBe(false);
|
||||
});
|
||||
|
||||
it("is non-fatal when an agent tab is already gone", async () => {
|
||||
const aw = fakeAgentWindow([agentWindowId]);
|
||||
aw.ensureActiveTab = vi.fn(async () => 10);
|
||||
const sm = new SessionManager({ agentWindow: aw });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.agentCreatedTabs.add(11); // no longer in state → remove throws
|
||||
const state: FakeState = {
|
||||
tabs: new Map([[10, { id: 10, windowId: agentWindowId } as chrome.tabs.Tab]]),
|
||||
windowsClosed: new Set(),
|
||||
moves: [],
|
||||
};
|
||||
const { tabs, windows } = makeApis(state);
|
||||
(tabs.remove as ReturnType<typeof vi.fn>).mockImplementation(async (id: number) => {
|
||||
if (id === 11) throw new Error("tab 11 already closed");
|
||||
state.tabs.delete(id); // home (10) removed normally
|
||||
});
|
||||
const query = makeQuery(state);
|
||||
|
||||
const res = await handleSessionStop(
|
||||
sm,
|
||||
{ session_id: "aa11" },
|
||||
{ tabManagement: { tabs, windows }, tabsQuery: query },
|
||||
);
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
// user tab 10? no — 10 is home, removed. window would be empty → close.
|
||||
expect(res.window_released).toBeFalsy();
|
||||
expect(aw.remove).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("degrades to close-window when tabsQuery is missing", async () => {
|
||||
const aw = fakeAgentWindow([agentWindowId]);
|
||||
aw.ensureActiveTab = vi.fn(async () => 10);
|
||||
const sm = new SessionManager({ agentWindow: aw });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.agentCreatedTabs.add(11);
|
||||
const state: FakeState = {
|
||||
tabs: new Map([
|
||||
[10, { id: 10, windowId: agentWindowId } as chrome.tabs.Tab],
|
||||
[11, { id: 11, windowId: agentWindowId } as chrome.tabs.Tab],
|
||||
]),
|
||||
windowsClosed: new Set(),
|
||||
moves: [],
|
||||
};
|
||||
const { tabs, windows } = makeApis(state);
|
||||
|
||||
const res = await handleSessionStop(
|
||||
sm,
|
||||
{ session_id: "aa11" },
|
||||
{ tabManagement: { tabs, windows } }, // no tabsQuery
|
||||
);
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
// No query available → conservatively close the window.
|
||||
expect(res.window_released).toBeFalsy();
|
||||
expect(aw.remove).toHaveBeenCalledWith(agentWindowId);
|
||||
});
|
||||
|
||||
it("degrades to close-window when tabsQuery.query throws", async () => {
|
||||
const aw = fakeAgentWindow([agentWindowId]);
|
||||
aw.ensureActiveTab = vi.fn(async () => 10);
|
||||
const sm = new SessionManager({ agentWindow: aw });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.agentCreatedTabs.add(11);
|
||||
const state: FakeState = {
|
||||
tabs: new Map([[11, { id: 11, windowId: agentWindowId } as chrome.tabs.Tab]]),
|
||||
windowsClosed: new Set(),
|
||||
moves: [],
|
||||
};
|
||||
const { tabs, windows } = makeApis(state);
|
||||
const query = makeQuery(state);
|
||||
(query.query as ReturnType<typeof vi.fn>).mockImplementation(async () => {
|
||||
throw new Error("query failed");
|
||||
});
|
||||
|
||||
const res = await handleSessionStop(
|
||||
sm,
|
||||
{ session_id: "aa11" },
|
||||
{ tabManagement: { tabs, windows }, tabsQuery: query },
|
||||
);
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
expect(res.window_released).toBeFalsy();
|
||||
expect(aw.remove).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("tracks agentCreatedTabs across tab_create / tab_close", async () => {
|
||||
const aw = fakeAgentWindow([agentWindowId]);
|
||||
aw.ensureActiveTab = vi.fn(async () => 10);
|
||||
const sm = new SessionManager({ agentWindow: aw });
|
||||
const ctx = await sm.start("aa11");
|
||||
// Simulate handleTabCreate adding, then handleTabClose removing.
|
||||
ctx.agentCreatedTabs.add(11);
|
||||
expect(ctx.agentCreatedTabs.has(11)).toBe(true);
|
||||
ctx.agentCreatedTabs.delete(11); // as handleTabClose does on success
|
||||
expect(ctx.agentCreatedTabs.has(11)).toBe(false);
|
||||
// Unknown id delete is a safe no-op (user-closed tab).
|
||||
expect(() => ctx.agentCreatedTabs.delete(999)).not.toThrow();
|
||||
});
|
||||
|
||||
it("closes the window (not release) when an agent tab fails to close", async () => {
|
||||
// Regression: a leaked agent tab (still in agentCreatedTabs, but
|
||||
// still present because Step 4 remove() threw) must NOT be mistaken
|
||||
// for a user tab. Otherwise the window would be released (dropOnly)
|
||||
// and the agent tab would leak (issue #57 regression).
|
||||
const aw = fakeAgentWindow([agentWindowId]);
|
||||
aw.ensureActiveTab = vi.fn(async () => 10);
|
||||
const sm = new SessionManager({ agentWindow: aw });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.agentCreatedTabs.add(11); // Step 4 remove() will throw for this tab
|
||||
const state: FakeState = {
|
||||
tabs: new Map([
|
||||
[10, { id: 10, windowId: agentWindowId } as chrome.tabs.Tab],
|
||||
[11, { id: 11, windowId: agentWindowId } as chrome.tabs.Tab],
|
||||
]),
|
||||
windowsClosed: new Set(),
|
||||
moves: [],
|
||||
};
|
||||
const { tabs, windows } = makeApis(state);
|
||||
(tabs.remove as ReturnType<typeof vi.fn>).mockImplementation(async (id: number) => {
|
||||
if (id === 11) throw new Error("tab 11 failed to close");
|
||||
state.tabs.delete(id); // home (10) removed normally; 11 lingers
|
||||
});
|
||||
const query = makeQuery(state);
|
||||
|
||||
const res = await handleSessionStop(
|
||||
sm,
|
||||
{ session_id: "aa11" },
|
||||
{ tabManagement: { tabs, windows }, tabsQuery: query },
|
||||
);
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
// The leaked agent tab must NOT keep the window open.
|
||||
expect(res.window_released).toBeFalsy();
|
||||
expect(aw.remove).toHaveBeenCalledWith(agentWindowId);
|
||||
expect(sm.has("aa11")).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -12,7 +12,7 @@ function fakeAgentWindow() {
|
||||
return {
|
||||
create: vi.fn(async () => 100),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -11,7 +11,7 @@ function fakeAgentWindow(ids: number[]) {
|
||||
return id;
|
||||
},
|
||||
remove: async () => {},
|
||||
ensureActiveTab: async () => 1,
|
||||
ensureActiveTab: async () => {},
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -1,7 +1,5 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { type CdpDebuggerApi, ChromiumCdp } from "@/browser-driver/chromium-cdp";
|
||||
import { SessionManager } from "@/session-manager/manager";
|
||||
import { handleHover } from "../interaction";
|
||||
import {
|
||||
type AgentOverlayResetApi,
|
||||
type ChromeWindowsApi,
|
||||
@@ -25,7 +23,7 @@ function fakeAgentWindow(ids: number[]) {
|
||||
return id;
|
||||
}),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -223,7 +221,6 @@ describe("handleTabCreate", () => {
|
||||
active: true,
|
||||
});
|
||||
expect(res).toMatchObject({ tab_id: 10, window_id: 100 });
|
||||
expect(sm.get("aa11")?.agentCreatedTabs.has(10)).toBe(true);
|
||||
});
|
||||
|
||||
it("forwards explicit url + index + active=false", async () => {
|
||||
@@ -259,41 +256,6 @@ describe("handleTabCreate", () => {
|
||||
const res = await handleTabCreate(sm, { session_id: "aa11", index: -1 }, { tabs: api });
|
||||
expect(res).toMatchObject({ code: "invalid_params" });
|
||||
});
|
||||
|
||||
it("does not claim any tab when chrome.tabs.create fails", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
await sm.start("aa11");
|
||||
const state: FakeTabState = { tabs: new Map(), nextTabId: 10, windowsClosed: new Set() };
|
||||
const { api, spies } = makeTabMutationApi(state);
|
||||
spies.create.mockRejectedValueOnce(new Error("chrome.tabs.create failed"));
|
||||
|
||||
const res = await handleTabCreate(sm, { session_id: "aa11" }, { tabs: api });
|
||||
expect(res).toMatchObject({ code: "protocol_error" });
|
||||
|
||||
expect(sm.get("aa11")?.agentCreatedTabs).toEqual(new Set([1]));
|
||||
});
|
||||
|
||||
it("claims the returned id even when a user tab appears during tab creation", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
const state: FakeTabState = { tabs: new Map(), nextTabId: 10, windowsClosed: new Set() };
|
||||
const { api, spies } = makeTabMutationApi(state);
|
||||
let resolveCreate!: (tab: chrome.tabs.Tab) => void;
|
||||
const createResult = new Promise<chrome.tabs.Tab>((resolve) => {
|
||||
resolveCreate = resolve;
|
||||
});
|
||||
spies.create.mockImplementationOnce(() => createResult);
|
||||
|
||||
const pending = handleTabCreate(sm, { session_id: "aa11" }, { tabs: api });
|
||||
// A user can create an unrelated same-window tab while Chrome is still
|
||||
// resolving the agent request. There is no pending slot for it to steal.
|
||||
expect(ctx.agentCreatedTabs.has(99)).toBe(false);
|
||||
resolveCreate({ id: 10, windowId: 100, active: true } as chrome.tabs.Tab);
|
||||
|
||||
await expect(pending).resolves.toMatchObject({ tab_id: 10 });
|
||||
expect(ctx.agentCreatedTabs.has(10)).toBe(true);
|
||||
expect(ctx.agentCreatedTabs.has(99)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("handleTabClose", () => {
|
||||
@@ -549,148 +511,6 @@ describe("handleTabBorrow", () => {
|
||||
});
|
||||
|
||||
describe("handleTabReturn", () => {
|
||||
it.each([
|
||||
"not_found",
|
||||
"cdp_failed",
|
||||
])("detaches after selector resolution fails with %s", async (code) => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.borrowedTabs.set(7, { tabId: 7, originalWindowId: 200, originalIndex: 4 });
|
||||
const state: FakeTabState = {
|
||||
tabs: new Map([[7, { id: 7, windowId: 100 } as chrome.tabs.Tab]]),
|
||||
nextTabId: 50,
|
||||
windowsClosed: new Set(),
|
||||
};
|
||||
const { api } = makeTabMutationApi(state);
|
||||
const { api: windows } = makeWindowsApi(state);
|
||||
const debuggerApi = {
|
||||
attach: vi.fn(async () => {}),
|
||||
detach: vi.fn(async () => {}),
|
||||
sendCommand: vi.fn(async (_target: unknown, method: string) => {
|
||||
if (method === "DOM.getDocument") return { root: { nodeId: 1 } };
|
||||
if (method === "DOM.querySelector") {
|
||||
if (code === "cdp_failed") throw new Error("selector query failed");
|
||||
return { nodeId: 0 };
|
||||
}
|
||||
return {};
|
||||
}),
|
||||
onEvent: { addListener: vi.fn(), removeListener: vi.fn() },
|
||||
onDetach: { addListener: vi.fn(), removeListener: vi.fn() },
|
||||
};
|
||||
const cdp = new ChromiumCdp(debuggerApi as unknown as CdpDebuggerApi);
|
||||
|
||||
const hover = await handleHover(
|
||||
sm,
|
||||
{ session_id: "aa11", tab_id: 7, selector: "#missing" },
|
||||
{
|
||||
cdp,
|
||||
tabsApi: { get: api.get, query: vi.fn(async () => []) },
|
||||
},
|
||||
);
|
||||
expect(hover).toMatchObject({ code });
|
||||
expect(cdp.isAttached(7)).toBe(true);
|
||||
|
||||
const returned = await handleTabReturn(
|
||||
sm,
|
||||
{ session_id: "aa11", tab_id: 7 },
|
||||
{
|
||||
tabs: api,
|
||||
windows,
|
||||
cdp,
|
||||
agentOverlayReset: { resetAgentOverlays: vi.fn(async () => {}) },
|
||||
},
|
||||
);
|
||||
expect(returned).toMatchObject({ returned_to_window_id: 200 });
|
||||
expect(ctx.borrowedTabs.has(7)).toBe(false);
|
||||
expect(cdp.isAttached(7)).toBe(false);
|
||||
expect(debuggerApi.detach).toHaveBeenCalledExactlyOnceWith({ tabId: 7 });
|
||||
cdp.dispose();
|
||||
});
|
||||
|
||||
it("honours cancellation while preparing a validated return", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.borrowedTabs.set(7, { tabId: 7, originalWindowId: 200, originalIndex: 4 });
|
||||
const state: FakeTabState = {
|
||||
tabs: new Map([[7, { id: 7, windowId: 100 } as chrome.tabs.Tab]]),
|
||||
nextTabId: 50,
|
||||
windowsClosed: new Set(),
|
||||
};
|
||||
const { api, spies } = makeTabMutationApi(state);
|
||||
const { api: windows } = makeWindowsApi(state);
|
||||
const controller = new AbortController();
|
||||
const beforeReturn = vi.fn(async () => {
|
||||
controller.abort();
|
||||
});
|
||||
const result = await handleTabReturn(
|
||||
sm,
|
||||
{ session_id: "aa11", tab_id: 7 },
|
||||
{
|
||||
tabs: api,
|
||||
windows,
|
||||
signal: controller.signal,
|
||||
beforeReturn,
|
||||
},
|
||||
);
|
||||
|
||||
expect(result).toMatchObject({ code: "cancelled" });
|
||||
expect(beforeReturn).toHaveBeenCalledExactlyOnceWith("aa11", 7);
|
||||
expect(spies.move).not.toHaveBeenCalled();
|
||||
expect(ctx.borrowedTabs.has(7)).toBe(true);
|
||||
});
|
||||
|
||||
it.each([
|
||||
"original",
|
||||
"fallback",
|
||||
"cleanup_failed",
|
||||
"failed",
|
||||
"cancelled",
|
||||
])("releases CDP only after a successful return (%s)", async (mode) => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.borrowedTabs.set(7, { tabId: 7, originalWindowId: 200, originalIndex: 4 });
|
||||
const state: FakeTabState = {
|
||||
tabs: new Map([[7, { id: 7, windowId: 100 } as chrome.tabs.Tab]]),
|
||||
nextTabId: 50,
|
||||
windowsClosed: new Set(),
|
||||
};
|
||||
const { api, spies } = makeTabMutationApi(state);
|
||||
const { api: windowsApi } = makeWindowsApi(state, { lastFocused: 500 });
|
||||
const success = mode !== "failed" && mode !== "cancelled";
|
||||
if (mode === "fallback") spies.move.mockRejectedValueOnce(new Error("original move failed"));
|
||||
if (mode === "failed") spies.move.mockRejectedValue(new Error("move failed"));
|
||||
let windowAtRelease: number | undefined;
|
||||
const cdp = {
|
||||
releaseSessionTab: vi.fn(async () => {
|
||||
windowAtRelease = state.tabs.get(7)?.windowId;
|
||||
if (mode === "cleanup_failed") throw new Error("debugger release failed");
|
||||
}),
|
||||
};
|
||||
const controller = new AbortController();
|
||||
if (mode === "cancelled") controller.abort();
|
||||
|
||||
const result = await handleTabReturn(
|
||||
sm,
|
||||
{ session_id: "aa11", tab_id: 7 },
|
||||
{
|
||||
tabs: api,
|
||||
windows: windowsApi,
|
||||
cdp,
|
||||
signal: controller.signal,
|
||||
agentOverlayReset: { resetAgentOverlays: vi.fn(async () => {}) },
|
||||
},
|
||||
);
|
||||
|
||||
expect("code" in result).toBe(!success);
|
||||
expect(ctx.borrowedTabs.has(7)).toBe(!success);
|
||||
expect(cdp.releaseSessionTab).toHaveBeenCalledTimes(success ? 1 : 0);
|
||||
if (success) {
|
||||
expect(cdp.releaseSessionTab).toHaveBeenCalledWith("aa11", 7);
|
||||
expect(windowAtRelease).toBe(mode === "fallback" ? 500 : 200);
|
||||
expect(spies.move).toHaveBeenCalledTimes(mode === "fallback" ? 2 : 1);
|
||||
}
|
||||
});
|
||||
|
||||
it("returns the tab to its original window/index and clears borrowedTabs", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
|
||||
@@ -12,7 +12,7 @@ function fakeAgentWindow(ids: number[]) {
|
||||
return id;
|
||||
}),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -11,7 +11,7 @@ function fakeAgentWindow(ids: number[]) {
|
||||
return id;
|
||||
});
|
||||
const remove = vi.fn(async () => {});
|
||||
const ensureActiveTab = vi.fn(async () => 1);
|
||||
const ensureActiveTab = vi.fn(async () => {});
|
||||
return { create, remove, ensureActiveTab };
|
||||
}
|
||||
|
||||
|
||||
@@ -1,110 +0,0 @@
|
||||
import type { RpcError, WaitUntil } from "@/transport/types";
|
||||
import { cdpError } from "./errors";
|
||||
|
||||
interface NavigationEvent {
|
||||
tabId: number;
|
||||
frameId: number;
|
||||
documentId?: string;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
interface NavigationEvents {
|
||||
addListener(listener: (details: NavigationEvent) => void): void;
|
||||
removeListener(listener: (details: NavigationEvent) => void): void;
|
||||
}
|
||||
|
||||
/** Normal browser navigation remains available when chrome.debugger is denied. */
|
||||
export interface BrowserNavigationApi {
|
||||
update(tabId: number, props: { url: string }): Promise<unknown>;
|
||||
reload(tabId: number, props: { bypassCache: boolean }): Promise<void>;
|
||||
onCommitted: NavigationEvents;
|
||||
onDOMContentLoaded: NavigationEvents;
|
||||
onCompleted: NavigationEvents;
|
||||
onErrorOccurred: NavigationEvents;
|
||||
}
|
||||
|
||||
export const chromeBrowserNavigationApi: BrowserNavigationApi = {
|
||||
update: (tabId, props) => chrome.tabs.update(tabId, props),
|
||||
reload: (tabId, props) => chrome.tabs.reload(tabId, props),
|
||||
get onCommitted() {
|
||||
return chrome.webNavigation.onCommitted;
|
||||
},
|
||||
get onDOMContentLoaded() {
|
||||
return chrome.webNavigation.onDOMContentLoaded;
|
||||
},
|
||||
get onCompleted() {
|
||||
return chrome.webNavigation.onCompleted;
|
||||
},
|
||||
get onErrorOccurred() {
|
||||
return chrome.webNavigation.onErrorOccurred;
|
||||
},
|
||||
};
|
||||
|
||||
export type NavigationOutcome =
|
||||
| { reached: "match" | "timeout" | "cancelled"; lastLifecycle?: string }
|
||||
| { reached: "failed"; error: RpcError };
|
||||
|
||||
/** Subscribe before dispatch and ignore the departing document's load events. */
|
||||
export async function navigateWithBrowserApi(
|
||||
api: BrowserNavigationApi,
|
||||
tabId: number,
|
||||
action: () => Promise<unknown>,
|
||||
waitUntil: Exclude<WaitUntil, "networkidle">,
|
||||
timeoutMs: number,
|
||||
signal?: AbortSignal,
|
||||
): Promise<NavigationOutcome> {
|
||||
if (signal?.aborted) return { reached: "cancelled" };
|
||||
if (timeoutMs <= 0) return { reached: "timeout" };
|
||||
let committed = false;
|
||||
let documentId: string | undefined;
|
||||
let lastLifecycle: string | undefined;
|
||||
let finish!: (outcome: NavigationOutcome) => void;
|
||||
const outcome = new Promise<NavigationOutcome>((resolve) => {
|
||||
finish = resolve;
|
||||
});
|
||||
const isMainFrame = (details: NavigationEvent) =>
|
||||
details.tabId === tabId && details.frameId === 0;
|
||||
const onCommitted = (details: NavigationEvent) => {
|
||||
if (!isMainFrame(details)) return;
|
||||
committed = true;
|
||||
documentId = details.documentId;
|
||||
lastLifecycle = "commit";
|
||||
if (waitUntil === "commit") finish({ reached: "match", lastLifecycle });
|
||||
};
|
||||
const onDOMContentLoaded = (details: NavigationEvent) => {
|
||||
if (!isMainFrame(details) || !committed || (documentId && details.documentId !== documentId))
|
||||
return;
|
||||
lastLifecycle = "DOMContentLoaded";
|
||||
if (waitUntil === "domcontentloaded") finish({ reached: "match", lastLifecycle });
|
||||
};
|
||||
const onCompleted = (details: NavigationEvent) => {
|
||||
if (!isMainFrame(details) || !committed || (documentId && details.documentId !== documentId))
|
||||
return;
|
||||
finish({ reached: "match", lastLifecycle: "load" });
|
||||
};
|
||||
const onError = (details: NavigationEvent) => {
|
||||
if (!isMainFrame(details)) return;
|
||||
finish({ reached: "failed", error: cdpError(details.error ?? "browser navigation failed") });
|
||||
};
|
||||
const onAbort = () => finish({ reached: "cancelled", lastLifecycle });
|
||||
const subscriptions = [
|
||||
[api.onCommitted, onCommitted],
|
||||
[api.onDOMContentLoaded, onDOMContentLoaded],
|
||||
[api.onCompleted, onCompleted],
|
||||
[api.onErrorOccurred, onError],
|
||||
] as const;
|
||||
for (const [event, listener] of subscriptions) event.addListener(listener);
|
||||
signal?.addEventListener("abort", onAbort, { once: true });
|
||||
const timer = setTimeout(() => finish({ reached: "timeout", lastLifecycle }), timeoutMs);
|
||||
try {
|
||||
if (signal?.aborted) return { reached: "cancelled" };
|
||||
await action();
|
||||
return await outcome;
|
||||
} catch (error) {
|
||||
return { reached: "failed", error: cdpError(error) };
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
signal?.removeEventListener("abort", onAbort);
|
||||
for (const [event, listener] of subscriptions) event.removeListener(listener);
|
||||
}
|
||||
}
|
||||
@@ -36,7 +36,6 @@ import { isRequestFrame } from "@/transport/types";
|
||||
import { handleConsole } from "./console";
|
||||
import { handleDownload } from "./download";
|
||||
import { type EmulateCdpRunner, handleEmulate } from "./emulate";
|
||||
import { classifyCdpError } from "./errors";
|
||||
import { handleEvaluate } from "./evaluate";
|
||||
import { handleRequestHelp } from "./human-loop";
|
||||
import { handleClick, handleFill, handleHover, handlePress, handleSelect } from "./interaction";
|
||||
@@ -70,7 +69,6 @@ import {
|
||||
import { chromeTabsApi, lookupSession, resolveTargetTab } from "./shared";
|
||||
import {
|
||||
type BorrowConfirmationApprover,
|
||||
chromeTabMutationApi,
|
||||
handleTabBorrow,
|
||||
handleTabClose,
|
||||
handleTabCreate,
|
||||
@@ -120,8 +118,6 @@ export interface DispatcherDeps {
|
||||
onSessionsChanged?: () => void;
|
||||
/** Invoked before a tool that dispatches page input or mutates browser state is forwarded. */
|
||||
onBrowserControlResumed?: (sessionId: string) => void;
|
||||
/** Invoked after a tab is explicitly claimed so its overlay can be refreshed immediately. */
|
||||
onAgentTabClaimed?: (tabId: number, windowId: number) => void;
|
||||
/** User approval for `tool.tab_borrow` (overlay in content script). */
|
||||
approveBorrow?: BorrowConfirmationApprover;
|
||||
/** i18n notification copy for `tool.request_help` (resolved per-call). */
|
||||
@@ -151,7 +147,6 @@ export class ToolDispatcher {
|
||||
private readonly recording?: RecordRuntimeDeps;
|
||||
private readonly onSessionsChanged?: () => void;
|
||||
private readonly onBrowserControlResumed?: (sessionId: string) => void;
|
||||
private readonly onAgentTabClaimed?: (tabId: number, windowId: number) => void;
|
||||
private readonly approveBorrow?: BorrowConfirmationApprover;
|
||||
private readonly helpNotificationCopy?: () => { title: string; body: string };
|
||||
private subscription: { dispose(): void } | null = null;
|
||||
@@ -172,7 +167,6 @@ export class ToolDispatcher {
|
||||
this.recording = deps.recording;
|
||||
this.onSessionsChanged = deps.onSessionsChanged;
|
||||
this.onBrowserControlResumed = deps.onBrowserControlResumed;
|
||||
this.onAgentTabClaimed = deps.onAgentTabClaimed;
|
||||
this.approveBorrow = deps.approveBorrow;
|
||||
this.helpNotificationCopy = deps.helpNotificationCopy;
|
||||
}
|
||||
@@ -240,7 +234,7 @@ export class ToolDispatcher {
|
||||
if (sessionId) this.onBrowserControlResumed?.(sessionId);
|
||||
const result = await this.invoke(req, ac.signal);
|
||||
if (isRpcError(result)) {
|
||||
body = { id: req.id, error: classifyCdpError(result) };
|
||||
body = { id: req.id, error: result };
|
||||
} else {
|
||||
body = { id: req.id, result };
|
||||
if (req.method === "tool.session_start") {
|
||||
@@ -308,25 +302,13 @@ export class ToolDispatcher {
|
||||
await this.releaseHoverLatch((req.params as SessionStopParams).session_id);
|
||||
return handleSessionStop(this.sessions, req.params as SessionStopParams, {
|
||||
cdp: this.cdp,
|
||||
// Must be wired in production: the agent-tab cleanup and the
|
||||
// window-release decision (issue #57) read these deps directly
|
||||
// and silently no-op when they are absent.
|
||||
tabManagement: { tabs: chromeTabMutationApi },
|
||||
tabsQuery: chromeTabsApi,
|
||||
signal,
|
||||
});
|
||||
}
|
||||
case "tool.tab_list":
|
||||
return handleTabList(this.sessions, req.params as TabListParams, chromeTabsApi, signal);
|
||||
case "tool.tab_create": {
|
||||
const result = await handleTabCreate(this.sessions, req.params as TabCreateParams, {
|
||||
signal,
|
||||
});
|
||||
if (!isRpcError(result)) {
|
||||
this.onAgentTabClaimed?.(result.tab_id, result.window_id);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
case "tool.tab_create":
|
||||
return handleTabCreate(this.sessions, req.params as TabCreateParams, { signal });
|
||||
case "tool.tab_close":
|
||||
return this.withHoverReleaseForRequest(
|
||||
req.params as TabCloseParams,
|
||||
@@ -335,22 +317,13 @@ export class ToolDispatcher {
|
||||
);
|
||||
case "tool.tab_select":
|
||||
return handleTabSelect(this.sessions, req.params as TabSelectParams, { signal });
|
||||
case "tool.tab_borrow": {
|
||||
const result = await handleTabBorrow(this.sessions, req.params as TabBorrowParams, {
|
||||
case "tool.tab_borrow":
|
||||
return handleTabBorrow(this.sessions, req.params as TabBorrowParams, {
|
||||
signal,
|
||||
approveBorrow: this.approveBorrow,
|
||||
});
|
||||
if (!isRpcError(result)) {
|
||||
this.onAgentTabClaimed?.(result.tab_id, result.agent_window_id);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
case "tool.tab_return":
|
||||
return handleTabReturn(this.sessions, req.params as TabReturnParams, {
|
||||
signal,
|
||||
cdp: this.cdp,
|
||||
beforeReturn: (sessionId, tabId) => this.releaseHoverLatch(sessionId, tabId),
|
||||
});
|
||||
return handleTabReturn(this.sessions, req.params as TabReturnParams, { signal });
|
||||
case "tool.window_resize":
|
||||
return handleWindowResize(
|
||||
this.sessions,
|
||||
|
||||
@@ -9,6 +9,8 @@ import { type CdpRunner, isRpcError } from "./shared";
|
||||
|
||||
const CORRELATION_GRACE_MS = 750;
|
||||
const UNIQUE_SETTLE_MS = 50;
|
||||
const CLEANUP_RECONCILE_MS = 1_000;
|
||||
const CLEANUP_POLL_MS = 25;
|
||||
const SIZE_POLL_MS = 250;
|
||||
|
||||
type DeterminingFilenameListener = (
|
||||
@@ -110,26 +112,35 @@ function captureError(
|
||||
|
||||
async function cleanupClaimedDownload(downloads: DownloadsApi, downloadId: number): Promise<void> {
|
||||
const lookup = async () => (await downloads.search({ id: downloadId }))[0];
|
||||
const item = await lookup();
|
||||
let item = await lookup();
|
||||
if (!item || item.state === "interrupted") return;
|
||||
if (item.state === "complete") {
|
||||
await downloads.removeFile(downloadId);
|
||||
return;
|
||||
}
|
||||
|
||||
let cancelError: unknown;
|
||||
try {
|
||||
await downloads.cancel(downloadId);
|
||||
} catch (cancelError) {
|
||||
// Completion can win the race after the lookup but before cancellation.
|
||||
// Reconcile against Chrome's authoritative state before declaring cleanup
|
||||
// failed so every terminal state has one explicit cleanup path.
|
||||
const reconciled = await lookup();
|
||||
if (!reconciled || reconciled.state === "interrupted") return;
|
||||
if (reconciled.state === "complete") {
|
||||
} catch (error) {
|
||||
cancelError = error;
|
||||
}
|
||||
|
||||
const deadline = Date.now() + CLEANUP_RECONCILE_MS;
|
||||
while (true) {
|
||||
item = await lookup();
|
||||
if (!item || item.state === "interrupted") return;
|
||||
if (item.state === "complete") {
|
||||
await downloads.removeFile(downloadId);
|
||||
return;
|
||||
}
|
||||
throw cancelError;
|
||||
if (Date.now() >= deadline) {
|
||||
if (cancelError instanceof Error) throw cancelError;
|
||||
throw new Error("download remained in progress after cancellation");
|
||||
}
|
||||
await new Promise((resolve) =>
|
||||
setTimeout(resolve, Math.min(CLEANUP_POLL_MS, Math.max(1, deadline - Date.now()))),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -13,34 +13,6 @@ import type {
|
||||
|
||||
export type { RpcErrorData, RpcErrorReason };
|
||||
|
||||
/** Chrome checks the whole tab's frame tree, including other extensions' frames. */
|
||||
export function isCdpExtensionAccessDenied(error: unknown): boolean {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
return message.includes("Cannot access a chrome-extension:// URL of different extension");
|
||||
}
|
||||
|
||||
/** Preserve more specific reasons, especially file-transfer outcome/cleanup details. */
|
||||
export function classifyCdpError(error: RpcError): RpcError {
|
||||
if (
|
||||
error.code !== "cdp_failed" ||
|
||||
error.data?.reason ||
|
||||
!isCdpExtensionAccessDenied(error.message)
|
||||
) {
|
||||
return error;
|
||||
}
|
||||
return {
|
||||
...error,
|
||||
data: { ...error.data, reason: "cdp_extension_access_denied" },
|
||||
};
|
||||
}
|
||||
|
||||
export function cdpError(error: unknown): RpcError {
|
||||
return classifyCdpError({
|
||||
code: "cdp_failed",
|
||||
message: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
}
|
||||
|
||||
export interface TransferErrorOptions {
|
||||
effectState: TransferEffectState;
|
||||
phase: string;
|
||||
|
||||
@@ -1,319 +0,0 @@
|
||||
// One native file-drop transaction bound to an explicit page target. CDP drag
|
||||
// events are sent to the target that owns the node: root targets use top-level
|
||||
// coordinates, while OOPIF targets require coordinates local to their session.
|
||||
|
||||
import type { RpcError, TransferEffectState } from "@/transport/types";
|
||||
import { transferError } from "./errors";
|
||||
import { resolveNodeGeometry } from "./frame-geometry";
|
||||
import type { ResolvedActionTarget } from "./interaction";
|
||||
import { type CdpRunner, isRpcError, sendToCdpTarget } from "./shared";
|
||||
import { BoundedWaitError, waitBounded } from "./transfer-transaction";
|
||||
|
||||
const CLEANUP_TIMEOUT_MS = 1_000;
|
||||
|
||||
type FileDropPhase = "resolve_target" | "drag_enter" | "drag_over" | "drop" | "cleanup";
|
||||
|
||||
interface RuntimeReply {
|
||||
result?: { value?: unknown; objectId?: string };
|
||||
exceptionDetails?: { text?: string; exception?: { description?: string } };
|
||||
}
|
||||
|
||||
interface DragData {
|
||||
items: [];
|
||||
files: string[];
|
||||
dragOperationsMask: number;
|
||||
}
|
||||
|
||||
export interface FileDropTransactionOptions {
|
||||
cdp: CdpRunner;
|
||||
actionTarget: ResolvedActionTarget;
|
||||
files: string[];
|
||||
timeoutMs: number;
|
||||
signal?: AbortSignal;
|
||||
withOverlayHidden: <T>(operation: () => Promise<T>) => Promise<T>;
|
||||
}
|
||||
|
||||
export interface FileDropTransactionResult {
|
||||
x: number;
|
||||
y: number;
|
||||
}
|
||||
|
||||
function failure(
|
||||
code: RpcError["code"],
|
||||
reason: "upload_mechanism_unsupported" | "file_drop_target_unavailable" | "file_drop_failed",
|
||||
message: string,
|
||||
effectState: TransferEffectState,
|
||||
phase: FileDropPhase,
|
||||
): RpcError {
|
||||
const error = transferError(code, reason, message, { effectState, phase });
|
||||
error.data = { ...error.data, mechanism: "drop" };
|
||||
return error;
|
||||
}
|
||||
|
||||
function enrichFailure(
|
||||
error: RpcError,
|
||||
effectState: TransferEffectState,
|
||||
phase: FileDropPhase,
|
||||
): RpcError {
|
||||
return {
|
||||
...error,
|
||||
data: {
|
||||
...error.data,
|
||||
mechanism: "drop",
|
||||
effect_state: effectState,
|
||||
phase,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function messageOf(error: unknown): string {
|
||||
return error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
|
||||
function methodUnavailable(error: unknown): boolean {
|
||||
return /(?:method|command).*(?:not found|wasn't found|unsupported)|wasn't found|-32601/i.test(
|
||||
messageOf(error),
|
||||
);
|
||||
}
|
||||
|
||||
function runtimeError(reply: RuntimeReply, fallback: string): Error | null {
|
||||
if (!reply.exceptionDetails) return null;
|
||||
return new Error(
|
||||
reply.exceptionDetails.exception?.description ?? reply.exceptionDetails.text ?? fallback,
|
||||
);
|
||||
}
|
||||
|
||||
async function pointBelongsToTarget(
|
||||
options: FileDropTransactionOptions,
|
||||
x: number,
|
||||
y: number,
|
||||
objectGroup: string,
|
||||
): Promise<boolean> {
|
||||
const target = options.actionTarget.cdpTarget;
|
||||
const hit = await sendToCdpTarget<{ backendNodeId?: number; frameId?: string }>(
|
||||
options.cdp,
|
||||
target,
|
||||
"DOM.getNodeForLocation",
|
||||
{ x: Math.round(x), y: Math.round(y), includeUserAgentShadowDOM: true },
|
||||
);
|
||||
if (typeof hit.backendNodeId !== "number") return false;
|
||||
if (options.actionTarget.frameId && hit.frameId !== options.actionTarget.frameId) return false;
|
||||
|
||||
const [resolvedTarget, resolvedHit] = await Promise.all([
|
||||
sendToCdpTarget<{ object?: { objectId?: string } }>(options.cdp, target, "DOM.resolveNode", {
|
||||
backendNodeId: options.actionTarget.backendNodeId,
|
||||
objectGroup,
|
||||
}),
|
||||
sendToCdpTarget<{ object?: { objectId?: string } }>(options.cdp, target, "DOM.resolveNode", {
|
||||
backendNodeId: hit.backendNodeId,
|
||||
objectGroup,
|
||||
}),
|
||||
]);
|
||||
const targetObjectId = resolvedTarget.object?.objectId;
|
||||
const hitObjectId = resolvedHit.object?.objectId;
|
||||
if (!targetObjectId || !hitObjectId) return false;
|
||||
|
||||
const reply = await sendToCdpTarget<RuntimeReply>(options.cdp, target, "Runtime.callFunctionOn", {
|
||||
objectId: targetObjectId,
|
||||
functionDeclaration: `function(hit) {
|
||||
for (let node = hit; node; ) {
|
||||
if (node === this) return true;
|
||||
if (this.contains?.(node)) return true;
|
||||
const root = node.getRootNode?.();
|
||||
node = node.parentNode || (root && root !== node ? root.host : null);
|
||||
}
|
||||
return false;
|
||||
}`,
|
||||
arguments: [{ objectId: hitObjectId }],
|
||||
returnByValue: true,
|
||||
});
|
||||
const error = runtimeError(reply, "failed to verify file-drop target");
|
||||
if (error) throw error;
|
||||
return reply.result?.value === true;
|
||||
}
|
||||
|
||||
export async function uploadThroughFileDrop(
|
||||
options: FileDropTransactionOptions,
|
||||
): Promise<FileDropTransactionResult | RpcError> {
|
||||
const deadline = Date.now() + options.timeoutMs;
|
||||
const target = options.actionTarget.cdpTarget;
|
||||
const objectGroup = `bsk-file-drop-${crypto.randomUUID()}`;
|
||||
const dragData: DragData = {
|
||||
items: [],
|
||||
files: options.files,
|
||||
dragOperationsMask: 1,
|
||||
};
|
||||
let dragStarted = false;
|
||||
let dropSent = false;
|
||||
let dragPoint = { x: 0, y: 0 };
|
||||
let cleanupFailed = false;
|
||||
let outcome: FileDropTransactionResult | RpcError = failure(
|
||||
"protocol_error",
|
||||
"file_drop_failed",
|
||||
"file-drop transaction ended without an outcome",
|
||||
"none",
|
||||
"cleanup",
|
||||
);
|
||||
|
||||
try {
|
||||
const geometry = await waitBounded(
|
||||
resolveNodeGeometry(
|
||||
options.cdp,
|
||||
options.actionTarget.tab.tabId,
|
||||
{
|
||||
target,
|
||||
backendNodeId: options.actionTarget.backendNodeId,
|
||||
...(options.actionTarget.frameId ? { frameId: options.actionTarget.frameId } : {}),
|
||||
},
|
||||
{ scrollIntoView: true },
|
||||
),
|
||||
deadline,
|
||||
options.signal,
|
||||
"resolving file-drop target timed out",
|
||||
);
|
||||
if (isRpcError(geometry)) {
|
||||
outcome = enrichFailure(geometry, "none", "resolve_target");
|
||||
return outcome;
|
||||
}
|
||||
const point = geometry.targetActionPoint;
|
||||
dragPoint = point;
|
||||
|
||||
outcome = await options.withOverlayHidden(async () => {
|
||||
try {
|
||||
try {
|
||||
const belongs = await waitBounded(
|
||||
pointBelongsToTarget(options, point.x, point.y, objectGroup),
|
||||
deadline,
|
||||
options.signal,
|
||||
"verifying file-drop target timed out",
|
||||
);
|
||||
if (!belongs) {
|
||||
return failure(
|
||||
"permission_denied",
|
||||
"file_drop_target_unavailable",
|
||||
"file-drop target is not the topmost element at its action point",
|
||||
"none",
|
||||
"resolve_target",
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
return failure(
|
||||
error instanceof BoundedWaitError ? "timeout" : "cdp_failed",
|
||||
"file_drop_target_unavailable",
|
||||
messageOf(error),
|
||||
"none",
|
||||
"resolve_target",
|
||||
);
|
||||
}
|
||||
|
||||
for (const [type, phase] of [
|
||||
["dragEnter", "drag_enter"],
|
||||
["dragOver", "drag_over"],
|
||||
] as const) {
|
||||
try {
|
||||
if (type === "dragEnter") dragStarted = true;
|
||||
await waitBounded(
|
||||
sendToCdpTarget(options.cdp, target, "Input.dispatchDragEvent", {
|
||||
type,
|
||||
x: point.x,
|
||||
y: point.y,
|
||||
data: dragData,
|
||||
}),
|
||||
deadline,
|
||||
options.signal,
|
||||
`${type} timed out`,
|
||||
);
|
||||
} catch (error) {
|
||||
const unsupported = methodUnavailable(error);
|
||||
if (type === "dragEnter" && unsupported) dragStarted = false;
|
||||
return failure(
|
||||
unsupported
|
||||
? "unsupported"
|
||||
: error instanceof BoundedWaitError
|
||||
? "timeout"
|
||||
: "cdp_failed",
|
||||
unsupported ? "upload_mechanism_unsupported" : "file_drop_failed",
|
||||
messageOf(error),
|
||||
"none",
|
||||
phase,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
dropSent = true;
|
||||
await waitBounded(
|
||||
sendToCdpTarget(options.cdp, target, "Input.dispatchDragEvent", {
|
||||
type: "drop",
|
||||
x: point.x,
|
||||
y: point.y,
|
||||
data: dragData,
|
||||
}),
|
||||
deadline,
|
||||
options.signal,
|
||||
"file drop timed out",
|
||||
);
|
||||
return { x: point.x, y: point.y };
|
||||
} catch (error) {
|
||||
outcome = failure(
|
||||
error instanceof BoundedWaitError ? "timeout" : "cdp_failed",
|
||||
"file_drop_failed",
|
||||
messageOf(error),
|
||||
"unknown",
|
||||
"drop",
|
||||
);
|
||||
return outcome;
|
||||
}
|
||||
} finally {
|
||||
if (dragStarted && !dropSent) {
|
||||
try {
|
||||
await waitBounded(
|
||||
sendToCdpTarget(options.cdp, target, "Input.dispatchDragEvent", {
|
||||
type: "dragCancel",
|
||||
x: dragPoint.x,
|
||||
y: dragPoint.y,
|
||||
data: dragData,
|
||||
}),
|
||||
Date.now() + CLEANUP_TIMEOUT_MS,
|
||||
undefined,
|
||||
"cancelling file drag timed out",
|
||||
);
|
||||
} catch {
|
||||
cleanupFailed = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
return outcome;
|
||||
} catch (error) {
|
||||
outcome = failure(
|
||||
error instanceof BoundedWaitError ? "timeout" : "cdp_failed",
|
||||
"file_drop_target_unavailable",
|
||||
messageOf(error),
|
||||
"none",
|
||||
"resolve_target",
|
||||
);
|
||||
return outcome;
|
||||
} finally {
|
||||
try {
|
||||
await waitBounded(
|
||||
sendToCdpTarget(options.cdp, target, "Runtime.releaseObjectGroup", { objectGroup }),
|
||||
Date.now() + CLEANUP_TIMEOUT_MS,
|
||||
undefined,
|
||||
"releasing file-drop object group timed out",
|
||||
);
|
||||
} catch {
|
||||
cleanupFailed = true;
|
||||
}
|
||||
|
||||
const effect = isRpcError(outcome)
|
||||
? (outcome.data?.effect_state as TransferEffectState | undefined)
|
||||
: "committed";
|
||||
if (effect === "unknown" || cleanupFailed) {
|
||||
await options.cdp.detach?.(target.tabId);
|
||||
}
|
||||
if (cleanupFailed && isRpcError(outcome)) {
|
||||
outcome.data = { ...outcome.data, cleanup_state: "failed" };
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -7,9 +7,9 @@ import type { ClickResult, RpcError, TransferEffectState } from "@/transport/typ
|
||||
import { transferError } from "./errors";
|
||||
import type { ResolvedActionTarget } from "./interaction";
|
||||
import { type CdpRunner, isRpcError, sendToCdpTarget } from "./shared";
|
||||
import { BoundedWaitError, remainingMs, waitBounded } from "./transfer-transaction";
|
||||
|
||||
const CLEANUP_TIMEOUT_MS = 1_000;
|
||||
const DETACH_TIMEOUT_MS = 500;
|
||||
const ACTIVATION_GRACE_MS = 1_000;
|
||||
const PROBE_INTERVAL_MS = 20;
|
||||
|
||||
@@ -46,6 +46,51 @@ export interface FileInputTransactionResult {
|
||||
multiple: boolean;
|
||||
}
|
||||
|
||||
class BoundedWaitError extends Error {
|
||||
constructor(
|
||||
readonly kind: "timeout" | "aborted",
|
||||
message: string,
|
||||
) {
|
||||
super(message);
|
||||
}
|
||||
}
|
||||
|
||||
function remainingMs(deadline: number): number {
|
||||
return Math.max(0, deadline - Date.now());
|
||||
}
|
||||
|
||||
async function waitBounded<T>(
|
||||
operation: () => Promise<T>,
|
||||
deadline: number,
|
||||
signal: AbortSignal | undefined,
|
||||
timeoutMessage: string,
|
||||
): Promise<T> {
|
||||
const remaining = remainingMs(deadline);
|
||||
if (signal?.aborted) throw new BoundedWaitError("aborted", "upload transaction aborted");
|
||||
if (remaining === 0) throw new BoundedWaitError("timeout", timeoutMessage);
|
||||
|
||||
let timer: ReturnType<typeof setTimeout> | undefined;
|
||||
let onAbort: (() => void) | undefined;
|
||||
const boundary = new Promise<never>((_resolve, reject) => {
|
||||
timer = setTimeout(() => reject(new BoundedWaitError("timeout", timeoutMessage)), remaining);
|
||||
if (signal) {
|
||||
onAbort = () => reject(new BoundedWaitError("aborted", "upload transaction aborted"));
|
||||
signal.addEventListener("abort", onAbort, { once: true });
|
||||
}
|
||||
});
|
||||
try {
|
||||
const work = Promise.resolve().then(() => {
|
||||
if (signal?.aborted) throw new BoundedWaitError("aborted", "upload transaction aborted");
|
||||
if (remainingMs(deadline) === 0) throw new BoundedWaitError("timeout", timeoutMessage);
|
||||
return operation();
|
||||
});
|
||||
return await Promise.race([work, boundary]);
|
||||
} finally {
|
||||
if (timer) clearTimeout(timer);
|
||||
if (signal && onAbort) signal.removeEventListener("abort", onAbort);
|
||||
}
|
||||
}
|
||||
|
||||
function runtimeError(reply: RuntimeReply, fallback: string): Error | null {
|
||||
if (!reply.exceptionDetails) return null;
|
||||
return new Error(
|
||||
@@ -83,13 +128,18 @@ function sameTarget(
|
||||
|
||||
async function delay(ms: number, signal?: AbortSignal): Promise<void> {
|
||||
await waitBounded(
|
||||
new Promise((resolve) => setTimeout(resolve, ms)),
|
||||
() => new Promise((resolve) => setTimeout(resolve, ms)),
|
||||
Date.now() + ms + 1,
|
||||
signal,
|
||||
"upload activation probe timed out",
|
||||
);
|
||||
}
|
||||
|
||||
function waitErrorCode(error: unknown): RpcError["code"] {
|
||||
if (!(error instanceof BoundedWaitError)) return "cdp_failed";
|
||||
return error.kind === "aborted" ? "cancelled" : "timeout";
|
||||
}
|
||||
|
||||
async function callOnTrigger<T = RuntimeReply>(
|
||||
options: FileInputTransactionOptions,
|
||||
objectId: string,
|
||||
@@ -114,8 +164,8 @@ export async function uploadThroughActivatedFileInput(
|
||||
const objectGroup = `bsk-upload-${crypto.randomUUID()}`;
|
||||
const stateKey = `__bskUpload_${crypto.randomUUID().replaceAll("-", "")}`;
|
||||
const chooserEvents: ChooserEvent[] = [];
|
||||
let interceptionArmed = false;
|
||||
let probeArmed = false;
|
||||
let interceptionState: "not_started" | "possibly_active" | "active" = "not_started";
|
||||
let probeState: "not_started" | "possibly_active" | "active" = "not_started";
|
||||
let triggerObjectId: string | undefined;
|
||||
let outcome: FileInputTransactionResult | RpcError = transferFailure(
|
||||
"protocol_error",
|
||||
@@ -133,18 +183,20 @@ export async function uploadThroughActivatedFileInput(
|
||||
|
||||
try {
|
||||
try {
|
||||
interceptionState = "possibly_active";
|
||||
await waitBounded(
|
||||
sendToCdpTarget(options.cdp, target, "Page.setInterceptFileChooserDialog", {
|
||||
enabled: true,
|
||||
}),
|
||||
() =>
|
||||
sendToCdpTarget(options.cdp, target, "Page.setInterceptFileChooserDialog", {
|
||||
enabled: true,
|
||||
}),
|
||||
deadline,
|
||||
options.signal,
|
||||
"arming native file chooser interception timed out",
|
||||
);
|
||||
interceptionArmed = true;
|
||||
interceptionState = "active";
|
||||
} catch (err) {
|
||||
outcome = transferFailure(
|
||||
err instanceof BoundedWaitError ? "timeout" : "cdp_failed",
|
||||
waitErrorCode(err),
|
||||
"file_input_probe_failed",
|
||||
err instanceof Error ? err.message : String(err),
|
||||
"none",
|
||||
@@ -155,15 +207,16 @@ export async function uploadThroughActivatedFileInput(
|
||||
|
||||
try {
|
||||
const resolved = await waitBounded(
|
||||
sendToCdpTarget<{ object?: { objectId?: string } }>(
|
||||
options.cdp,
|
||||
target,
|
||||
"DOM.resolveNode",
|
||||
{
|
||||
backendNodeId: options.actionTarget.backendNodeId,
|
||||
objectGroup,
|
||||
},
|
||||
),
|
||||
() =>
|
||||
sendToCdpTarget<{ object?: { objectId?: string } }>(
|
||||
options.cdp,
|
||||
target,
|
||||
"DOM.resolveNode",
|
||||
{
|
||||
backendNodeId: options.actionTarget.backendNodeId,
|
||||
objectGroup,
|
||||
},
|
||||
),
|
||||
deadline,
|
||||
options.signal,
|
||||
"resolving upload trigger timed out",
|
||||
@@ -171,11 +224,13 @@ export async function uploadThroughActivatedFileInput(
|
||||
triggerObjectId = resolved.object?.objectId;
|
||||
if (!triggerObjectId) throw new Error("DOM.resolveNode returned no trigger objectId");
|
||||
|
||||
probeState = "possibly_active";
|
||||
const armed = await waitBounded(
|
||||
callOnTrigger<RuntimeReply>(
|
||||
options,
|
||||
triggerObjectId,
|
||||
`function(key) {
|
||||
() =>
|
||||
callOnTrigger<RuntimeReply>(
|
||||
options,
|
||||
triggerObjectId as string,
|
||||
`function(key) {
|
||||
const doc = this.ownerDocument;
|
||||
const owner = doc.defaultView;
|
||||
if (!owner) return false;
|
||||
@@ -193,9 +248,9 @@ export async function uploadThroughActivatedFileInput(
|
||||
doc.addEventListener("click", state.listener, true);
|
||||
return true;
|
||||
}`,
|
||||
[stateKey],
|
||||
true,
|
||||
),
|
||||
[stateKey],
|
||||
true,
|
||||
),
|
||||
deadline,
|
||||
options.signal,
|
||||
"arming frame-scoped file input probe timed out",
|
||||
@@ -204,10 +259,10 @@ export async function uploadThroughActivatedFileInput(
|
||||
if (armError || armed.result?.value !== true) {
|
||||
throw armError ?? new Error("file input probe did not arm");
|
||||
}
|
||||
probeArmed = true;
|
||||
probeState = "active";
|
||||
} catch (err) {
|
||||
outcome = transferFailure(
|
||||
err instanceof BoundedWaitError ? "timeout" : "cdp_failed",
|
||||
waitErrorCode(err),
|
||||
"file_input_probe_failed",
|
||||
err instanceof Error ? err.message : String(err),
|
||||
"none",
|
||||
@@ -219,14 +274,14 @@ export async function uploadThroughActivatedFileInput(
|
||||
let click: ClickResult | RpcError;
|
||||
try {
|
||||
click = await waitBounded(
|
||||
options.trigger(),
|
||||
options.trigger,
|
||||
deadline,
|
||||
options.signal,
|
||||
"upload trigger timed out",
|
||||
);
|
||||
} catch (err) {
|
||||
outcome = transferFailure(
|
||||
err instanceof BoundedWaitError ? "timeout" : "cdp_failed",
|
||||
waitErrorCode(err),
|
||||
"file_input_probe_failed",
|
||||
err instanceof Error ? err.message : String(err),
|
||||
"none",
|
||||
@@ -244,17 +299,23 @@ export async function uploadThroughActivatedFileInput(
|
||||
let summary: { count: number; multiple: boolean } = { count: 0, multiple: false };
|
||||
while (Date.now() < activationDeadline) {
|
||||
if (chooserEvents.length > 0) break;
|
||||
const reply = await callOnTrigger<RuntimeReply>(
|
||||
options,
|
||||
triggerObjectId,
|
||||
`function(key) {
|
||||
const reply = await waitBounded(
|
||||
() =>
|
||||
callOnTrigger<RuntimeReply>(
|
||||
options,
|
||||
triggerObjectId as string,
|
||||
`function(key) {
|
||||
const state = this.ownerDocument.defaultView?.[key];
|
||||
return state
|
||||
? { count: state.inputs.length, multiple: state.inputs[0]?.multiple === true }
|
||||
: { count: 0, multiple: false };
|
||||
}`,
|
||||
[stateKey],
|
||||
true,
|
||||
[stateKey],
|
||||
true,
|
||||
),
|
||||
activationDeadline,
|
||||
options.signal,
|
||||
"inspecting activated file input timed out",
|
||||
);
|
||||
const summaryError = runtimeError(reply, "failed to inspect activated file input");
|
||||
if (summaryError) throw summaryError;
|
||||
@@ -317,21 +378,33 @@ export async function uploadThroughActivatedFileInput(
|
||||
);
|
||||
return outcome;
|
||||
}
|
||||
const input = await callOnTrigger<RuntimeReply>(
|
||||
options,
|
||||
triggerObjectId,
|
||||
`function(key) { return this.ownerDocument.defaultView?.[key]?.inputs[0]; }`,
|
||||
[stateKey],
|
||||
false,
|
||||
const input = await waitBounded(
|
||||
() =>
|
||||
callOnTrigger<RuntimeReply>(
|
||||
options,
|
||||
triggerObjectId as string,
|
||||
`function(key) { return this.ownerDocument.defaultView?.[key]?.inputs[0]; }`,
|
||||
[stateKey],
|
||||
false,
|
||||
),
|
||||
deadline,
|
||||
options.signal,
|
||||
"resolving activated file input timed out",
|
||||
);
|
||||
const inputError = runtimeError(input, "failed to resolve activated file input object");
|
||||
if (inputError) throw inputError;
|
||||
if (!input.result?.objectId) throw new Error("activated file input returned no objectId");
|
||||
const described = await sendToCdpTarget<{ node?: { backendNodeId?: number } }>(
|
||||
options.cdp,
|
||||
target,
|
||||
"DOM.describeNode",
|
||||
{ objectId: input.result.objectId },
|
||||
const described = await waitBounded(
|
||||
() =>
|
||||
sendToCdpTarget<{ node?: { backendNodeId?: number } }>(
|
||||
options.cdp,
|
||||
target,
|
||||
"DOM.describeNode",
|
||||
{ objectId: input.result?.objectId },
|
||||
),
|
||||
deadline,
|
||||
options.signal,
|
||||
"describing activated file input timed out",
|
||||
);
|
||||
backendNodeId = described.node?.backendNodeId;
|
||||
if (typeof backendNodeId !== "number") {
|
||||
@@ -350,17 +423,18 @@ export async function uploadThroughActivatedFileInput(
|
||||
|
||||
try {
|
||||
await waitBounded(
|
||||
sendToCdpTarget(options.cdp, target, "DOM.setFileInputFiles", {
|
||||
files: options.files,
|
||||
backendNodeId,
|
||||
}),
|
||||
() =>
|
||||
sendToCdpTarget(options.cdp, target, "DOM.setFileInputFiles", {
|
||||
files: options.files,
|
||||
backendNodeId,
|
||||
}),
|
||||
deadline,
|
||||
options.signal,
|
||||
"setting file input files timed out",
|
||||
);
|
||||
} catch (err) {
|
||||
outcome = transferFailure(
|
||||
err instanceof BoundedWaitError ? "timeout" : "cdp_failed",
|
||||
waitErrorCode(err),
|
||||
"set_file_input_failed",
|
||||
err instanceof Error ? err.message : String(err),
|
||||
"unknown",
|
||||
@@ -372,7 +446,7 @@ export async function uploadThroughActivatedFileInput(
|
||||
return outcome;
|
||||
} catch (err) {
|
||||
outcome = transferFailure(
|
||||
err instanceof BoundedWaitError ? "timeout" : "cdp_failed",
|
||||
waitErrorCode(err),
|
||||
"file_input_probe_failed",
|
||||
err instanceof Error ? err.message : String(err),
|
||||
"none",
|
||||
@@ -382,61 +456,70 @@ export async function uploadThroughActivatedFileInput(
|
||||
}
|
||||
} finally {
|
||||
chooserSubscription?.dispose();
|
||||
let cleanupFailed = false;
|
||||
if (probeArmed && triggerObjectId) {
|
||||
const cleanupDeadline = Date.now() + CLEANUP_TIMEOUT_MS;
|
||||
const cleanupTasks: Promise<boolean>[] = [];
|
||||
const cleanup = async (operation: () => Promise<unknown>, message: string) => {
|
||||
try {
|
||||
await waitBounded(
|
||||
callOnTrigger(
|
||||
options,
|
||||
triggerObjectId,
|
||||
`function(key) {
|
||||
await waitBounded(operation, cleanupDeadline, undefined, message);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
};
|
||||
if (probeState !== "not_started" && triggerObjectId) {
|
||||
cleanupTasks.push(
|
||||
cleanup(
|
||||
() =>
|
||||
callOnTrigger(
|
||||
options,
|
||||
triggerObjectId as string,
|
||||
`function(key) {
|
||||
const owner = this.ownerDocument.defaultView;
|
||||
const state = owner?.[key];
|
||||
if (state?.listener) this.ownerDocument.removeEventListener("click", state.listener, true);
|
||||
if (owner) delete owner[key];
|
||||
}`,
|
||||
[stateKey],
|
||||
true,
|
||||
),
|
||||
Date.now() + CLEANUP_TIMEOUT_MS,
|
||||
undefined,
|
||||
[stateKey],
|
||||
true,
|
||||
),
|
||||
"cleaning file input probe timed out",
|
||||
);
|
||||
} catch {
|
||||
cleanupFailed = true;
|
||||
}
|
||||
}
|
||||
if (interceptionArmed) {
|
||||
try {
|
||||
await waitBounded(
|
||||
sendToCdpTarget(options.cdp, target, "Page.setInterceptFileChooserDialog", {
|
||||
enabled: false,
|
||||
cancel: true,
|
||||
}),
|
||||
Date.now() + CLEANUP_TIMEOUT_MS,
|
||||
undefined,
|
||||
"disabling file chooser interception timed out",
|
||||
);
|
||||
} catch {
|
||||
cleanupFailed = true;
|
||||
}
|
||||
}
|
||||
try {
|
||||
await waitBounded(
|
||||
sendToCdpTarget(options.cdp, target, "Runtime.releaseObjectGroup", { objectGroup }),
|
||||
Date.now() + CLEANUP_TIMEOUT_MS,
|
||||
undefined,
|
||||
"releasing upload object group timed out",
|
||||
),
|
||||
);
|
||||
} catch {
|
||||
cleanupFailed = true;
|
||||
}
|
||||
if (interceptionState !== "not_started") {
|
||||
cleanupTasks.push(
|
||||
cleanup(
|
||||
() =>
|
||||
sendToCdpTarget(options.cdp, target, "Page.setInterceptFileChooserDialog", {
|
||||
enabled: false,
|
||||
cancel: true,
|
||||
}),
|
||||
"disabling file chooser interception timed out",
|
||||
),
|
||||
);
|
||||
}
|
||||
let cleanupFailed = (await Promise.all(cleanupTasks)).some((completed) => !completed);
|
||||
if (!cleanupFailed) {
|
||||
cleanupFailed = !(await cleanup(
|
||||
() => sendToCdpTarget(options.cdp, target, "Runtime.releaseObjectGroup", { objectGroup }),
|
||||
"releasing upload object group timed out",
|
||||
));
|
||||
}
|
||||
|
||||
const effect = isRpcError(outcome)
|
||||
? (outcome.data?.effect_state as TransferEffectState | undefined)
|
||||
: "committed";
|
||||
if (effect === "unknown" || cleanupFailed) {
|
||||
await options.cdp.detach?.(target.tabId);
|
||||
try {
|
||||
await waitBounded(
|
||||
() => options.cdp.detach?.(target.tabId) ?? Promise.resolve(),
|
||||
Date.now() + DETACH_TIMEOUT_MS,
|
||||
undefined,
|
||||
"detaching upload target timed out",
|
||||
);
|
||||
} catch {
|
||||
cleanupFailed = true;
|
||||
}
|
||||
}
|
||||
if (cleanupFailed && isRpcError(outcome)) {
|
||||
outcome.data = { ...outcome.data, cleanup_state: "failed" };
|
||||
|
||||
@@ -1,21 +1,24 @@
|
||||
import { type CdpTarget, cdpTargetKey } from "@/browser-driver/frame-graph";
|
||||
import { type CdpFrame, type CdpFrameGraph, type CdpTarget } from "@/browser-driver/frame-graph";
|
||||
import type { RpcError } from "@/transport/types";
|
||||
import { nodeContentRegion, scrollNodeIntoView } from "./element-geometry";
|
||||
import {
|
||||
clipPolygon,
|
||||
type GeometryProjection,
|
||||
type Point,
|
||||
type Polygon,
|
||||
type ProjectiveEdge,
|
||||
parseCdpQuad,
|
||||
polygonArea,
|
||||
polygonCentroid,
|
||||
projectRegionToViewport,
|
||||
type Quad,
|
||||
type Region,
|
||||
rectPolygon,
|
||||
regionBounds,
|
||||
type Size,
|
||||
type ViewportRect,
|
||||
} from "./geometry";
|
||||
import type { CssViewport } from "./geometry/coordinate-types";
|
||||
import { cssViewport, GeometryContext } from "./geometry/frame-context";
|
||||
import { type CdpRunner, cdpRunnerForTarget, isRpcError } from "./shared";
|
||||
import { type CdpRunner, cdpRunnerForTarget, isRpcError, sendToCdpTarget } from "./shared";
|
||||
|
||||
export interface NodeAddress {
|
||||
target: CdpTarget;
|
||||
@@ -24,30 +27,154 @@ export interface NodeAddress {
|
||||
}
|
||||
|
||||
export interface ResolvedNodeGeometry {
|
||||
/** The same top-level measurement used by projection, for screenshot coordinate adaptation. */
|
||||
topViewport: CssViewport;
|
||||
topVisibleRegions: Region;
|
||||
topBounds: ViewportRect;
|
||||
/** Point in the top-level tab viewport, used by root-target input events. */
|
||||
actionPoint: Point;
|
||||
/** Point in the addressed CDP target's viewport, used by OOPIF-local input events. */
|
||||
targetActionPoint: Point;
|
||||
}
|
||||
|
||||
function geometryError(message: string): RpcError {
|
||||
return { code: "cdp_failed", message };
|
||||
}
|
||||
|
||||
function frameMap(graph: CdpFrameGraph): Map<string, CdpFrame> {
|
||||
return new Map(graph.frames.map((frame) => [frame.frameId, frame]));
|
||||
}
|
||||
|
||||
function targetRootFrame(byId: Map<string, CdpFrame>, frame: CdpFrame): CdpFrame | null {
|
||||
const seen = new Set<string>();
|
||||
let current = frame;
|
||||
while (current.parentFrameId) {
|
||||
if (seen.has(current.frameId)) return null;
|
||||
seen.add(current.frameId);
|
||||
const parent = byId.get(current.parentFrameId);
|
||||
if (!parent || parent.target.sessionId !== current.target.sessionId) break;
|
||||
current = parent;
|
||||
}
|
||||
return current;
|
||||
}
|
||||
|
||||
function frameAncestry(graph: CdpFrameGraph, frameId: string): CdpFrame[] | null {
|
||||
const byId = frameMap(graph);
|
||||
const path: CdpFrame[] = [];
|
||||
const seen = new Set<string>();
|
||||
let current = byId.get(frameId);
|
||||
if (!current) return null;
|
||||
while (current.parentFrameId) {
|
||||
if (seen.has(current.frameId)) return null;
|
||||
seen.add(current.frameId);
|
||||
path.push(current);
|
||||
const parent = byId.get(current.parentFrameId);
|
||||
if (!parent) return null;
|
||||
current = parent;
|
||||
}
|
||||
return path;
|
||||
}
|
||||
|
||||
async function targetViewport(cdp: CdpRunner, target: CdpTarget): Promise<Size | null> {
|
||||
const metrics = await sendToCdpTarget<{
|
||||
cssLayoutViewport?: { clientWidth?: number; clientHeight?: number };
|
||||
layoutViewport?: { clientWidth?: number; clientHeight?: number };
|
||||
}>(cdp, target, "Page.getLayoutMetrics", {});
|
||||
const viewport = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
|
||||
const width = viewport.clientWidth ?? 0;
|
||||
const height = viewport.clientHeight ?? 0;
|
||||
return width > 0 && height > 0 ? { width, height } : null;
|
||||
}
|
||||
|
||||
async function ownerContentQuad(
|
||||
cdp: CdpRunner,
|
||||
parent: CdpFrame,
|
||||
ownerBackendNodeId: number,
|
||||
): Promise<Quad | null> {
|
||||
const result = await sendToCdpTarget<{ model?: { content?: number[] } }>(
|
||||
cdp,
|
||||
parent.target,
|
||||
"DOM.getBoxModel",
|
||||
{ backendNodeId: ownerBackendNodeId },
|
||||
);
|
||||
return parseCdpQuad(result.model?.content);
|
||||
}
|
||||
|
||||
async function sameTargetFrameClips(
|
||||
cdp: CdpRunner,
|
||||
byId: Map<string, CdpFrame>,
|
||||
frame: CdpFrame,
|
||||
targetRoot: CdpFrame,
|
||||
): Promise<Polygon[] | null> {
|
||||
const clips: Polygon[] = [];
|
||||
let current = frame;
|
||||
const seen = new Set<string>();
|
||||
while (current.frameId !== targetRoot.frameId) {
|
||||
if (seen.has(current.frameId) || !current.parentFrameId) return null;
|
||||
seen.add(current.frameId);
|
||||
const parent = byId.get(current.parentFrameId);
|
||||
if (!parent || current.ownerBackendNodeId === undefined) return null;
|
||||
const clip = await ownerContentQuad(cdp, parent, current.ownerBackendNodeId);
|
||||
if (!clip) return null;
|
||||
clips.push(clip);
|
||||
current = parent;
|
||||
}
|
||||
return clips;
|
||||
}
|
||||
|
||||
export async function resolveFrameProjection(
|
||||
cdp: CdpRunner,
|
||||
graph: CdpFrameGraph,
|
||||
frameId: string,
|
||||
): Promise<GeometryProjection | null> {
|
||||
const byId = frameMap(graph);
|
||||
const frame = byId.get(frameId);
|
||||
if (!frame) return null;
|
||||
let targetRoot = targetRootFrame(byId, frame);
|
||||
if (!targetRoot) return null;
|
||||
const sourceViewport = await targetViewport(cdp, targetRoot.target);
|
||||
if (!sourceViewport) return null;
|
||||
const sourceClips = await sameTargetFrameClips(cdp, byId, frame, targetRoot);
|
||||
if (!sourceClips) return null;
|
||||
|
||||
const edges: ProjectiveEdge[] = [];
|
||||
while (targetRoot.parentFrameId) {
|
||||
const parent = byId.get(targetRoot.parentFrameId);
|
||||
if (!parent || targetRoot.ownerBackendNodeId === undefined) return null;
|
||||
const destinationQuad = await ownerContentQuad(cdp, parent, targetRoot.ownerBackendNodeId);
|
||||
if (!destinationQuad) return null;
|
||||
const source =
|
||||
edges.length === 0 ? sourceViewport : await targetViewport(cdp, targetRoot.target);
|
||||
if (!source) return null;
|
||||
const parentTargetRoot = targetRootFrame(byId, parent);
|
||||
if (!parentTargetRoot) return null;
|
||||
const destinationClips = await sameTargetFrameClips(cdp, byId, parent, parentTargetRoot);
|
||||
if (!destinationClips) return null;
|
||||
edges.push({ sourceViewport: source, destinationQuad, destinationClips });
|
||||
targetRoot = parentTargetRoot;
|
||||
}
|
||||
|
||||
const topViewport =
|
||||
edges.length === 0 ? sourceViewport : await targetViewport(cdp, targetRoot.target);
|
||||
return topViewport ? { sourceClips, edges, topViewport } : null;
|
||||
}
|
||||
|
||||
async function loadFrameGraph(cdp: CdpRunner, tabId: number): Promise<CdpFrameGraph | null> {
|
||||
if (!cdp.getFrameGraph) return null;
|
||||
try {
|
||||
return await cdp.getFrameGraph(tabId);
|
||||
} catch (error) {
|
||||
console.debug("[bsk frame-geometry] frame graph resolution failed", error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function scrollFrameOwners(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
context: GeometryContext,
|
||||
graph: CdpFrameGraph,
|
||||
frameId: string,
|
||||
): Promise<RpcError | null> {
|
||||
const ancestry = await context.ancestry(frameId);
|
||||
const byId = frameMap(graph);
|
||||
const ancestry = frameAncestry(graph, frameId);
|
||||
if (!ancestry) return geometryError(`could not resolve frame ancestry for ${frameId}`);
|
||||
for (const child of [...ancestry].reverse()) {
|
||||
const parent = child.parentFrameId ? await context.frame(child.parentFrameId) : undefined;
|
||||
const parent = child.parentFrameId ? byId.get(child.parentFrameId) : undefined;
|
||||
if (!parent || child.ownerBackendNodeId === undefined) {
|
||||
return geometryError(`could not resolve frame owner for ${child.frameId}`);
|
||||
}
|
||||
@@ -61,20 +188,17 @@ async function scrollFrameOwners(
|
||||
return null;
|
||||
}
|
||||
|
||||
async function scrollElementWithContext(
|
||||
async function scrollElementWithFrameGraph(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
target: CdpTarget,
|
||||
backendNodeId: number,
|
||||
frameId: string | undefined,
|
||||
context: GeometryContext,
|
||||
graph: CdpFrameGraph | null,
|
||||
): Promise<RpcError | null> {
|
||||
if (frameId) {
|
||||
const frame = await context.frame(frameId);
|
||||
if (!frame || cdpTargetKey(frame.target) !== cdpTargetKey(target)) {
|
||||
return geometryError("node frame does not belong to its target");
|
||||
}
|
||||
const error = await scrollFrameOwners(cdp, tabId, context, frameId);
|
||||
if (!graph) return geometryError(`could not resolve frame graph for ${frameId}`);
|
||||
const error = await scrollFrameOwners(cdp, tabId, graph, frameId);
|
||||
if (error) return error;
|
||||
} else if (target.sessionId) {
|
||||
return geometryError("an OOPIF node address requires frameId");
|
||||
@@ -89,15 +213,8 @@ export async function scrollElementAndFramesIntoView(
|
||||
backendNodeId: number,
|
||||
frameId?: string,
|
||||
): Promise<RpcError | null> {
|
||||
if (target.tabId !== tabId) return geometryError("node target belongs to another tab");
|
||||
return scrollElementWithContext(
|
||||
cdp,
|
||||
tabId,
|
||||
target,
|
||||
backendNodeId,
|
||||
frameId,
|
||||
new GeometryContext(cdp, tabId),
|
||||
);
|
||||
const graph = frameId ? await loadFrameGraph(cdp, tabId) : null;
|
||||
return scrollElementWithFrameGraph(cdp, tabId, target, backendNodeId, frameId, graph);
|
||||
}
|
||||
|
||||
function largestRegion(regions: Region): Polygon | null {
|
||||
@@ -120,32 +237,23 @@ export async function resolveNodeGeometry(
|
||||
if (address.target.sessionId && !address.frameId) {
|
||||
return geometryError("an OOPIF node address requires frameId");
|
||||
}
|
||||
if (address.target.tabId !== tabId) return geometryError("node target belongs to another tab");
|
||||
const context = new GeometryContext(cdp, tabId);
|
||||
const graph = address.frameId ? await context.graph() : null;
|
||||
if (address.frameId && graph) {
|
||||
const frame = await context.frame(address.frameId);
|
||||
if (!frame || cdpTargetKey(frame.target) !== cdpTargetKey(address.target)) {
|
||||
return geometryError("node frame does not belong to its target");
|
||||
}
|
||||
}
|
||||
const graph = address.frameId ? await loadFrameGraph(cdp, tabId) : null;
|
||||
if (address.frameId && !graph) {
|
||||
return geometryError(`could not resolve frame graph for ${address.frameId}`);
|
||||
}
|
||||
|
||||
if (options.scrollIntoView) {
|
||||
const scrollError = await scrollElementWithContext(
|
||||
const scrollError = await scrollElementWithFrameGraph(
|
||||
cdp,
|
||||
tabId,
|
||||
address.target,
|
||||
address.backendNodeId,
|
||||
address.frameId,
|
||||
context,
|
||||
graph,
|
||||
);
|
||||
if (scrollError) return scrollError;
|
||||
}
|
||||
|
||||
// Only topology was read above. Begin live measurements after scrolling.
|
||||
const localRegion = await nodeContentRegion(
|
||||
cdpRunnerForTarget(cdp, address.target),
|
||||
tabId,
|
||||
@@ -154,32 +262,13 @@ export async function resolveNodeGeometry(
|
||||
if (isRpcError(localRegion)) return localRegion;
|
||||
|
||||
let topVisibleRegions: Region;
|
||||
let targetActionPoint: Point | null = null;
|
||||
if (address.frameId && graph) {
|
||||
const projection = await context.targetProjection(address.frameId);
|
||||
const projection = await resolveFrameProjection(cdp, graph, address.frameId);
|
||||
if (!projection)
|
||||
return geometryError(`could not resolve frame geometry for ${address.frameId}`);
|
||||
topVisibleRegions = projectRegionToViewport(localRegion, projection);
|
||||
if (address.target.sessionId) {
|
||||
const localViewport = projection.edges[0]?.sourceViewport ?? projection.topViewport;
|
||||
const localVisibleRegions = localRegion
|
||||
.map((polygon) =>
|
||||
clipPolygon(
|
||||
polygon,
|
||||
rectPolygon({
|
||||
x: 0,
|
||||
y: 0,
|
||||
w: localViewport.width,
|
||||
h: localViewport.height,
|
||||
}),
|
||||
),
|
||||
)
|
||||
.filter((polygon) => polygon.length >= 3);
|
||||
const localActionRegion = largestRegion(localVisibleRegions);
|
||||
targetActionPoint = localActionRegion ? polygonCentroid(localActionRegion) : null;
|
||||
}
|
||||
} else {
|
||||
const viewport = await context.viewport(address.target);
|
||||
const viewport = await targetViewport(cdp, address.target);
|
||||
if (!viewport) return geometryError("could not resolve top viewport geometry");
|
||||
topVisibleRegions = localRegion
|
||||
.map((polygon) =>
|
||||
@@ -194,17 +283,7 @@ export async function resolveNodeGeometry(
|
||||
if (!topBounds || !actionPoint) {
|
||||
return { code: "permission_denied", message: "element not visible" };
|
||||
}
|
||||
if (address.target.sessionId && !targetActionPoint) {
|
||||
return { code: "permission_denied", message: "element not visible in its target" };
|
||||
}
|
||||
targetActionPoint ??= actionPoint;
|
||||
return {
|
||||
topVisibleRegions,
|
||||
topBounds,
|
||||
actionPoint,
|
||||
targetActionPoint,
|
||||
topViewport: cssViewport(await context.layoutMetrics({ tabId })),
|
||||
};
|
||||
return { topVisibleRegions, topBounds, actionPoint };
|
||||
} catch (error) {
|
||||
return geometryError(error instanceof Error ? error.message : String(error));
|
||||
}
|
||||
|
||||
@@ -245,3 +245,22 @@ export function projectRectToViewport(
|
||||
if (!rect) return null;
|
||||
return regionBounds(projectRegionToViewport([rectPolygon(rect)], projection));
|
||||
}
|
||||
|
||||
export function childFrameProjection(
|
||||
parent: GeometryProjection,
|
||||
ownerRectInParent: { x: number; y: number; w: number; h: number },
|
||||
): GeometryProjection {
|
||||
const destinationQuad = rectPolygon(ownerRectInParent);
|
||||
return {
|
||||
sourceClips: [],
|
||||
edges: [
|
||||
{
|
||||
sourceViewport: { width: ownerRectInParent.w, height: ownerRectInParent.h },
|
||||
destinationQuad,
|
||||
destinationClips: parent.sourceClips,
|
||||
},
|
||||
...parent.edges,
|
||||
],
|
||||
topViewport: parent.topViewport,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1,102 +0,0 @@
|
||||
import { type CdpTarget, cdpTargetKey } from "@/browser-driver/frame-graph";
|
||||
import type { GeometryProjection, ViewportRect } from "../geometry";
|
||||
import { projectRectToViewport } from "../geometry";
|
||||
|
||||
export interface CoordinateOwner {
|
||||
target: CdpTarget;
|
||||
frameId?: string;
|
||||
}
|
||||
|
||||
export interface FrameViewportRect {
|
||||
space: "frame-viewport-css";
|
||||
owner: CoordinateOwner;
|
||||
rect: ViewportRect;
|
||||
}
|
||||
|
||||
export interface SnapshotProjection {
|
||||
source: CoordinateOwner;
|
||||
geometry: GeometryProjection;
|
||||
}
|
||||
|
||||
/** A failed owner read is different from a successfully projected, clipped-out rect. */
|
||||
export interface UnavailableFrameProjection {
|
||||
status: "unavailable";
|
||||
source: CoordinateOwner;
|
||||
ownerBackendNodeId: number;
|
||||
}
|
||||
|
||||
export type SnapshotProjectionResult =
|
||||
| { status: "available"; projection: SnapshotProjection }
|
||||
| UnavailableFrameProjection;
|
||||
|
||||
/** Descendants inherit the failed boundary without attempting another owner read. */
|
||||
export type FrameProjectionIssue =
|
||||
| UnavailableFrameProjection
|
||||
| { status: "blocked"; source: CoordinateOwner; cause: UnavailableFrameProjection };
|
||||
|
||||
export type FrameProjectionState = SnapshotProjectionResult | FrameProjectionIssue;
|
||||
|
||||
/** DOMSnapshot bounds are document-relative CSS pixels, independent of raster DPR. */
|
||||
export function snapshotViewportRect(
|
||||
bounds: number[],
|
||||
owner: CoordinateOwner,
|
||||
scroll: { x: number; y: number },
|
||||
): FrameViewportRect | null {
|
||||
if (bounds.length < 4 || !bounds.slice(0, 4).every(Number.isFinite)) return null;
|
||||
const [x, y, width, height] = bounds;
|
||||
if (width <= 0 || height <= 0 || !Number.isFinite(scroll.x) || !Number.isFinite(scroll.y)) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
space: "frame-viewport-css",
|
||||
owner,
|
||||
rect: { x: x - scroll.x, y: y - scroll.y, width, height },
|
||||
};
|
||||
}
|
||||
|
||||
export function projectSnapshotRect(
|
||||
input: FrameViewportRect,
|
||||
projection: SnapshotProjection,
|
||||
): ViewportRect | null {
|
||||
if (
|
||||
input.owner.frameId !== projection.source.frameId ||
|
||||
cdpTargetKey(input.owner.target) !== cdpTargetKey(projection.source.target)
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
const { x, y, width: w, height: h } = input.rect;
|
||||
return projectRectToViewport({ x, y, w, h }, projection.geometry);
|
||||
}
|
||||
|
||||
/** CSS viewport metadata; browser zoom is distinct from raster devicePixelRatio. */
|
||||
export interface CssViewport {
|
||||
width: number;
|
||||
height: number;
|
||||
scrollX: number;
|
||||
scrollY: number;
|
||||
cssToDip: number;
|
||||
}
|
||||
|
||||
/** Page.captureScreenshot takes page-relative DIP, not viewport-relative CSS pixels. */
|
||||
export function screenshotPageRect(
|
||||
rect: ViewportRect,
|
||||
viewport: CssViewport,
|
||||
): { space: "page-dip"; rect: ViewportRect } | null {
|
||||
const { cssToDip, scrollX, scrollY } = viewport;
|
||||
if (
|
||||
![rect.x, rect.y, rect.width, rect.height, cssToDip, scrollX, scrollY].every(Number.isFinite) ||
|
||||
cssToDip <= 0 ||
|
||||
rect.width <= 0 ||
|
||||
rect.height <= 0
|
||||
)
|
||||
return null;
|
||||
return {
|
||||
space: "page-dip",
|
||||
rect: {
|
||||
x: (rect.x + scrollX) * cssToDip,
|
||||
y: (rect.y + scrollY) * cssToDip,
|
||||
width: rect.width * cssToDip,
|
||||
height: rect.height * cssToDip,
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -1,333 +0,0 @@
|
||||
import {
|
||||
type CdpFrame,
|
||||
type CdpFrameGraph,
|
||||
type CdpTarget,
|
||||
cdpTargetKey,
|
||||
} from "@/browser-driver/frame-graph";
|
||||
import {
|
||||
type GeometryProjection,
|
||||
type Polygon,
|
||||
type ProjectiveEdge,
|
||||
parseCdpQuad,
|
||||
type Quad,
|
||||
type Size,
|
||||
} from "../geometry";
|
||||
import { type CdpRunner, sendToCdpTarget } from "../shared";
|
||||
import type { CoordinateOwner, CssViewport, SnapshotProjectionResult } from "./coordinate-types";
|
||||
|
||||
import { readSnapshotOwnerSizes } from "./snapshot-owner-sizes";
|
||||
|
||||
export interface LayoutMetrics {
|
||||
cssVisualViewport?: { zoom?: number };
|
||||
visualViewport?: { zoom?: number };
|
||||
cssLayoutViewport?: {
|
||||
clientWidth?: number;
|
||||
clientHeight?: number;
|
||||
pageX?: number;
|
||||
pageY?: number;
|
||||
};
|
||||
layoutViewport?: { clientWidth?: number; clientHeight?: number; pageX?: number; pageY?: number };
|
||||
}
|
||||
|
||||
/** The legacy field is a compatibility fallback, never a source of raster DPR. */
|
||||
export function cssViewport(metrics: LayoutMetrics): CssViewport {
|
||||
const value = metrics.cssLayoutViewport ?? metrics.layoutViewport;
|
||||
return {
|
||||
width: value?.clientWidth ?? 0,
|
||||
height: value?.clientHeight ?? 0,
|
||||
scrollX: value?.pageX ?? 0,
|
||||
scrollY: value?.pageY ?? 0,
|
||||
cssToDip: metrics.cssVisualViewport?.zoom ?? metrics.visualViewport?.zoom ?? 1,
|
||||
};
|
||||
}
|
||||
|
||||
/** One read-only measurement phase. Discard after scrolling or any later operation. */
|
||||
export class GeometryContext {
|
||||
private readonly metrics = new Map<string, Promise<LayoutMetrics>>();
|
||||
private readonly owners = new Map<string, Promise<Quad | null>>();
|
||||
private readonly projections = new Map<string, Promise<GeometryProjection | null>>();
|
||||
private readonly snapshotOwners = new Map<string, Promise<{ quad: Quad; size: Size } | null>>();
|
||||
private readonly sizeBatches = new Map<string, () => Promise<Map<number, Size>>>();
|
||||
private active = 0;
|
||||
private readonly waiting: Array<() => void> = [];
|
||||
private graphPromise?: Promise<CdpFrameGraph | null>;
|
||||
private framesPromise?: Promise<Map<string, CdpFrame>>;
|
||||
|
||||
constructor(
|
||||
private readonly cdp: CdpRunner,
|
||||
private readonly tabId: number,
|
||||
graph?: CdpFrameGraph,
|
||||
private readonly signal?: AbortSignal,
|
||||
) {
|
||||
if (graph) this.graphPromise = Promise.resolve(graph);
|
||||
}
|
||||
|
||||
private async request<T>(
|
||||
target: CdpTarget,
|
||||
method: string,
|
||||
params: object,
|
||||
cleanup = false,
|
||||
): Promise<T> {
|
||||
if (!cleanup && this.signal?.aborted) throw new DOMException("geometry aborted", "AbortError");
|
||||
if (this.active >= 4) await new Promise<void>((resolve) => this.waiting.push(resolve));
|
||||
else this.active++;
|
||||
try {
|
||||
if (!cleanup && this.signal?.aborted)
|
||||
throw new DOMException("geometry aborted", "AbortError");
|
||||
return await sendToCdpTarget<T>(this.cdp, target, method, params);
|
||||
} finally {
|
||||
const next = this.waiting.shift();
|
||||
if (next) next();
|
||||
else this.active--;
|
||||
}
|
||||
}
|
||||
|
||||
graph(): Promise<CdpFrameGraph | null> {
|
||||
this.graphPromise ??=
|
||||
this.cdp.getFrameGraph?.(this.tabId).catch(() => null) ?? Promise.resolve(null);
|
||||
return this.graphPromise;
|
||||
}
|
||||
|
||||
private frames(): Promise<Map<string, CdpFrame>> {
|
||||
this.framesPromise ??= this.graph().then(
|
||||
(graph) => new Map(graph?.frames.map((frame) => [frame.frameId, frame]) ?? []),
|
||||
);
|
||||
return this.framesPromise;
|
||||
}
|
||||
|
||||
async frame(frameId: string): Promise<CdpFrame | undefined> {
|
||||
return (await this.frames()).get(frameId);
|
||||
}
|
||||
|
||||
/** Topology only: safe to resolve before the caller scrolls frame owners. */
|
||||
async ancestry(frameId: string): Promise<CdpFrame[] | null> {
|
||||
const frames = await this.frames();
|
||||
const path: CdpFrame[] = [];
|
||||
const seen = new Set<string>();
|
||||
let current = frames.get(frameId);
|
||||
if (!current) return null;
|
||||
while (current.parentFrameId) {
|
||||
if (seen.has(current.frameId)) return null;
|
||||
seen.add(current.frameId);
|
||||
path.push(current);
|
||||
const parent = frames.get(current.parentFrameId);
|
||||
if (!parent) return null;
|
||||
current = parent;
|
||||
}
|
||||
return path;
|
||||
}
|
||||
|
||||
layoutMetrics(target: CdpTarget): Promise<LayoutMetrics> {
|
||||
const key = cdpTargetKey(target);
|
||||
let promise = this.metrics.get(key);
|
||||
if (!promise) {
|
||||
promise = this.request<LayoutMetrics>(target, "Page.getLayoutMetrics", {});
|
||||
this.metrics.set(key, promise);
|
||||
}
|
||||
return promise;
|
||||
}
|
||||
|
||||
async viewport(target: CdpTarget): Promise<Size | null> {
|
||||
const { width, height } = cssViewport(await this.layoutMetrics(target));
|
||||
return Number.isFinite(width) && Number.isFinite(height) && width > 0 && height > 0
|
||||
? { width, height }
|
||||
: null;
|
||||
}
|
||||
|
||||
ownerContent(target: CdpTarget, backendNodeId: number): Promise<Quad | null> {
|
||||
const key = `${cdpTargetKey(target)}:${backendNodeId}`;
|
||||
let promise = this.owners.get(key);
|
||||
if (!promise) {
|
||||
promise = this.request<{ model?: { content?: number[] } }>(target, "DOM.getBoxModel", {
|
||||
backendNodeId,
|
||||
}).then((reply) => parseCdpQuad(reply.model?.content));
|
||||
this.owners.set(key, promise);
|
||||
}
|
||||
return promise;
|
||||
}
|
||||
|
||||
/** Same-target owner quads are already target-relative, including nested owners. */
|
||||
async snapshotProjection(
|
||||
source: CoordinateOwner,
|
||||
ownerBackendNodeId: number,
|
||||
ancestorClips: Polygon[],
|
||||
viewport: Size,
|
||||
): Promise<SnapshotProjectionResult> {
|
||||
const key = `${cdpTargetKey(source.target)}:${ownerBackendNodeId}`;
|
||||
let promise = this.snapshotOwners.get(key);
|
||||
if (!promise) {
|
||||
promise = this.snapshotOwner(source.target, ownerBackendNodeId);
|
||||
this.snapshotOwners.set(key, promise);
|
||||
}
|
||||
let owner: { quad: Quad; size: Size } | null;
|
||||
try {
|
||||
owner = await promise;
|
||||
} catch (error) {
|
||||
if (error && typeof error === "object" && "name" in error && error.name === "AbortError")
|
||||
throw error;
|
||||
console.debug("[bsk geometry] frame owner unavailable", error);
|
||||
owner = null;
|
||||
}
|
||||
return owner
|
||||
? {
|
||||
status: "available",
|
||||
projection: {
|
||||
source,
|
||||
geometry: {
|
||||
sourceClips: [],
|
||||
edges: [
|
||||
{
|
||||
sourceViewport: owner.size,
|
||||
destinationQuad: owner.quad,
|
||||
destinationClips: ancestorClips,
|
||||
},
|
||||
],
|
||||
topViewport: viewport,
|
||||
},
|
||||
},
|
||||
}
|
||||
: { status: "unavailable", source, ownerBackendNodeId };
|
||||
}
|
||||
|
||||
/** Register one target's snapshot owners; the batch starts only when a
|
||||
* projection actually needs a size. Single owners keep the cheaper old path. */
|
||||
registerSnapshotOwners(target: CdpTarget, owners: ReadonlySet<number>): void {
|
||||
const key = cdpTargetKey(target);
|
||||
if (owners.size < 2 || this.sizeBatches.has(key)) return;
|
||||
let pending: Promise<Map<number, Size>> | undefined;
|
||||
this.sizeBatches.set(key, () => {
|
||||
pending ??= readSnapshotOwnerSizes(
|
||||
(method, params, cleanup) => this.request(target, method, params, cleanup),
|
||||
owners,
|
||||
).catch((error) => {
|
||||
if (error && typeof error === "object" && "name" in error && error.name === "AbortError")
|
||||
throw error;
|
||||
return new Map<number, Size>();
|
||||
});
|
||||
return pending;
|
||||
});
|
||||
}
|
||||
|
||||
private async snapshotOwner(
|
||||
target: CdpTarget,
|
||||
backendNodeId: number,
|
||||
): Promise<{ quad: Quad; size: Size } | null> {
|
||||
const quad = await this.ownerContent(target, backendNodeId);
|
||||
if (!quad) return null;
|
||||
const batchSize = (await this.sizeBatches.get(cdpTargetKey(target))?.())?.get(backendNodeId);
|
||||
if (this.signal?.aborted) throw new DOMException("geometry aborted", "AbortError");
|
||||
if (batchSize) return { quad, size: batchSize };
|
||||
const resolved = await this.request<{ object?: { objectId?: string } }>(
|
||||
target,
|
||||
"DOM.resolveNode",
|
||||
{ backendNodeId },
|
||||
);
|
||||
const objectId = resolved.object?.objectId;
|
||||
if (!objectId) return null;
|
||||
try {
|
||||
const reply = await this.request<{ result?: { value?: Size }; exceptionDetails?: unknown }>(
|
||||
target,
|
||||
"Runtime.callFunctionOn",
|
||||
{
|
||||
objectId,
|
||||
functionDeclaration: `function() {
|
||||
if (!this.isConnected) return null;
|
||||
const style = getComputedStyle(this);
|
||||
return {
|
||||
width: this.clientWidth - parseFloat(style.paddingLeft) - parseFloat(style.paddingRight),
|
||||
height: this.clientHeight - parseFloat(style.paddingTop) - parseFloat(style.paddingBottom)
|
||||
};
|
||||
}`,
|
||||
returnByValue: true,
|
||||
},
|
||||
);
|
||||
const size = reply.result?.value;
|
||||
return !reply.exceptionDetails &&
|
||||
size &&
|
||||
Number.isFinite(size.width) &&
|
||||
Number.isFinite(size.height) &&
|
||||
size.width > 0 &&
|
||||
size.height > 0
|
||||
? { quad, size }
|
||||
: null;
|
||||
} finally {
|
||||
// Cleanup must still run if cancellation arrived after resolving the object.
|
||||
await this.request(target, "Runtime.releaseObject", { objectId }, true).catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
private targetRoot(frames: Map<string, CdpFrame>, frame: CdpFrame): CdpFrame | null {
|
||||
const seen = new Set<string>();
|
||||
let current = frame;
|
||||
while (current.parentFrameId) {
|
||||
if (seen.has(current.frameId)) return null;
|
||||
seen.add(current.frameId);
|
||||
const parent = frames.get(current.parentFrameId);
|
||||
if (!parent) return null;
|
||||
if (cdpTargetKey(parent.target) !== cdpTargetKey(current.target)) break;
|
||||
current = parent;
|
||||
}
|
||||
return current;
|
||||
}
|
||||
|
||||
private async clips(
|
||||
frames: Map<string, CdpFrame>,
|
||||
frame: CdpFrame,
|
||||
root: CdpFrame,
|
||||
): Promise<Polygon[] | null> {
|
||||
const clips: Polygon[] = [];
|
||||
let current = frame;
|
||||
const seen = new Set<string>();
|
||||
while (current.frameId !== root.frameId) {
|
||||
if (seen.has(current.frameId) || !current.parentFrameId) return null;
|
||||
seen.add(current.frameId);
|
||||
const parent = frames.get(current.parentFrameId);
|
||||
if (!parent || current.ownerBackendNodeId === undefined) return null;
|
||||
const quad = await this.ownerContent(parent.target, current.ownerBackendNodeId);
|
||||
if (!quad) return null;
|
||||
clips.push(quad);
|
||||
current = parent;
|
||||
}
|
||||
return clips;
|
||||
}
|
||||
|
||||
/** Input is already in the frame's CDP target viewport, not its local viewport. */
|
||||
targetProjection(frameId: string): Promise<GeometryProjection | null> {
|
||||
let promise = this.projections.get(frameId);
|
||||
if (!promise) {
|
||||
promise = this.buildTargetProjection(frameId);
|
||||
this.projections.set(frameId, promise);
|
||||
}
|
||||
return promise;
|
||||
}
|
||||
|
||||
private async buildTargetProjection(frameId: string): Promise<GeometryProjection | null> {
|
||||
const frames = await this.frames();
|
||||
const frame = frames.get(frameId);
|
||||
if (!frame) return null;
|
||||
let root = this.targetRoot(frames, frame);
|
||||
if (!root) return null;
|
||||
const sourceViewport = await this.viewport(root.target);
|
||||
if (!sourceViewport) return null;
|
||||
const sourceClips = await this.clips(frames, frame, root);
|
||||
if (!sourceClips) return null;
|
||||
const edges: ProjectiveEdge[] = [];
|
||||
const seen = new Set<string>();
|
||||
while (root.parentFrameId) {
|
||||
if (seen.has(root.frameId)) return null;
|
||||
seen.add(root.frameId);
|
||||
const parent = frames.get(root.parentFrameId);
|
||||
if (!parent || root.ownerBackendNodeId === undefined) return null;
|
||||
const destinationQuad = await this.ownerContent(parent.target, root.ownerBackendNodeId);
|
||||
const source = await this.viewport(root.target);
|
||||
const parentRoot = this.targetRoot(frames, parent);
|
||||
if (!destinationQuad || !source || !parentRoot) return null;
|
||||
const destinationClips = await this.clips(frames, parent, parentRoot);
|
||||
if (!destinationClips) return null;
|
||||
edges.push({ sourceViewport: source, destinationQuad, destinationClips });
|
||||
root = parentRoot;
|
||||
}
|
||||
const topViewport = await this.viewport(root.target);
|
||||
return topViewport ? { sourceClips, edges, topViewport } : null;
|
||||
}
|
||||
}
|
||||
@@ -1,74 +0,0 @@
|
||||
import type { Size } from "../geometry";
|
||||
|
||||
interface SerializedValue {
|
||||
type: string;
|
||||
value?: unknown;
|
||||
}
|
||||
|
||||
type Send = <T>(method: string, params: object, cleanup?: boolean) => Promise<T>;
|
||||
|
||||
/** Batch only accessible frame owners. Missing/shadow/cross-origin owners keep
|
||||
* the existing node-scoped read; no page-wide style columns are requested. */
|
||||
export async function readSnapshotOwnerSizes(
|
||||
send: Send,
|
||||
owners: ReadonlySet<number>,
|
||||
): Promise<Map<number, Size>> {
|
||||
const sizes = new Map<number, Size>();
|
||||
const objectGroup = `bsk-frame-sizes-${crypto.randomUUID()}`;
|
||||
try {
|
||||
const reply = await send<{ result?: { deepSerializedValue?: SerializedValue } }>(
|
||||
"Runtime.evaluate",
|
||||
{
|
||||
expression: `(() => {
|
||||
const result = [];
|
||||
const documents = [document];
|
||||
for (let i = 0; i < documents.length && result.length < ${owners.size}; i++) {
|
||||
for (const owner of documents[i].querySelectorAll('iframe,frame')) {
|
||||
if (result.length >= ${owners.size}) break;
|
||||
if (!owner.isConnected) continue;
|
||||
const style = getComputedStyle(owner);
|
||||
result.push([owner, JSON.stringify({
|
||||
width: owner.clientWidth - parseFloat(style.paddingLeft) - parseFloat(style.paddingRight),
|
||||
height: owner.clientHeight - parseFloat(style.paddingTop) - parseFloat(style.paddingBottom)
|
||||
})]);
|
||||
try { if (owner.contentDocument) documents.push(owner.contentDocument); } catch {}
|
||||
}
|
||||
}
|
||||
return result;
|
||||
})()`,
|
||||
objectGroup,
|
||||
serializationOptions: {
|
||||
serialization: "deep",
|
||||
maxDepth: 3,
|
||||
additionalParameters: { maxNodeDepth: 0, includeShadowTree: "none" },
|
||||
},
|
||||
},
|
||||
);
|
||||
const result = reply.result?.deepSerializedValue;
|
||||
if (result?.type !== "array" || !Array.isArray(result.value)) return sizes;
|
||||
for (const entry of result.value as SerializedValue[]) {
|
||||
if (entry.type !== "array" || !Array.isArray(entry.value)) continue;
|
||||
const [node, data] = entry.value as SerializedValue[];
|
||||
if (node?.type !== "node" || data?.type !== "string" || typeof data.value !== "string")
|
||||
continue;
|
||||
const id = (node.value as { backendNodeId?: number } | undefined)?.backendNodeId;
|
||||
if (id === undefined || !owners.has(id)) continue;
|
||||
try {
|
||||
const size = JSON.parse(data.value) as Size;
|
||||
if (
|
||||
size &&
|
||||
Number.isFinite(size.width) &&
|
||||
Number.isFinite(size.height) &&
|
||||
size.width > 0 &&
|
||||
size.height > 0
|
||||
)
|
||||
sizes.set(id, size);
|
||||
} catch {
|
||||
/* An invalid entry uses the existing individual read. */
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
await send("Runtime.releaseObjectGroup", { objectGroup }, true).catch(() => {});
|
||||
}
|
||||
return sizes;
|
||||
}
|
||||
@@ -183,8 +183,6 @@ export async function resolveBackendNode(
|
||||
}
|
||||
// selector path
|
||||
try {
|
||||
// Selector lookup itself attaches CDP, even when no element is found.
|
||||
cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
const doc = await cdp.send<{ root?: { nodeId?: number } }>(target.tabId, "DOM.getDocument", {
|
||||
depth: 0,
|
||||
});
|
||||
@@ -609,53 +607,6 @@ type SelectMutationResult =
|
||||
// tool.fill
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type FillFailureReason =
|
||||
| "target_not_fillable"
|
||||
| "fill_value_invalid"
|
||||
| "fill_target_changed"
|
||||
| "fill_focus_lost"
|
||||
| "fill_value_mismatch"
|
||||
| "fill_failed";
|
||||
|
||||
type FillPreparation =
|
||||
| { before: string; expected: string }
|
||||
| { reason: FillFailureReason; error: string };
|
||||
|
||||
type FillReadiness = "ready" | "background" | "fill_target_changed" | "fill_focus_lost";
|
||||
|
||||
function fillError(reason: FillFailureReason, message: string): RpcError {
|
||||
return rpcError(
|
||||
reason === "target_not_fillable" || reason === "fill_value_invalid"
|
||||
? "invalid_params"
|
||||
: "cdp_failed",
|
||||
reason,
|
||||
message,
|
||||
);
|
||||
}
|
||||
|
||||
interface FillScriptReply<T> {
|
||||
result?: { value?: T };
|
||||
exceptionDetails?: unknown;
|
||||
}
|
||||
|
||||
const FILL_EDITABLE_FUNCTION = `function() {
|
||||
const tag = this.tagName.toLowerCase();
|
||||
const supported = tag === 'input'
|
||||
? ['text', 'search', 'tel', 'url', 'email', 'password', 'number'].includes(this.type)
|
||||
: tag === 'textarea' || this.isContentEditable;
|
||||
return this.isConnected && supported && !this.readOnly && !this.matches(':disabled');
|
||||
}`;
|
||||
|
||||
// Chrome renders a trailing editable newline with an empty <div><br></div>.
|
||||
// innerText includes the padding break; it is not an extra typed character.
|
||||
const FILL_VALUE_FUNCTION = `function() {
|
||||
if (!this.isContentEditable) return this.value;
|
||||
const value = this.innerText;
|
||||
const tail = this.lastChild;
|
||||
const padding = tail && tail.nodeName === 'DIV' && tail.childNodes.length === 1 && tail.firstChild.nodeName === 'BR';
|
||||
return padding && value.endsWith('\\n\\n') ? value.slice(0, -1) : value;
|
||||
}`;
|
||||
|
||||
export async function handleFill(
|
||||
manager: SessionManager,
|
||||
params: FillParams,
|
||||
@@ -707,8 +658,12 @@ export async function handleFill(
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
await nodeCdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
|
||||
} catch (err) {
|
||||
return fillError("fill_failed", err instanceof Error ? err.message : String(err));
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
message: err instanceof Error ? err.message : String(err),
|
||||
};
|
||||
}
|
||||
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
@@ -721,252 +676,57 @@ export async function handleFill(
|
||||
const clearBefore = params.clear_before ?? true;
|
||||
|
||||
try {
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
try {
|
||||
await nodeCdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
|
||||
} catch (error) {
|
||||
// Chrome rejects DOM.focus for disabled controls, including inherited
|
||||
// fieldset state. Classify the live target before treating this as a
|
||||
// browser failure; the normal path needs no extra round trip.
|
||||
const editable = await nodeCdp.send<FillScriptReply<boolean>>(
|
||||
target.tabId,
|
||||
"Runtime.callFunctionOn",
|
||||
{ objectId, functionDeclaration: FILL_EDITABLE_FUNCTION, returnByValue: true },
|
||||
);
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
if (!editable.exceptionDetails && editable.result?.value === false) {
|
||||
return fillError(
|
||||
"target_not_fillable",
|
||||
"fill target is not editable or its input type is unsupported",
|
||||
);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
// Check editability before clearing. Keep the expected result tied to
|
||||
// this object, including the existing value on the append path.
|
||||
const prepared = await nodeCdp.send<FillScriptReply<FillPreparation>>(
|
||||
target.tabId,
|
||||
"Runtime.callFunctionOn",
|
||||
{
|
||||
if (clearBefore) {
|
||||
// Clear input/textarea value or wipe contenteditable innerText,
|
||||
// then fire `input` so frameworks observe the empty state.
|
||||
await nodeCdp.send(target.tabId, "Runtime.callFunctionOn", {
|
||||
objectId,
|
||||
functionDeclaration: `function(value, clearBefore) {
|
||||
const tag = this.tagName.toLowerCase();
|
||||
const native = tag === 'input' || tag === 'textarea';
|
||||
if (!(${FILL_EDITABLE_FUNCTION}).call(this)) return { reason: 'target_not_fillable', error: 'fill target is not editable or its input type is unsupported' };
|
||||
if (this.getRootNode().activeElement !== this) return { reason: 'fill_focus_lost', error: 'fill target does not have focus' };
|
||||
const before = clearBefore ? '' : (${FILL_VALUE_FUNCTION}).call(this);
|
||||
let expected = before + value;
|
||||
if (native) {
|
||||
// Use the browser's own value sanitization (e.g. textarea
|
||||
// line endings) without changing the live control.
|
||||
const normalizer = this.ownerDocument.createElement(tag);
|
||||
if (tag === 'input') {
|
||||
normalizer.type = this.type;
|
||||
normalizer.multiple = this.multiple;
|
||||
}
|
||||
// insertText treats line breaks in a single-line input as spaces.
|
||||
if (tag === 'input') expected = expected.replace(/\\r\\n|\\r|\\n/g, ' ');
|
||||
normalizer.value = expected;
|
||||
if (expected !== '' && normalizer.value === '' && this.type === 'number') {
|
||||
return { reason: 'fill_value_invalid', error: 'fill value is not valid for a number input' };
|
||||
}
|
||||
expected = normalizer.value;
|
||||
if (this.type !== 'number' && this.maxLength >= 0 && expected.length > this.maxLength) {
|
||||
return { reason: 'fill_value_invalid', error: 'fill value exceeds the target maxlength' };
|
||||
}
|
||||
} else {
|
||||
expected = expected.replace(/\\r\\n?/g, '\\n');
|
||||
functionDeclaration: `function() {
|
||||
if (this.isContentEditable) { this.textContent = ''; }
|
||||
else {
|
||||
const proto = this instanceof HTMLTextAreaElement
|
||||
? HTMLTextAreaElement.prototype
|
||||
: HTMLInputElement.prototype;
|
||||
const descriptor = Object.getOwnPropertyDescriptor(proto, 'value');
|
||||
if (descriptor && descriptor.set) descriptor.set.call(this, '');
|
||||
else this.value = '';
|
||||
}
|
||||
if (clearBefore) {
|
||||
if (native) {
|
||||
const proto = tag === 'textarea' ? HTMLTextAreaElement.prototype : HTMLInputElement.prototype;
|
||||
Object.getOwnPropertyDescriptor(proto, 'value').set.call(this, '');
|
||||
} else {
|
||||
this.textContent = '';
|
||||
}
|
||||
this.dispatchEvent(new Event('input', { bubbles: true }));
|
||||
}
|
||||
return { before, expected };
|
||||
this.dispatchEvent(new Event('input', { bubbles: true }));
|
||||
}`,
|
||||
arguments: [{ value: params.value }, { value: clearBefore }],
|
||||
returnByValue: true,
|
||||
},
|
||||
);
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
if (prepared.exceptionDetails) {
|
||||
return fillError("fill_failed", "fill preparation script failed");
|
||||
}
|
||||
const preparation = prepared.result?.value;
|
||||
if (preparation && "error" in preparation) {
|
||||
return fillError(preparation.reason, preparation.error);
|
||||
}
|
||||
if (typeof preparation?.before !== "string" || typeof preparation?.expected !== "string") {
|
||||
return fillError("fill_failed", "fill preparation returned an unexpected result");
|
||||
}
|
||||
// A separate call runs after the clearing event's microtasks drain.
|
||||
// DOM focus works in background tabs. Use document.hasFocus() only to
|
||||
// detect deferred focus events, never to reject background input.
|
||||
const checkReady = async (): Promise<FillReadiness | RpcError> => {
|
||||
const reply = await nodeCdp.send<FillScriptReply<FillReadiness>>(
|
||||
target.tabId,
|
||||
"Runtime.callFunctionOn",
|
||||
{
|
||||
objectId,
|
||||
functionDeclaration: `function(before) {
|
||||
if (!(${FILL_EDITABLE_FUNCTION}).call(this) ||
|
||||
(${FILL_VALUE_FUNCTION}).call(this) !== before) return 'fill_target_changed';
|
||||
if (this.getRootNode().activeElement !== this) return 'fill_focus_lost';
|
||||
return this.ownerDocument.hasFocus() ? 'ready' : 'background';
|
||||
}`,
|
||||
arguments: [{ value: preparation.before }],
|
||||
returnByValue: true,
|
||||
},
|
||||
);
|
||||
const state = reply.result?.value;
|
||||
if (
|
||||
reply.exceptionDetails ||
|
||||
(state !== "ready" &&
|
||||
state !== "background" &&
|
||||
state !== "fill_target_changed" &&
|
||||
state !== "fill_focus_lost")
|
||||
) {
|
||||
return fillError(
|
||||
"fill_failed",
|
||||
"fill readiness script failed or returned an unexpected result",
|
||||
);
|
||||
}
|
||||
return state;
|
||||
};
|
||||
let ready = await checkReady();
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
// Restore focus once only if clearing left this same target editable
|
||||
// and unchanged. Recheck after focus handlers have run.
|
||||
if (clearBefore && ready === "fill_focus_lost") {
|
||||
await nodeCdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
ready = await checkReady();
|
||||
});
|
||||
}
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
if (typeof ready !== "string") return ready;
|
||||
if (ready !== "ready" && ready !== "background") {
|
||||
return fillError(ready, "fill target changed or lost focus before typing");
|
||||
}
|
||||
if (params.value !== "" && (!clearBefore || ready === "background")) {
|
||||
// Native editing commands also support number/email inputs, whose
|
||||
// selection APIs cannot set a caret, and multiline contenteditables.
|
||||
// Also do this for background replacement: CDP input focuses the
|
||||
// renderer, delivering deferred focus events before it inserts text.
|
||||
// Use a command-only event so this does not invoke an End shortcut.
|
||||
try {
|
||||
await deps.cdp.send(target.tabId, "Input.dispatchKeyEvent", {
|
||||
type: "rawKeyDown",
|
||||
commands: ["moveToEndOfDocument"],
|
||||
});
|
||||
} finally {
|
||||
await deps.cdp.send(target.tabId, "Input.dispatchKeyEvent", { type: "keyUp" });
|
||||
}
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
// Focus and key handlers can change the target or move focus too.
|
||||
ready = await checkReady();
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
if (typeof ready !== "string") return ready;
|
||||
if (ready !== "ready" && ready !== "background") {
|
||||
return fillError(ready, "fill target changed or lost focus while positioning the caret");
|
||||
}
|
||||
}
|
||||
// CDP `Input.insertText` handles IME / multi-byte input out of the
|
||||
// box, much more reliably than per-key `dispatchKeyEvent`.
|
||||
if (params.value !== "") {
|
||||
await deps.cdp.send(target.tabId, "Input.insertText", { text: params.value });
|
||||
}
|
||||
await deps.cdp.send(target.tabId, "Input.insertText", { text: params.value });
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
// Keep the existing notifications, then read in a separate call so
|
||||
// nested microtasks queued by the page cannot race the verification.
|
||||
const notified = await nodeCdp.send<FillScriptReply<unknown>>(
|
||||
target.tabId,
|
||||
"Runtime.callFunctionOn",
|
||||
{
|
||||
objectId,
|
||||
functionDeclaration: `function() {
|
||||
this.dispatchEvent(new Event('input', { bubbles: true }));
|
||||
this.dispatchEvent(new Event('change', { bubbles: true }));
|
||||
}`,
|
||||
returnByValue: true,
|
||||
},
|
||||
);
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
if (notified.exceptionDetails) {
|
||||
return fillError("fill_failed", "fill notification script failed");
|
||||
}
|
||||
const verified = await nodeCdp.send<
|
||||
FillScriptReply<{ connected: boolean; matches: boolean; valueLength: number }>
|
||||
>(target.tabId, "Runtime.callFunctionOn", {
|
||||
// Fire `input` + `change` so React / Vue controlled inputs commit.
|
||||
await nodeCdp.send(target.tabId, "Runtime.callFunctionOn", {
|
||||
objectId,
|
||||
functionDeclaration: `function(expected) {
|
||||
const value = (${FILL_VALUE_FUNCTION}).call(this);
|
||||
return { connected: this.isConnected, matches: value === expected, valueLength: value.length };
|
||||
}`,
|
||||
arguments: [{ value: preparation.expected }],
|
||||
functionDeclaration: `function() {
|
||||
this.dispatchEvent(new Event('input', { bubbles: true }));
|
||||
this.dispatchEvent(new Event('change', { bubbles: true }));
|
||||
}`,
|
||||
returnByValue: true,
|
||||
});
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
const verification = verified.result?.value;
|
||||
if (
|
||||
verified.exceptionDetails ||
|
||||
typeof verification?.connected !== "boolean" ||
|
||||
typeof verification?.matches !== "boolean" ||
|
||||
typeof verification?.valueLength !== "number"
|
||||
) {
|
||||
return fillError(
|
||||
"fill_failed",
|
||||
"fill verification script failed or returned an unexpected result",
|
||||
);
|
||||
}
|
||||
if (!verification.connected) {
|
||||
return fillError("fill_target_changed", "fill target was removed or replaced during input");
|
||||
}
|
||||
if (!verification.matches) {
|
||||
return fillError(
|
||||
"fill_value_mismatch",
|
||||
"fill could not verify the expected value; observe the page before deciding whether to retry",
|
||||
);
|
||||
}
|
||||
return attachDialogs(deps.cdp, target.tabId, dialogCursor, {
|
||||
tab_id: target.tabId,
|
||||
used_ref: node.usedRef,
|
||||
used_selector: node.usedSelector,
|
||||
value_length: verification.valueLength,
|
||||
});
|
||||
} catch (err) {
|
||||
return fillError("fill_failed", err instanceof Error ? err.message : String(err));
|
||||
} finally {
|
||||
await nodeCdp.send(target.tabId, "Runtime.releaseObject", { objectId }).catch(() => undefined);
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
message: err instanceof Error ? err.message : String(err),
|
||||
};
|
||||
}
|
||||
|
||||
return attachDialogs(deps.cdp, target.tabId, dialogCursor, {
|
||||
tab_id: target.tabId,
|
||||
used_ref: node.usedRef,
|
||||
used_selector: node.usedSelector,
|
||||
value_length: params.value.length,
|
||||
});
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
@@ -28,13 +28,7 @@ import type {
|
||||
RpcError,
|
||||
WaitUntil,
|
||||
} from "@/transport/types";
|
||||
import {
|
||||
type BrowserNavigationApi,
|
||||
chromeBrowserNavigationApi,
|
||||
navigateWithBrowserApi,
|
||||
} from "./browser-navigation";
|
||||
import { attachDialogs, markDialogCursor } from "./dialogs";
|
||||
import { cdpError, isCdpExtensionAccessDenied } from "./errors";
|
||||
import {
|
||||
type CdpRunner,
|
||||
type ChromeTabsApi,
|
||||
@@ -48,7 +42,6 @@ import {
|
||||
export interface NavigationDeps {
|
||||
cdp: CdpRunner;
|
||||
tabsApi: ChromeTabsApi;
|
||||
browserNavigation?: BrowserNavigationApi;
|
||||
/** Optional AbortSignal — M7 abort hook (M10.2 will wire the full chain). */
|
||||
signal?: AbortSignal;
|
||||
/** Override default timeout when the caller omits `timeout_ms`. */
|
||||
@@ -486,81 +479,6 @@ async function readTabUrl(api: ChromeTabsApi, tabId: number): Promise<string | u
|
||||
}
|
||||
}
|
||||
|
||||
/** Recover only a failed preflight; never replay an already-dispatched navigation. */
|
||||
async function recoverBrowserNavigation(
|
||||
deps: NavigationDeps,
|
||||
tabId: number,
|
||||
action: (api: BrowserNavigationApi) => Promise<unknown>,
|
||||
waitUntil: WaitUntil,
|
||||
timeoutMs: number,
|
||||
): Promise<ReloadResult | RpcError> {
|
||||
const abort = linkedAbortSignal(deps.signal);
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
const api = deps.browserNavigation ?? chromeBrowserNavigationApi;
|
||||
try {
|
||||
let outcome = await navigateWithBrowserApi(
|
||||
api,
|
||||
tabId,
|
||||
() => action(api),
|
||||
waitUntil === "networkidle" ? "commit" : waitUntil,
|
||||
timeoutMs,
|
||||
abort.signal,
|
||||
);
|
||||
if (outcome.reached === "failed") return outcome.error;
|
||||
if (outcome.reached === "cancelled" || abort.signal.aborted) {
|
||||
return { code: "cancelled", message: "navigation recovery aborted" };
|
||||
}
|
||||
if (outcome.reached === "match") {
|
||||
// A completed reload is not a recovery if the restricted frame remains.
|
||||
await deps.cdp.send(tabId, "Page.enable", {});
|
||||
if (abort.signal.aborted)
|
||||
return { code: "cancelled", message: "navigation recovery aborted" };
|
||||
if (waitUntil === "networkidle") {
|
||||
// webNavigation cannot prove network idle. Subscribe on the new
|
||||
// document before enabling CDP lifecycle events after its commit.
|
||||
const frame = await readMainFrameInfo(deps.cdp, tabId);
|
||||
if (!frame.frameId) return cdpError("no main frame after navigation");
|
||||
const remaining = deadline - Date.now();
|
||||
if (remaining <= 0) {
|
||||
outcome = { reached: "timeout", lastLifecycle: "commit" };
|
||||
} else {
|
||||
const wait = startLifecycleWait(
|
||||
deps.cdp,
|
||||
tabId,
|
||||
frame.frameId,
|
||||
"networkIdle",
|
||||
remaining,
|
||||
abort.signal,
|
||||
{ loaderId: frame.loaderId ?? undefined },
|
||||
);
|
||||
await deps.cdp.send(tabId, "Page.setLifecycleEventsEnabled", { enabled: true });
|
||||
outcome = await wait.promise;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (outcome.reached === "cancelled" || abort.signal.aborted) {
|
||||
return { code: "cancelled", message: "navigation recovery aborted" };
|
||||
}
|
||||
return {
|
||||
tab_id: tabId,
|
||||
final_url: await readTabUrl(deps.tabsApi, tabId),
|
||||
reached: outcome.reached === "match" ? waitUntil : "timeout",
|
||||
...(outcome.reached === "timeout"
|
||||
? {
|
||||
error_text: `timed out waiting for ${waitUntil} during navigation recovery after ${timeoutMs}ms`,
|
||||
}
|
||||
: {}),
|
||||
};
|
||||
} catch (error) {
|
||||
return abort.signal.aborted
|
||||
? { code: "cancelled", message: "navigation recovery aborted" }
|
||||
: cdpError(error);
|
||||
} finally {
|
||||
abort.abort();
|
||||
abort.cleanup();
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// tool.navigate
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -587,20 +505,7 @@ export async function handleNavigate(
|
||||
|
||||
try {
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
try {
|
||||
await ensureCdpReady(deps.cdp, target.tabId);
|
||||
} catch (error) {
|
||||
if (!isCdpExtensionAccessDenied(error)) throw error;
|
||||
const recovered = await recoverBrowserNavigation(
|
||||
deps,
|
||||
target.tabId,
|
||||
(api) => api.update(target.tabId, { url: params.url }),
|
||||
waitUntil,
|
||||
timeoutMs,
|
||||
);
|
||||
if (isRpcError(recovered)) return recovered;
|
||||
return attachDialogs(deps.cdp, target.tabId, dialogCursor, { ...recovered, url: params.url });
|
||||
}
|
||||
await ensureCdpReady(deps.cdp, target.tabId);
|
||||
const expected = cdpLifecycleName(waitUntil);
|
||||
const beforeReadyState = await probeMainFrameReadyState(deps.cdp, target.tabId);
|
||||
const beforeFrame = await readMainFrameInfo(deps.cdp, target.tabId);
|
||||
@@ -669,7 +574,10 @@ export async function handleNavigate(
|
||||
}`,
|
||||
});
|
||||
} catch (err) {
|
||||
return cdpError(err);
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
message: err instanceof Error ? err.message : String(err),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@@ -819,24 +727,7 @@ export async function handleReload(
|
||||
|
||||
try {
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
try {
|
||||
await ensureCdpReady(deps.cdp, target.tabId);
|
||||
} catch (error) {
|
||||
if (!isCdpExtensionAccessDenied(error)) throw error;
|
||||
const previousUrl = await readTabUrl(deps.tabsApi, target.tabId);
|
||||
const recovered = await recoverBrowserNavigation(
|
||||
deps,
|
||||
target.tabId,
|
||||
(api) => api.reload(target.tabId, { bypassCache: ignoreCache }),
|
||||
waitUntil,
|
||||
timeoutMs,
|
||||
);
|
||||
if (isRpcError(recovered)) return recovered;
|
||||
return attachDialogs(deps.cdp, target.tabId, dialogCursor, {
|
||||
...recovered,
|
||||
previous_url: previousUrl,
|
||||
});
|
||||
}
|
||||
await ensureCdpReady(deps.cdp, target.tabId);
|
||||
const previousUrl = await readTabUrl(deps.tabsApi, target.tabId);
|
||||
const beforeFrame = await readMainFrameInfo(deps.cdp, target.tabId);
|
||||
const expected = cdpLifecycleName(waitUntil);
|
||||
@@ -885,7 +776,10 @@ export async function handleReload(
|
||||
}`,
|
||||
});
|
||||
} catch (err) {
|
||||
return cdpError(err);
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
message: err instanceof Error ? err.message : String(err),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
import type { CapturedSceneInput } from "./vom/facts";
|
||||
// Observation handlers — `tool.snapshot`, `tool.get_html`, `tool.screenshot`,
|
||||
// and semantic `tool.observe` (design §7). Each handler resolves the target
|
||||
// tab (defaulting to the Agent Window's active tab when omitted) and
|
||||
@@ -18,7 +17,7 @@ import { ChromiumCdp } from "@/browser-driver/chromium-cdp";
|
||||
import type { CdpTarget } from "@/browser-driver/frame-graph";
|
||||
import {
|
||||
type CaptureSuppressSendToTab,
|
||||
withExtensionOverlayHidden,
|
||||
withOverlaysHiddenForCapture,
|
||||
} from "@/lib/capture-suppress-bridge";
|
||||
import type { SessionContext, SessionManager } from "@/session-manager/manager";
|
||||
import type {
|
||||
@@ -35,7 +34,6 @@ import type {
|
||||
import { attachDialogs, markDialogCursor } from "./dialogs";
|
||||
import { rpcError } from "./errors";
|
||||
import { resolveNodeGeometry } from "./frame-geometry";
|
||||
import { screenshotPageRect } from "./geometry/coordinate-types";
|
||||
import {
|
||||
type ChromeTabsApi,
|
||||
enforceToolTargetScope,
|
||||
@@ -49,9 +47,15 @@ import {
|
||||
type ToolEffect,
|
||||
} from "./shared";
|
||||
import { resolveSnapshotRef } from "./snapshot-ref";
|
||||
import { type CapturedNode, type CapturedSurfaceProbe, probeHoverSurfaces } from "./vom/capture";
|
||||
import { captureObservationFacts, semanticCapture } from "./vom/capture-coordinator";
|
||||
import type { FrameDocument as CapturedFrameDocument } from "./vom/frame-document";
|
||||
import {
|
||||
type CapturedNode,
|
||||
type CapturedSurfaceProbe,
|
||||
type CapturedViewModel,
|
||||
captureViewModel,
|
||||
collectOverlayExcludedBackendIds,
|
||||
probeHoverSurfaces,
|
||||
} from "./vom/capture";
|
||||
import { type CapturedFrameDocument, captureFrameData } from "./vom/frame-capture";
|
||||
import { withOverlayBypass } from "./vom/hover-perception";
|
||||
import { probeTooltipNames } from "./vom/name-enrichment";
|
||||
import {
|
||||
@@ -193,14 +197,15 @@ async function captureElementScreenshot(
|
||||
if (isRpcError(geometry)) return geometry;
|
||||
if (signal?.aborted) return cancelled("screenshot");
|
||||
const rect = geometry.topBounds;
|
||||
const clip = screenshotPageRect(rect, geometry.topViewport);
|
||||
if (!clip) return { code: "cdp_failed", message: "invalid screenshot coordinate space" };
|
||||
|
||||
try {
|
||||
const shot = await cdp.send<{ data?: string }>(tabId, "Page.captureScreenshot", {
|
||||
format: "png",
|
||||
clip: {
|
||||
...clip.rect,
|
||||
x: rect.x,
|
||||
y: rect.y,
|
||||
width: rect.width,
|
||||
height: rect.height,
|
||||
scale: 1,
|
||||
},
|
||||
});
|
||||
@@ -319,7 +324,7 @@ export async function handleScreenshot(
|
||||
tabId: target.tabId,
|
||||
...(node.cdpSessionId ? { sessionId: node.cdpSessionId } : {}),
|
||||
};
|
||||
const captured = await withExtensionOverlayHidden(
|
||||
const captured = await withOverlaysHiddenForCapture(
|
||||
target.tabId,
|
||||
() =>
|
||||
captureElementScreenshot(
|
||||
@@ -351,7 +356,7 @@ export async function handleScreenshot(
|
||||
);
|
||||
}
|
||||
|
||||
const captured = await withExtensionOverlayHidden(
|
||||
const captured = await withOverlaysHiddenForCapture(
|
||||
target.tabId,
|
||||
() => captureFullTabPng(deps, ctx, target, signal),
|
||||
deps.sendToTab,
|
||||
@@ -525,7 +530,7 @@ function buildActiveScopeBlocks(nodes: VomNode[], signals: VomNodeDomSignals): A
|
||||
|
||||
function buildConditionalSurfaces(
|
||||
nodes: VomNode[],
|
||||
captured: CapturedSceneInput,
|
||||
captured: CapturedViewModel,
|
||||
probes: CapturedSurfaceProbe[],
|
||||
): CondSurface[] {
|
||||
if (probes.length === 0) return [];
|
||||
@@ -655,9 +660,161 @@ export interface BuildVomSceneOptions {
|
||||
|
||||
export type VomFrameDocument = CapturedFrameDocument<CdpAxNode>;
|
||||
|
||||
function legacyFrameDocuments(
|
||||
axNodes: CdpAxNode[],
|
||||
captured: CapturedViewModel,
|
||||
pageUrl?: string,
|
||||
rootTarget: VomFrameDocument["target"] = { tabId: 0 },
|
||||
): VomFrameDocument[] {
|
||||
const rootFrameId = captured.rootFrameId ?? "root";
|
||||
const documents: VomFrameDocument[] = [
|
||||
{
|
||||
frameId: rootFrameId,
|
||||
contextScopeId: rootFrameId,
|
||||
target: rootTarget,
|
||||
...(pageUrl ? { url: pageUrl } : {}),
|
||||
axNodes: [],
|
||||
domNodes: captured.nodes.map((node) => ({ ...node, frameId: node.frameId ?? rootFrameId })),
|
||||
},
|
||||
];
|
||||
const pending = [...captured.iframeNodes.entries()];
|
||||
let progress = true;
|
||||
let nextSyntheticFrame = 1;
|
||||
while (pending.length > 0 && progress) {
|
||||
progress = false;
|
||||
for (let index = pending.length - 1; index >= 0; index -= 1) {
|
||||
const [ownerBackendNodeId, domNodes] = pending[index];
|
||||
const parent = documents.find((document) =>
|
||||
document.domNodes.some((node) => node.backendNodeId === ownerBackendNodeId),
|
||||
);
|
||||
if (!parent) continue;
|
||||
const frameId =
|
||||
domNodes.find((node) => node.frameId)?.frameId ?? `legacy-frame-${nextSyntheticFrame++}`;
|
||||
documents.push({
|
||||
frameId,
|
||||
parentFrameId: parent.frameId,
|
||||
ownerBackendNodeId,
|
||||
contextScopeId: frameId,
|
||||
target: parent.target,
|
||||
axNodes: [],
|
||||
domNodes: domNodes.map((node) => ({ ...node, frameId: node.frameId ?? frameId })),
|
||||
});
|
||||
pending.splice(index, 1);
|
||||
progress = true;
|
||||
}
|
||||
}
|
||||
|
||||
const documentByFrameId = new Map(documents.map((document) => [document.frameId, document]));
|
||||
const backendFrame = new Map<number, string>();
|
||||
const childFrameByOwner = new Map<number, string>();
|
||||
for (const document of documents) {
|
||||
if (document.ownerBackendNodeId !== undefined) {
|
||||
childFrameByOwner.set(document.ownerBackendNodeId, document.frameId);
|
||||
}
|
||||
for (const node of document.domNodes) backendFrame.set(node.backendNodeId, document.frameId);
|
||||
}
|
||||
const axById = new Map(axNodes.map((node) => [node.nodeId, node]));
|
||||
const ownership = new Map<string, string>();
|
||||
const resolving = new Set<string>();
|
||||
const frameForAx = (node: CdpAxNode): string => {
|
||||
const cached = ownership.get(node.nodeId);
|
||||
if (cached) return cached;
|
||||
let frameId: string | undefined;
|
||||
if (node.frameId && documentByFrameId.has(node.frameId)) frameId = node.frameId;
|
||||
if (!frameId && typeof node.backendDOMNodeId === "number") {
|
||||
frameId = backendFrame.get(node.backendDOMNodeId);
|
||||
}
|
||||
if (!frameId && node.parentId && !resolving.has(node.nodeId)) {
|
||||
const parent = axById.get(node.parentId);
|
||||
if (parent) {
|
||||
frameId =
|
||||
typeof parent.backendDOMNodeId === "number"
|
||||
? childFrameByOwner.get(parent.backendDOMNodeId)
|
||||
: undefined;
|
||||
if (!frameId) {
|
||||
resolving.add(node.nodeId);
|
||||
frameId = frameForAx(parent);
|
||||
resolving.delete(node.nodeId);
|
||||
}
|
||||
}
|
||||
}
|
||||
frameId ??= rootFrameId;
|
||||
ownership.set(node.nodeId, frameId);
|
||||
return frameId;
|
||||
};
|
||||
for (const node of axNodes) frameForAx(node);
|
||||
for (const node of axNodes) {
|
||||
const frameId = ownership.get(node.nodeId) ?? rootFrameId;
|
||||
const document = documentByFrameId.get(frameId) ?? documents[0];
|
||||
document.axNodes.push({
|
||||
...node,
|
||||
frameId,
|
||||
...(node.parentId && ownership.get(node.parentId) === frameId
|
||||
? { parentId: node.parentId }
|
||||
: { parentId: undefined }),
|
||||
...(node.childIds
|
||||
? { childIds: node.childIds.filter((childId) => ownership.get(childId) === frameId) }
|
||||
: {}),
|
||||
});
|
||||
}
|
||||
return documents;
|
||||
}
|
||||
|
||||
function withLegacyBackendIds(scene: VomScene): VomScene {
|
||||
const used = new Set<number>();
|
||||
const idMap = new Map<number, number>();
|
||||
let nextVirtualId = -1;
|
||||
for (const node of scene.nodes) {
|
||||
const preferred = node.backendNodeId;
|
||||
const id = preferred !== undefined && !used.has(preferred) ? preferred : nextVirtualId--;
|
||||
used.add(id);
|
||||
idMap.set(node.id, id);
|
||||
}
|
||||
return {
|
||||
...scene,
|
||||
nodes: scene.nodes.map((node) => ({
|
||||
...node,
|
||||
id: idMap.get(node.id) as number,
|
||||
parentId: node.parentId === null ? null : (idMap.get(node.parentId) ?? null),
|
||||
...(node.domParentId !== undefined
|
||||
? { domParentId: node.domParentId === null ? null : (idMap.get(node.domParentId) ?? null) }
|
||||
: {}),
|
||||
...(node.domAncestorIds
|
||||
? { domAncestorIds: node.domAncestorIds.flatMap((id) => idMap.get(id) ?? []) }
|
||||
: {}),
|
||||
})),
|
||||
...(scene.surfaces
|
||||
? {
|
||||
surfaces: scene.surfaces.map((surface) => ({
|
||||
...surface,
|
||||
triggerId: idMap.get(surface.triggerId) ?? surface.triggerId,
|
||||
})),
|
||||
}
|
||||
: {}),
|
||||
...(scene.activeScopeBlocks
|
||||
? {
|
||||
activeScopeBlocks: scene.activeScopeBlocks.map((block) => ({
|
||||
...block,
|
||||
triggerId: idMap.get(block.triggerId) ?? block.triggerId,
|
||||
})),
|
||||
}
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
|
||||
export function buildVomScene(
|
||||
axNodes: CdpAxNode[],
|
||||
captured: CapturedViewModel,
|
||||
options: BuildVomSceneOptions = {},
|
||||
): VomScene {
|
||||
return withLegacyBackendIds(
|
||||
buildFrameVomScene(legacyFrameDocuments(axNodes, captured, options.pageUrl), captured, options),
|
||||
);
|
||||
}
|
||||
|
||||
export function buildFrameVomScene(
|
||||
documents: VomFrameDocument[],
|
||||
captured: CapturedSceneInput,
|
||||
captured: CapturedViewModel,
|
||||
options: BuildVomSceneOptions = {},
|
||||
): VomScene {
|
||||
const scene = buildSemanticVomScene({
|
||||
@@ -673,7 +830,7 @@ export function buildFrameVomScene(
|
||||
function attachCapturedSceneAnnotations(
|
||||
scene: VomScene,
|
||||
documents: VomFrameDocument[],
|
||||
captured: CapturedSceneInput,
|
||||
captured: CapturedViewModel,
|
||||
surfaceProbes: CapturedSurfaceProbe[],
|
||||
): VomScene {
|
||||
const rootDocument = documents.find((document) => document.frameId === scene.rootFrameId);
|
||||
@@ -823,6 +980,51 @@ export async function handleGetHtml(
|
||||
}
|
||||
}
|
||||
|
||||
function emptyCapturedViewModel(viewport = { width: 0, height: 0 }): CapturedViewModel {
|
||||
return { viewport, nodes: [], iframeNodes: new Map(), excludedBackendNodeIds: new Set() };
|
||||
}
|
||||
|
||||
interface LayoutMetricsViewportReply {
|
||||
cssLayoutViewport?: { clientWidth?: number; clientHeight?: number };
|
||||
layoutViewport?: { clientWidth?: number; clientHeight?: number };
|
||||
}
|
||||
|
||||
async function fallbackCapturedViewModel(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
signal?: AbortSignal,
|
||||
): Promise<CapturedViewModel> {
|
||||
throwIfAborted(signal, "observation");
|
||||
let viewport = { width: 0, height: 0 };
|
||||
try {
|
||||
const metrics = await cdp.send<LayoutMetricsViewportReply>(tabId, "Page.getLayoutMetrics", {});
|
||||
throwIfAborted(signal, "observation");
|
||||
const source = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
|
||||
viewport = {
|
||||
width: source.clientWidth ?? 0,
|
||||
height: source.clientHeight ?? 0,
|
||||
};
|
||||
} catch (error) {
|
||||
if (isAbortError(error)) throw error;
|
||||
}
|
||||
const excludedBackendNodeIds = await collectOverlayExcludedBackendIds(cdp, tabId, signal);
|
||||
throwIfAborted(signal, "observation");
|
||||
return { ...emptyCapturedViewModel(viewport), excludedBackendNodeIds };
|
||||
}
|
||||
|
||||
async function captureForVom(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
options: CaptureVomObservationOptions,
|
||||
): Promise<CapturedViewModel> {
|
||||
try {
|
||||
return await captureViewModel(cdp, tabId, { signal: options.signal });
|
||||
} catch (error) {
|
||||
if (isAbortError(error)) throw error;
|
||||
return fallbackCapturedViewModel(cdp, tabId, options.signal);
|
||||
}
|
||||
}
|
||||
|
||||
export interface HoverProbeOutcome {
|
||||
/** Whether any active hover was dispatched during this observation. */
|
||||
performed: boolean;
|
||||
@@ -855,7 +1057,7 @@ const NO_HOVER_PROBES: HoverProbeOutcome = {
|
||||
async function runHoverProbes(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
captured: CapturedSceneInput,
|
||||
captured: CapturedViewModel,
|
||||
documents: VomFrameDocument[],
|
||||
staticSemantics: ReturnType<typeof resolveSemanticGraph>,
|
||||
options: CaptureVomObservationOptions,
|
||||
@@ -894,9 +1096,14 @@ export async function captureVomObservation(
|
||||
throwIfAborted(options.signal, "observation");
|
||||
await cdp.ensureAttachedToUrl?.(tabId, url);
|
||||
throwIfAborted(options.signal, "observation");
|
||||
const facts = await captureObservationFacts<CdpAxNode>(cdp, tabId, options.signal, url);
|
||||
const { captured, documents: normalizedDocuments } = semanticCapture(facts);
|
||||
const captured = await captureForVom(cdp, tabId, options);
|
||||
throwIfAborted(options.signal, "observation");
|
||||
const documents = await captureFrameData<CdpAxNode>(cdp, tabId, captured, options.signal);
|
||||
throwIfAborted(options.signal, "observation");
|
||||
const normalizedDocuments =
|
||||
documents.length === 1 && captured.iframeNodes.size > 0
|
||||
? legacyFrameDocuments(documents[0].axNodes, captured, url, documents[0].target)
|
||||
: documents;
|
||||
const semanticGraph = buildSemanticGraph({
|
||||
documents: normalizedDocuments,
|
||||
viewport: captured.viewport,
|
||||
@@ -924,43 +1131,12 @@ export async function captureVomObservation(
|
||||
captured,
|
||||
hoverProbes.surfaceProbes,
|
||||
);
|
||||
// Reserve space using the renderer's character-based token estimate. Like VOM
|
||||
// headers, this integrity notice remains visible even under a tiny token budget.
|
||||
const notices: string[] = [];
|
||||
if (facts.issues.some((issue) => issue.stage === "geometry"))
|
||||
notices.push(
|
||||
"@warning iframe geometry incomplete: some frame content has no top-level coordinates.",
|
||||
);
|
||||
const incompleteStages = ["dom", "ax", "forms", "ownership"].filter((stage) =>
|
||||
facts.issues.some((issue) => issue.stage === stage),
|
||||
);
|
||||
if (incompleteStages.length)
|
||||
notices.push(
|
||||
`@warning observation incomplete: some ${incompleteStages.join(", ")} data is unavailable or omitted.`,
|
||||
);
|
||||
if (
|
||||
facts.issues.some(
|
||||
(issue) => issue.stage === "identity" && issue.reason !== "identity-unverified",
|
||||
)
|
||||
)
|
||||
notices.push(
|
||||
"@warning observation incomplete: some documents were omitted because their identity changed or could not be revalidated.",
|
||||
);
|
||||
if (facts.issues.some((issue) => issue.reason === "identity-unverified"))
|
||||
notices.push(
|
||||
"@warning document identity unverified: some retained documents could not be checked for changes during capture.",
|
||||
);
|
||||
const captureNotice = notices.join("\n");
|
||||
const rendered = renderVom(decoratedScene, {
|
||||
maxDepth: options.maxDepth,
|
||||
maxTokens:
|
||||
!captureNotice || options.maxTokens === undefined
|
||||
? options.maxTokens
|
||||
: Math.max(0, options.maxTokens - Math.ceil((captureNotice.length + 1) / 4)),
|
||||
maxTokens: options.maxTokens,
|
||||
redactValues: options.redactValues,
|
||||
activeRegionPolicy: options.activeRegionPolicy,
|
||||
});
|
||||
if (captureNotice) rendered.text += `\n${captureNotice}`;
|
||||
throwIfAborted(options.signal, "observation");
|
||||
return projectRecordSafeObservation({
|
||||
rootFrameId: captured.rootFrameId ?? normalizedDocuments[0]?.frameId ?? "root",
|
||||
|
||||
@@ -2,7 +2,6 @@ import { type SessionManager, SessionStartCleanupError } from "@/session-manager
|
||||
import type { RpcError } from "@/transport/types";
|
||||
import { rpcError } from "./errors";
|
||||
import { clearRecordingForSession } from "./record";
|
||||
import type { CdpRunner, ChromeTabsApi } from "./shared";
|
||||
import { isRpcError } from "./shared";
|
||||
import { returnBorrowedTab, type TabManagementDeps } from "./tabs";
|
||||
|
||||
@@ -74,17 +73,11 @@ export interface SessionStopResult {
|
||||
/** Tab ids whose return path failed; those entries remain borrowed so
|
||||
* shutdown can be retried without closing the Agent Window. */
|
||||
return_failures?: Array<{ tab_id: number; code: string; message: string }>;
|
||||
/**
|
||||
* True when the Agent Window was *released* to the user (instead of being
|
||||
* closed) because it still contained user-created tabs after the agent's own
|
||||
* tabs were closed. Single-session scope.
|
||||
*/
|
||||
window_released?: boolean;
|
||||
}
|
||||
|
||||
export interface SessionStopDeps {
|
||||
signal?: AbortSignal;
|
||||
cdp?: Pick<CdpRunner, "releaseSessionTab"> & {
|
||||
cdp?: {
|
||||
detachSession(sessionId: string): Promise<void>;
|
||||
};
|
||||
/**
|
||||
@@ -94,13 +87,6 @@ export interface SessionStopDeps {
|
||||
* without a real browser.
|
||||
*/
|
||||
tabManagement?: TabManagementDeps;
|
||||
/**
|
||||
* Read-only Chrome tabs API used to *query* remaining tabs before deciding
|
||||
* whether to release the window. Kept separate from `tabManagement.tabs`
|
||||
* (a `TabMutationApi` that has no `query`) so the mutation interface stays
|
||||
* pure. Tests can inject a fake `ChromeTabsApi` too.
|
||||
*/
|
||||
tabsQuery?: ChromeTabsApi;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -166,10 +152,8 @@ export async function handleSessionStart(
|
||||
* cleanly (review parity with M6).
|
||||
* 3. Detach CDP sessions the extension still holds for this
|
||||
* session (no-op if M6/M7 didn't attach to any tab).
|
||||
* 4. Close the agent-created tabs and the home tab (design §3.2).
|
||||
* 5. If any tab remains — those are user-created tabs — release the
|
||||
* window to the user (dropOnly) instead of closing it, so user
|
||||
* tabs survive. Otherwise close the window as before.
|
||||
* 4. Close the Agent Window. SessionManager.stop() removes the
|
||||
* Chrome window and forgets the context.
|
||||
*
|
||||
* Failures in step 1 keep the Agent Window open: a failed borrowed tab
|
||||
* may still be there, so closing the window would risk losing user
|
||||
@@ -204,19 +188,15 @@ export async function handleSessionStop(
|
||||
const returnFailures: SessionStopResult["return_failures"] = [];
|
||||
const borrowedIds = Array.from(ctx.borrowedTabs.keys());
|
||||
for (const tabId of borrowedIds) {
|
||||
// A close event may have removed this borrow while another tab returned.
|
||||
if (!ctx.borrowedTabs.has(tabId)) continue;
|
||||
try {
|
||||
const tabManagement = {
|
||||
...(deps.tabManagement ?? {}),
|
||||
cdp: deps.cdp ?? deps.tabManagement?.cdp,
|
||||
signal: deps.signal,
|
||||
isAgentWindowId:
|
||||
deps.tabManagement?.isAgentWindowId ??
|
||||
((windowId: number) => manager.findByWindowId(windowId) !== null),
|
||||
};
|
||||
const outcome = await returnBorrowedTab(ctx, tabId, tabManagement);
|
||||
if (!ctx.borrowedTabs.has(tabId)) continue;
|
||||
if (typeof outcome === "object" && "code" in outcome) {
|
||||
console.warn(`[bsk session_stop] auto-return failed for tab ${tabId}`, outcome);
|
||||
returnFailures?.push({
|
||||
@@ -229,7 +209,6 @@ export async function handleSessionStop(
|
||||
ctx.borrowedTabs.delete(tabId);
|
||||
}
|
||||
} catch (err) {
|
||||
if (!ctx.borrowedTabs.has(tabId)) continue;
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
console.warn(`[bsk session_stop] auto-return threw for tab ${tabId}: ${message}`);
|
||||
returnFailures?.push({
|
||||
@@ -240,15 +219,8 @@ export async function handleSessionStop(
|
||||
}
|
||||
}
|
||||
|
||||
// A previously failed tab may have closed during a later return. Only
|
||||
// confirmed close events remove borrows; ordinary move failures stay retryable.
|
||||
const pendingReturnFailures = returnFailures.filter((failure) =>
|
||||
ctx.borrowedTabs.has(failure.tab_id),
|
||||
);
|
||||
if (deps.signal?.aborted) {
|
||||
const nonCancelFailures = pendingReturnFailures.filter(
|
||||
(failure) => failure.code !== "cancelled",
|
||||
);
|
||||
const nonCancelFailures = returnFailures?.filter((failure) => failure.code !== "cancelled");
|
||||
if (nonCancelFailures && nonCancelFailures.length > 0) {
|
||||
return {
|
||||
...(returnedTabIds.length > 0 ? { returned_tab_ids: returnedTabIds } : {}),
|
||||
@@ -260,8 +232,8 @@ export async function handleSessionStop(
|
||||
|
||||
const result: SessionStopResult = {};
|
||||
if (returnedTabIds.length > 0) result.returned_tab_ids = returnedTabIds;
|
||||
if (pendingReturnFailures.length > 0) {
|
||||
result.return_failures = pendingReturnFailures;
|
||||
if (returnFailures && returnFailures.length > 0) {
|
||||
result.return_failures = returnFailures;
|
||||
// A failed return means at least one borrowed user tab may still be
|
||||
// inside the Agent Window. Keep the session/window alive so the user
|
||||
// can retry `bsk session stop` or explicitly `bsk tab return` after the
|
||||
@@ -281,66 +253,8 @@ export async function handleSessionStop(
|
||||
return { code: "cancelled", message: "session_stop aborted before window close" };
|
||||
}
|
||||
|
||||
// Step 4: close every tab explicitly created by the agent, including the
|
||||
// home tab. `tabsApi` is a
|
||||
// TabMutationApi (remove only); `queryApi` is a separate read-only
|
||||
// ChromeTabsApi. If neither is injected, we conservatively fall back to
|
||||
// closing the window (see Step 5).
|
||||
const tabsApi = deps.tabManagement?.tabs;
|
||||
const queryApi = deps.tabsQuery;
|
||||
|
||||
if (tabsApi) {
|
||||
// Close each agent-created tab that still exists.
|
||||
const agentCreatedTabIds = Array.from(ctx.agentCreatedTabs);
|
||||
for (const tabId of agentCreatedTabIds) {
|
||||
try {
|
||||
await tabsApi.remove(tabId);
|
||||
ctx.agentCreatedTabs.delete(tabId);
|
||||
} catch (err) {
|
||||
// Tab may already be gone (closed by the user). Non-fatal.
|
||||
console.warn(`[bsk session_stop] failed to close agent tab ${tabId}`, err);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Step 5: decide whether to release (keep) the window or close it.
|
||||
let shouldRelease = false;
|
||||
if (queryApi) {
|
||||
try {
|
||||
const liveWindowTabs = await queryApi.query({ windowId: ctx.agentWindowId });
|
||||
// Only genuine *user* tabs count toward keeping the window open.
|
||||
// An agent tab that failed to close in Step 4 may still be present
|
||||
// here; if we counted it as a reason to release (dropOnly), the
|
||||
// window would be kept and that agent tab would leak (issue #57
|
||||
// regression). Exclude any id still tracked in agentCreatedTabs.
|
||||
const userTabs = liveWindowTabs.filter((t) => {
|
||||
if (t.id === undefined) return false;
|
||||
return !ctx.agentCreatedTabs.has(t.id);
|
||||
});
|
||||
const leakedAgentTabs = liveWindowTabs.filter(
|
||||
(t) => t.id !== undefined && ctx.agentCreatedTabs.has(t.id),
|
||||
);
|
||||
if (leakedAgentTabs.length > 0) {
|
||||
console.warn(
|
||||
`[bsk session_stop] ${leakedAgentTabs.length} agent tab(s) failed to close; forcing window close instead of release`,
|
||||
leakedAgentTabs.map((t) => t.id),
|
||||
);
|
||||
}
|
||||
shouldRelease = userTabs.length > 0;
|
||||
} catch {
|
||||
// Query failed (e.g. window already gone) — conservatively close it.
|
||||
shouldRelease = false;
|
||||
}
|
||||
}
|
||||
|
||||
if (shouldRelease) {
|
||||
// Keep the window + its user tabs; only drop the session binding.
|
||||
await manager.stop(params.session_id, { dropOnly: true });
|
||||
result.window_released = true;
|
||||
} else {
|
||||
// Window is empty (or we couldn't verify state) — close it.
|
||||
await manager.stop(params.session_id);
|
||||
}
|
||||
// Step 4: close the Agent Window and drop the context.
|
||||
await manager.stop(params.session_id);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
@@ -55,10 +55,8 @@ export interface CdpRunner {
|
||||
sendToTarget?<T = unknown>(target: CdpTarget, method: string, params?: object): Promise<T>;
|
||||
detach?(tabId: number): Promise<void>;
|
||||
getFrameGraph?(tabId: number): Promise<CdpFrameGraph>;
|
||||
getAttachmentId?(tabId: number): string | undefined;
|
||||
ensureAttachedToUrl?(tabId: number, expectedUrl: string | undefined): Promise<void>;
|
||||
trackSessionTab?(sessionId: string, tabId: number): void;
|
||||
releaseSessionTab?(sessionId: string, tabId: number): Promise<void>;
|
||||
onEvent?(handler: (source: CdpDebuggee, method: string, params: unknown) => void): {
|
||||
dispose(): void;
|
||||
};
|
||||
|
||||
@@ -10,7 +10,7 @@ import {
|
||||
import type { SessionContext, SessionManager } from "@/session-manager/manager";
|
||||
import type { RpcError } from "@/transport/types";
|
||||
import { rpcError } from "./errors";
|
||||
import { type CdpRunner, isRpcError, lookupSession } from "./shared";
|
||||
import { isRpcError, lookupSession } from "./shared";
|
||||
|
||||
export type TabScope = "user" | "agent" | "all";
|
||||
|
||||
@@ -287,10 +287,6 @@ export interface TabManagementDeps {
|
||||
approveBorrow?: BorrowConfirmationApprover;
|
||||
/** Clears Agent-scoped overlays after a borrowed tab is returned. */
|
||||
agentOverlayReset?: AgentOverlayResetApi;
|
||||
/** Releases this session's CDP claim after a borrowed tab is returned. */
|
||||
cdp?: Pick<CdpRunner, "releaseSessionTab">;
|
||||
/** Runs after tab_return validation, before moving the borrowed tab. */
|
||||
beforeReturn?: (sessionId: string, tabId: number) => Promise<void>;
|
||||
/**
|
||||
* Reports whether `windowId` is any live session's Agent Window.
|
||||
* `tab_return`'s fallback window picker uses this to avoid moving a
|
||||
@@ -373,7 +369,6 @@ function buildCreateProps(
|
||||
* Returns the created tab on success, or an `RpcError` on failure.
|
||||
*/
|
||||
async function createTabAndCleanup(
|
||||
ctx: SessionContext,
|
||||
deps: TabManagementDeps,
|
||||
createProps: chrome.tabs.CreateProperties,
|
||||
): Promise<CreatedChromeTab | RpcError> {
|
||||
@@ -386,28 +381,25 @@ async function createTabAndCleanup(
|
||||
message: err instanceof Error ? err.message : String(err),
|
||||
};
|
||||
}
|
||||
if (typeof tab.id !== "number") {
|
||||
return { code: "protocol_error", message: "chrome.tabs.create returned no tab id" };
|
||||
}
|
||||
// Claim the concrete id returned by Chrome. Ownership never depends on
|
||||
// matching this request to an asynchronous onCreated event.
|
||||
ctx.agentCreatedTabs.add(tab.id);
|
||||
if (aborted(deps.signal, "tab_create")) {
|
||||
try {
|
||||
await getTabsApi(deps).remove(tab.id);
|
||||
ctx.agentCreatedTabs.delete(tab.id);
|
||||
} catch (cleanupErr) {
|
||||
// Keep the claim when cleanup fails so session_stop can retry instead
|
||||
// of releasing an agent-owned tab to the user.
|
||||
return rpcError(
|
||||
"protocol_error",
|
||||
"cleanup_failed",
|
||||
`tab_create aborted but cleanup of tab ${tab.id} failed: ${describeError(cleanupErr)}`,
|
||||
{ resource_type: "tab", resource_id: tab.id },
|
||||
);
|
||||
// We already opened the tab; close it on abort so we don't leak.
|
||||
if (typeof tab.id === "number") {
|
||||
try {
|
||||
await getTabsApi(deps).remove(tab.id);
|
||||
} catch (cleanupErr) {
|
||||
return rpcError(
|
||||
"protocol_error",
|
||||
"cleanup_failed",
|
||||
`tab_create aborted but cleanup of tab ${tab.id} failed: ${describeError(cleanupErr)}`,
|
||||
{ resource_type: "tab", resource_id: tab.id },
|
||||
);
|
||||
}
|
||||
}
|
||||
return { code: "cancelled", message: "tab_create aborted" };
|
||||
}
|
||||
if (typeof tab.id !== "number") {
|
||||
return { code: "protocol_error", message: "chrome.tabs.create returned no tab id" };
|
||||
}
|
||||
return { ...tab, id: tab.id };
|
||||
}
|
||||
|
||||
@@ -434,7 +426,7 @@ export async function handleTabCreate(
|
||||
const paramErr = validateTabCreateParams(params);
|
||||
if (paramErr) return paramErr;
|
||||
|
||||
const tab = await createTabAndCleanup(ctx, deps, buildCreateProps(ctx, params));
|
||||
const tab = await createTabAndCleanup(deps, buildCreateProps(ctx, params));
|
||||
if (isRpcError(tab)) return tab;
|
||||
|
||||
return {
|
||||
@@ -534,9 +526,6 @@ export async function handleTabClose(
|
||||
}
|
||||
try {
|
||||
await getTabsApi(deps).remove(params.tab_id);
|
||||
// Keep the tracking set accurate so session_stop won't try to close a
|
||||
// tab that's already gone (design §3.1).
|
||||
ctx.agentCreatedTabs.delete(params.tab_id);
|
||||
} catch (err) {
|
||||
return {
|
||||
code: "protocol_error",
|
||||
@@ -764,6 +753,13 @@ async function executeBorrowCore(
|
||||
);
|
||||
if (moveErr) return moveErr;
|
||||
|
||||
// Best-effort activation — failure is non-fatal.
|
||||
try {
|
||||
await p.tabsApi.update(p.tabId, { active: true });
|
||||
} catch (err) {
|
||||
console.debug("[bsk tab_borrow] activate after move failed", err);
|
||||
}
|
||||
|
||||
return { originalWindowId, originalIndex };
|
||||
}
|
||||
|
||||
@@ -827,12 +823,6 @@ export async function handleTabBorrow(
|
||||
message: err instanceof Error ? err.message : String(err),
|
||||
};
|
||||
}
|
||||
// Activate after commit so overlay/event observers see the tab as claimed.
|
||||
try {
|
||||
await tabsApi.update(params.tab_id, { active: true });
|
||||
} catch (err) {
|
||||
console.debug("[bsk tab_borrow] activate after move failed", err);
|
||||
}
|
||||
return {
|
||||
tab_id: params.tab_id,
|
||||
original_window_id: coreResult.originalWindowId,
|
||||
@@ -949,22 +939,16 @@ async function cleanupUnusedFallbackWindow(
|
||||
}
|
||||
}
|
||||
|
||||
async function releaseReturnedTabState(
|
||||
function resetAgentOverlaysInReturnedTab(
|
||||
ctx: SessionContext,
|
||||
tabId: number,
|
||||
deps: TabManagementDeps,
|
||||
): Promise<void> {
|
||||
): void {
|
||||
void getAgentOverlayResetApi(deps)
|
||||
.resetAgentOverlays(tabId, ctx.sessionId)
|
||||
.catch((err) => {
|
||||
console.debug("[bsk tab_return] agent overlay reset failed", err);
|
||||
});
|
||||
try {
|
||||
await deps.cdp?.releaseSessionTab?.(ctx.sessionId, tabId);
|
||||
} catch (err) {
|
||||
// A successful move must not be retried because debugger cleanup failed.
|
||||
console.debug("[bsk tab_return] CDP release failed", err);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -1038,7 +1022,7 @@ export async function returnBorrowedTab(
|
||||
});
|
||||
const movedTab = Array.isArray(moved) ? moved[0] : moved;
|
||||
const finalIndex = typeof movedTab?.index === "number" ? movedTab.index : targetIndex;
|
||||
await releaseReturnedTabState(ctx, tabId, deps);
|
||||
resetAgentOverlaysInReturnedTab(ctx, tabId, deps);
|
||||
return {
|
||||
tabId,
|
||||
toWindowId: targetWindowId,
|
||||
@@ -1080,7 +1064,7 @@ export async function returnBorrowedTab(
|
||||
});
|
||||
const movedTab = Array.isArray(moved) ? moved[0] : moved;
|
||||
const finalIndex = typeof movedTab?.index === "number" ? movedTab.index : target.index;
|
||||
await releaseReturnedTabState(ctx, tabId, deps);
|
||||
resetAgentOverlaysInReturnedTab(ctx, tabId, deps);
|
||||
return {
|
||||
tabId,
|
||||
toWindowId: target.windowId,
|
||||
@@ -1126,7 +1110,6 @@ export async function handleTabReturn(
|
||||
message: `tab_return: tab ${params.tab_id} is not borrowed by this session`,
|
||||
};
|
||||
}
|
||||
await deps.beforeReturn?.(ctx.sessionId, params.tab_id);
|
||||
const outcome = await returnBorrowedTab(ctx, params.tab_id, {
|
||||
...deps,
|
||||
isAgentWindowId:
|
||||
|
||||
@@ -1,42 +0,0 @@
|
||||
// Shared time boundary for browser-side file-transfer transactions. Concrete
|
||||
// upload/download mechanisms own their state machines and effect semantics.
|
||||
|
||||
export class BoundedWaitError extends Error {
|
||||
constructor(
|
||||
readonly kind: "timeout" | "aborted",
|
||||
message: string,
|
||||
) {
|
||||
super(message);
|
||||
}
|
||||
}
|
||||
|
||||
export function remainingMs(deadline: number): number {
|
||||
return Math.max(0, deadline - Date.now());
|
||||
}
|
||||
|
||||
export async function waitBounded<T>(
|
||||
promise: Promise<T>,
|
||||
deadline: number,
|
||||
signal: AbortSignal | undefined,
|
||||
timeoutMessage: string,
|
||||
): Promise<T> {
|
||||
const remaining = remainingMs(deadline);
|
||||
if (signal?.aborted) throw new BoundedWaitError("aborted", "file transfer aborted");
|
||||
if (remaining === 0) throw new BoundedWaitError("timeout", timeoutMessage);
|
||||
|
||||
let timer: ReturnType<typeof setTimeout> | undefined;
|
||||
let onAbort: (() => void) | undefined;
|
||||
const boundary = new Promise<never>((_resolve, reject) => {
|
||||
timer = setTimeout(() => reject(new BoundedWaitError("timeout", timeoutMessage)), remaining);
|
||||
if (signal) {
|
||||
onAbort = () => reject(new BoundedWaitError("aborted", "file transfer aborted"));
|
||||
signal.addEventListener("abort", onAbort, { once: true });
|
||||
}
|
||||
});
|
||||
try {
|
||||
return await Promise.race([promise, boundary]);
|
||||
} finally {
|
||||
if (timer) clearTimeout(timer);
|
||||
if (signal && onAbort) signal.removeEventListener("abort", onAbort);
|
||||
}
|
||||
}
|
||||
@@ -2,13 +2,8 @@
|
||||
// target, then delegate the browser protocol transaction to the file-input
|
||||
// transaction module.
|
||||
|
||||
import {
|
||||
type CaptureSuppressSendToTab,
|
||||
withExtensionOverlayHidden,
|
||||
} from "@/lib/capture-suppress-bridge";
|
||||
import type { SessionManager } from "@/session-manager/manager";
|
||||
import type { RpcError, UploadParams, UploadResult } from "@/transport/types";
|
||||
import { uploadThroughFileDrop } from "./file-drop-transaction";
|
||||
import { uploadThroughActivatedFileInput } from "./file-input-transaction";
|
||||
import { clickResolvedTarget, type InteractionDeps, resolveActionTarget } from "./interaction";
|
||||
import {
|
||||
@@ -23,7 +18,6 @@ const DEFAULT_TIMEOUT_MS = 120_000;
|
||||
|
||||
export interface UploadDeps extends InteractionDeps {
|
||||
cdp: CdpRunner;
|
||||
sendToTab?: CaptureSuppressSendToTab;
|
||||
}
|
||||
|
||||
export async function handleUpload(
|
||||
@@ -47,39 +41,19 @@ export async function handleUpload(
|
||||
const address = await resolveActionTarget(deps.cdp, ctx, target, params, "upload");
|
||||
if (isRpcError(address)) return address;
|
||||
const timeoutMs = params.timeout_ms ?? DEFAULT_TIMEOUT_MS;
|
||||
const files = params.files.map((file) => file.staged_path as string);
|
||||
const mode = params.mode ?? "input";
|
||||
let usedRef = address.usedRef;
|
||||
let usedSelector = address.usedSelector;
|
||||
if (mode === "drop") {
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
const transaction = await uploadThroughFileDrop({
|
||||
cdp: deps.cdp,
|
||||
actionTarget: address,
|
||||
files,
|
||||
timeoutMs,
|
||||
signal: deps.signal,
|
||||
withOverlayHidden: (operation) =>
|
||||
withExtensionOverlayHidden(target.tabId, operation, deps.sendToTab),
|
||||
});
|
||||
if (isRpcError(transaction)) return transaction;
|
||||
} else {
|
||||
const transaction = await uploadThroughActivatedFileInput({
|
||||
cdp: deps.cdp,
|
||||
actionTarget: address,
|
||||
files,
|
||||
timeoutMs,
|
||||
signal: deps.signal,
|
||||
trigger: () => clickResolvedTarget(ctx, address, {}, deps),
|
||||
});
|
||||
if (isRpcError(transaction)) return transaction;
|
||||
usedRef = transaction.click.used_ref;
|
||||
usedSelector = transaction.click.used_selector;
|
||||
}
|
||||
const transaction = await uploadThroughActivatedFileInput({
|
||||
cdp: deps.cdp,
|
||||
actionTarget: address,
|
||||
files: params.files.map((file) => file.staged_path as string),
|
||||
timeoutMs,
|
||||
signal: deps.signal,
|
||||
trigger: () => clickResolvedTarget(ctx, address, {}, deps),
|
||||
});
|
||||
if (isRpcError(transaction)) return transaction;
|
||||
return {
|
||||
tab_id: target.tabId,
|
||||
used_ref: usedRef,
|
||||
used_selector: usedSelector,
|
||||
used_ref: transaction.click.used_ref,
|
||||
used_selector: transaction.click.used_selector,
|
||||
file_names: params.files.map((file) => file.name),
|
||||
};
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,7 +1,6 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { OVERLAY_HOST_MARKER_ATTR, OVERLAY_HOST_NAME } from "../../../lib/overlay-bridge";
|
||||
import { collectOverlayExcludedBackendIds, probeHoverSurfaces } from "../capture";
|
||||
import { captureObservationFacts, semanticCapture } from "../capture-coordinator";
|
||||
import { captureViewModel, collectOverlayExcludedBackendIds, probeHoverSurfaces } from "../capture";
|
||||
|
||||
// Minimal but format-accurate captureSnapshot reply: a body with one
|
||||
// fixed full-screen overlay div carrying a password input.
|
||||
@@ -27,7 +26,6 @@ function fakeSnapshotReply() {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
frameId: "root",
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1, 2],
|
||||
nodeType: [1, 1, 1, 1],
|
||||
@@ -75,7 +73,6 @@ function hoverTriggerSnapshotReply() {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
frameId: "root",
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1],
|
||||
nodeType: [1, 1, 1],
|
||||
@@ -120,7 +117,6 @@ function twoHoverTriggerSnapshotReply() {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
frameId: "root",
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1, 1],
|
||||
nodeType: [1, 1, 1, 1],
|
||||
@@ -174,7 +170,6 @@ function nestedHoverTriggerSnapshotReply() {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
frameId: "root",
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1, 2, 3],
|
||||
nodeType: [1, 1, 1, 1, 1],
|
||||
@@ -220,39 +215,33 @@ function makeCdp(snapshot: unknown) {
|
||||
if (method === "Runtime.evaluate") {
|
||||
return {
|
||||
result: {
|
||||
deepSerializedValue: {
|
||||
type: "array",
|
||||
value: [
|
||||
value: {
|
||||
controls: [
|
||||
{
|
||||
type: "array",
|
||||
value: [
|
||||
{ type: "node", value: { backendNodeId: 13 } },
|
||||
{
|
||||
type: "string",
|
||||
value: JSON.stringify({
|
||||
state: "filled",
|
||||
sensitive: true,
|
||||
placeholder: "secret",
|
||||
}),
|
||||
},
|
||||
],
|
||||
state: "filled",
|
||||
sensitive: true,
|
||||
defaultValue: "hunter2",
|
||||
value: "hunter2",
|
||||
placeholder: "secret",
|
||||
},
|
||||
],
|
||||
childFrames: [],
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
if (method === "Runtime.releaseObjectGroup") return {};
|
||||
throw new Error(`unexpected ${method}`);
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
}
|
||||
|
||||
describe("snapshot capture and hover", () => {
|
||||
describe("captureViewModel", () => {
|
||||
it("parses nodes, attrs, rects, paint order and styles", async () => {
|
||||
const { nodes, viewport } = semanticCapture(
|
||||
await captureObservationFacts(makeCdp(fakeSnapshotReply()), 4),
|
||||
).captured;
|
||||
const { nodes, viewport, iframeNodes } = await captureViewModel(
|
||||
makeCdp(fakeSnapshotReply()),
|
||||
4,
|
||||
);
|
||||
expect(iframeNodes.size).toBe(0);
|
||||
expect(viewport).toEqual({ width: 1000, height: 800 });
|
||||
const div = nodes.find((n) => n.backendNodeId === 12);
|
||||
expect(div).toMatchObject({ tag: "div", position: "fixed", parentBackendNodeId: 11 });
|
||||
@@ -264,11 +253,32 @@ describe("snapshot capture and hover", () => {
|
||||
|
||||
it("never hovers the page while capturing", async () => {
|
||||
const cdp = makeCdp(fakeSnapshotReply());
|
||||
semanticCapture(await captureObservationFacts(cdp, 4)).captured;
|
||||
await captureViewModel(cdp, 4);
|
||||
|
||||
expect(cdp.send).not.toHaveBeenCalledWith(4, "Input.dispatchMouseEvent", expect.anything());
|
||||
});
|
||||
|
||||
it("batch-enriches form controls without per-node object resolution", async () => {
|
||||
const cdp = makeCdp(fakeSnapshotReply());
|
||||
const result = await captureViewModel(cdp, 4);
|
||||
const input = result.nodes.find((node) => node.backendNodeId === 13);
|
||||
|
||||
expect(input).toMatchObject({
|
||||
formState: "filled",
|
||||
formPlaceholder: "secret",
|
||||
attrs: { type: "password" },
|
||||
});
|
||||
expect(input?.formValue).toBeUndefined();
|
||||
expect(input?.formDefaultValue).toBeUndefined();
|
||||
expect(cdp.send).toHaveBeenCalledWith(
|
||||
4,
|
||||
"Runtime.evaluate",
|
||||
expect.objectContaining({ returnByValue: true }),
|
||||
);
|
||||
expect(cdp.send).not.toHaveBeenCalledWith(4, "DOM.resolveNode", expect.anything());
|
||||
expect(cdp.send).not.toHaveBeenCalledWith(4, "Runtime.callFunctionOn", expect.anything());
|
||||
});
|
||||
|
||||
it("runs hover surface probes when explicitly enabled", async () => {
|
||||
let hoverStateCalls = 0;
|
||||
const cdp = {
|
||||
@@ -308,7 +318,7 @@ describe("snapshot capture and hover", () => {
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
|
||||
const captured = semanticCapture(await captureObservationFacts(cdp, 4)).captured;
|
||||
const captured = await captureViewModel(cdp, 4);
|
||||
const surfaceProbes = await probeHoverSurfaces(cdp, 4, captured.nodes);
|
||||
|
||||
expect(surfaceProbes).toEqual([
|
||||
@@ -368,7 +378,7 @@ describe("snapshot capture and hover", () => {
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
|
||||
const captured = semanticCapture(await captureObservationFacts(cdp, 4)).captured;
|
||||
const captured = await captureViewModel(cdp, 4);
|
||||
const surfaceProbes = await probeHoverSurfaces(cdp, 4, captured.nodes);
|
||||
|
||||
expect(surfaceProbes).toEqual([
|
||||
@@ -413,7 +423,7 @@ describe("snapshot capture and hover", () => {
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
|
||||
const captured = semanticCapture(await captureObservationFacts(cdp, 4)).captured;
|
||||
const captured = await captureViewModel(cdp, 4);
|
||||
await probeHoverSurfaces(cdp, 4, captured.nodes);
|
||||
|
||||
expect(hoverMoves).toBe(1);
|
||||
@@ -445,7 +455,6 @@ describe("snapshot capture and hover", () => {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
frameId: "root",
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1, 0, 3, 4],
|
||||
nodeName: [i("html"), i("body"), i("div"), i(OVERLAY_HOST_NAME), i("div"), i("button")],
|
||||
@@ -476,7 +485,7 @@ describe("snapshot capture and hover", () => {
|
||||
],
|
||||
};
|
||||
|
||||
const { nodes } = semanticCapture(await captureObservationFacts(makeCdp(snapshot), 4)).captured;
|
||||
const { nodes } = await captureViewModel(makeCdp(snapshot), 4);
|
||||
const ids = nodes.map((n) => n.backendNodeId);
|
||||
// Real page nodes survive.
|
||||
expect(ids).toContain(12);
|
||||
@@ -507,7 +516,6 @@ describe("snapshot capture and hover", () => {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
frameId: "root",
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1, 0, 3, 4],
|
||||
nodeName: [i("html"), i("body"), i("div"), i(OVERLAY_HOST_NAME), i("div"), i("button")],
|
||||
@@ -538,9 +546,7 @@ describe("snapshot capture and hover", () => {
|
||||
],
|
||||
};
|
||||
|
||||
const { excludedBackendNodeIds } = semanticCapture(
|
||||
await captureObservationFacts(makeCdp(snapshot), 4),
|
||||
).captured;
|
||||
const { excludedBackendNodeIds } = await captureViewModel(makeCdp(snapshot), 4);
|
||||
expect(excludedBackendNodeIds).toEqual(new Set([13, 14, 15]));
|
||||
});
|
||||
|
||||
@@ -561,7 +567,6 @@ describe("snapshot capture and hover", () => {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
frameId: "root",
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1],
|
||||
nodeName: [i("html"), i("body"), i(OVERLAY_HOST_NAME)],
|
||||
@@ -584,9 +589,7 @@ describe("snapshot capture and hover", () => {
|
||||
],
|
||||
};
|
||||
|
||||
const { nodes, excludedBackendNodeIds } = semanticCapture(
|
||||
await captureObservationFacts(makeCdp(snapshot), 4),
|
||||
).captured;
|
||||
const { nodes, excludedBackendNodeIds } = await captureViewModel(makeCdp(snapshot), 4);
|
||||
expect(nodes.map((n) => n.backendNodeId)).not.toContain(13);
|
||||
expect(excludedBackendNodeIds).toEqual(new Set([13]));
|
||||
});
|
||||
@@ -609,7 +612,6 @@ describe("snapshot capture and hover", () => {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
frameId: "root",
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1],
|
||||
nodeName: [i("html"), i("body"), i("div")],
|
||||
@@ -632,9 +634,7 @@ describe("snapshot capture and hover", () => {
|
||||
],
|
||||
};
|
||||
|
||||
const { nodes, excludedBackendNodeIds } = semanticCapture(
|
||||
await captureObservationFacts(makeCdp(snapshot), 4),
|
||||
).captured;
|
||||
const { nodes, excludedBackendNodeIds } = await captureViewModel(makeCdp(snapshot), 4);
|
||||
expect(nodes.map((n) => n.backendNodeId)).not.toContain(13);
|
||||
expect(excludedBackendNodeIds).toEqual(new Set([13]));
|
||||
});
|
||||
@@ -657,7 +657,6 @@ describe("snapshot capture and hover", () => {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
frameId: "root",
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1],
|
||||
nodeName: [i("html"), i("body"), i("div")],
|
||||
@@ -680,7 +679,7 @@ describe("snapshot capture and hover", () => {
|
||||
},
|
||||
],
|
||||
};
|
||||
const { nodes } = semanticCapture(await captureObservationFacts(makeCdp(snapshot), 4)).captured;
|
||||
const { nodes } = await captureViewModel(makeCdp(snapshot), 4);
|
||||
expect(nodes.find((n) => n.backendNodeId === 12)?.cursor).toBe("pointer");
|
||||
expect(nodes.find((n) => n.backendNodeId === 11)?.cursor).toBe("auto");
|
||||
});
|
||||
@@ -692,7 +691,6 @@ describe("snapshot capture and hover", () => {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
frameId: "root",
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1, 1, 1],
|
||||
nodeName: [i("html"), i("body"), i("button"), i("button"), i("button")],
|
||||
@@ -719,7 +717,7 @@ describe("snapshot capture and hover", () => {
|
||||
],
|
||||
};
|
||||
|
||||
const { nodes } = semanticCapture(await captureObservationFacts(makeCdp(snapshot), 4)).captured;
|
||||
const { nodes } = await captureViewModel(makeCdp(snapshot), 4);
|
||||
|
||||
expect(nodes.find((node) => node.backendNodeId === 12)?.rendered).toBe(true);
|
||||
expect(nodes.find((node) => node.backendNodeId === 13)?.rendered).toBe(false);
|
||||
@@ -739,20 +737,19 @@ describe("snapshot capture and hover", () => {
|
||||
throw new Error(method);
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
const { nodes } = semanticCapture(await captureObservationFacts(cdp, 4)).captured;
|
||||
const { nodes } = await captureViewModel(cdp, 4);
|
||||
const div = nodes.find((n) => n.backendNodeId === 12);
|
||||
expect(div?.localRect?.y).toBe(-200);
|
||||
expect(div?.rect).toMatchObject({ y: 0, h: 600 });
|
||||
});
|
||||
|
||||
it("keeps CSS snapshot bounds independent of legacy metrics ratio", async () => {
|
||||
it("normalizes device-pixel bounds by devicePixelRatio", async () => {
|
||||
const S = ["html", "body", "div", "position", "fixed", "static", "pointer-events", "auto"];
|
||||
const i = (s: string) => S.indexOf(s);
|
||||
const snapshot = {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
frameId: "root",
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1],
|
||||
nodeName: [i("html"), i("body"), i("div")],
|
||||
@@ -787,19 +784,13 @@ describe("snapshot capture and hover", () => {
|
||||
throw new Error(method);
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
const { nodes } = semanticCapture(await captureObservationFacts(cdp, 4)).captured;
|
||||
const { nodes } = await captureViewModel(cdp, 4);
|
||||
expect(nodes.find((n) => n.backendNodeId === 12)?.rect).toEqual({
|
||||
x: 0,
|
||||
y: 0,
|
||||
w: 1000,
|
||||
h: 800,
|
||||
});
|
||||
expect(nodes.find((n) => n.backendNodeId === 12)?.localRect).toEqual({
|
||||
x: 0,
|
||||
y: 0,
|
||||
w: 2000,
|
||||
h: 1600,
|
||||
});
|
||||
});
|
||||
|
||||
it("extracts textContent from #text child nodes via nodeValue", async () => {
|
||||
@@ -823,7 +814,6 @@ describe("snapshot capture and hover", () => {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
frameId: "root",
|
||||
scrollOffsetX: 0,
|
||||
scrollOffsetY: 0,
|
||||
nodes: {
|
||||
@@ -862,7 +852,7 @@ describe("snapshot capture and hover", () => {
|
||||
throw new Error(method);
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
const { nodes } = semanticCapture(await captureObservationFacts(cdp, 4)).captured;
|
||||
const { nodes } = await captureViewModel(cdp, 4);
|
||||
const btn = nodes.find((n) => n.backendNodeId === 12);
|
||||
expect(btn?.textContent).toBe("登录");
|
||||
const link = nodes.find((n) => n.backendNodeId === 14);
|
||||
@@ -872,7 +862,7 @@ describe("snapshot capture and hover", () => {
|
||||
expect(body?.textContent).toBeUndefined();
|
||||
});
|
||||
|
||||
it("retains explicit iframe document ownership and projected nodes", async () => {
|
||||
it("parses iframe sub-documents and returns iframeNodes keyed by iframe backendNodeId", async () => {
|
||||
const S = [
|
||||
"html",
|
||||
"body",
|
||||
@@ -949,33 +939,36 @@ describe("snapshot capture and hover", () => {
|
||||
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
|
||||
};
|
||||
}
|
||||
if (method === "DOM.getBoxModel")
|
||||
return { model: { content: [100, 300, 900, 300, 900, 700, 100, 700] } };
|
||||
if (method === "DOM.resolveNode") return { object: { objectId: "owner" } };
|
||||
if (method === "Runtime.callFunctionOn")
|
||||
return { result: { value: { width: 800, height: 400 } } };
|
||||
if (method === "Runtime.releaseObject") return {};
|
||||
throw new Error(method);
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
const facts = await captureObservationFacts(cdp, 4);
|
||||
const root = facts.documents.find((doc) => doc.frame.frameId === "main-frame")!;
|
||||
const child = facts.documents.find((doc) => doc.frame.frameId === "child-frame")!;
|
||||
expect(facts.documents).toHaveLength(2);
|
||||
expect(root.domNodes.find((n) => n.backendNodeId === 13)?.tag).toBe("iframe");
|
||||
const subNodes = child.domNodes;
|
||||
const {
|
||||
nodes,
|
||||
iframeNodes,
|
||||
frameNodes,
|
||||
frameOwnerBackendNodeIds,
|
||||
frameParentIds,
|
||||
rootFrameId,
|
||||
} = await captureViewModel(cdp, 4);
|
||||
// Main frame has 4 nodes; iframe is included
|
||||
expect(nodes.find((n) => n.backendNodeId === 13)?.tag).toBe("iframe");
|
||||
// iframeNodes keyed by the <iframe> element's backendNodeId=13
|
||||
expect(iframeNodes.size).toBe(1);
|
||||
expect(iframeNodes.has(13)).toBe(true);
|
||||
const subNodes = iframeNodes.get(13)!;
|
||||
const input = subNodes.find((n) => n.backendNodeId === 101);
|
||||
expect(input?.tag).toBe("input");
|
||||
expect(input?.attrs.type).toBe("text");
|
||||
expect(input?.ownerFrameBackendNodeId).toBe(13);
|
||||
expect(input?.localRect).toEqual({ x: 0, y: 0, w: 200, h: 40 });
|
||||
expect(input?.rect).toEqual({ x: 100, y: 300, w: 200, h: 40 });
|
||||
expect(facts.rootFrameId).toBe("main-frame");
|
||||
expect(child.frame.ownerBackendNodeId).toBe(13);
|
||||
expect(child.frame.parentFrameId).toBe("main-frame");
|
||||
expect(rootFrameId).toBe("main-frame");
|
||||
expect(frameNodes?.get("child-frame")).toBe(subNodes);
|
||||
expect(frameOwnerBackendNodeIds?.get("child-frame")).toBe(13);
|
||||
expect(frameParentIds?.get("child-frame")).toBe("main-frame");
|
||||
});
|
||||
|
||||
it("projects iframe documents independently of owner snapshot geometry", async () => {
|
||||
it("captures iframe documents without requiring owner-frame geometry", async () => {
|
||||
const S = [
|
||||
"html",
|
||||
"body",
|
||||
@@ -993,7 +986,6 @@ describe("snapshot capture and hover", () => {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
frameId: "root",
|
||||
scrollOffsetX: 0,
|
||||
scrollOffsetY: 0,
|
||||
nodes: {
|
||||
@@ -1011,7 +1003,6 @@ describe("snapshot capture and hover", () => {
|
||||
},
|
||||
},
|
||||
{
|
||||
frameId: "child",
|
||||
scrollOffsetX: 0,
|
||||
scrollOffsetY: 0,
|
||||
nodes: {
|
||||
@@ -1038,23 +1029,15 @@ describe("snapshot capture and hover", () => {
|
||||
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
|
||||
};
|
||||
}
|
||||
if (method === "DOM.getBoxModel")
|
||||
return { model: { content: [100, 300, 900, 300, 900, 700, 100, 700] } };
|
||||
if (method === "DOM.resolveNode") return { object: { objectId: "owner" } };
|
||||
if (method === "Runtime.callFunctionOn")
|
||||
return { result: { value: { width: 800, height: 400 } } };
|
||||
if (method === "Runtime.releaseObject") return {};
|
||||
throw new Error(method);
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
|
||||
const { documents } = await captureObservationFacts(cdp, 4);
|
||||
const input = documents
|
||||
.find((doc) => doc.frame.frameId === "child")
|
||||
?.domNodes?.find((n) => n.backendNodeId === 101);
|
||||
const { iframeNodes } = await captureViewModel(cdp, 4);
|
||||
const input = iframeNodes.get(13)?.find((n) => n.backendNodeId === 101);
|
||||
|
||||
expect(input?.localRect).toEqual({ x: 20, y: 20, w: 120, h: 60 });
|
||||
expect(input?.rect).toEqual({ x: 120, y: 320, w: 120, h: 60 });
|
||||
expect(input?.rect).toBeNull();
|
||||
});
|
||||
|
||||
it("recursively normalizes nested iframe sub-documents", async () => {
|
||||
@@ -1075,7 +1058,6 @@ describe("snapshot capture and hover", () => {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
frameId: "root",
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1],
|
||||
nodeName: [i("html"), i("body"), i("iframe")],
|
||||
@@ -1097,7 +1079,6 @@ describe("snapshot capture and hover", () => {
|
||||
},
|
||||
},
|
||||
{
|
||||
frameId: "child",
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1],
|
||||
nodeName: [i("html"), i("body"), i("iframe")],
|
||||
@@ -1119,7 +1100,6 @@ describe("snapshot capture and hover", () => {
|
||||
},
|
||||
},
|
||||
{
|
||||
frameId: "nested",
|
||||
nodes: {
|
||||
parentIndex: [-1, 0],
|
||||
nodeName: [i("body"), i("input")],
|
||||
@@ -1136,52 +1116,21 @@ describe("snapshot capture and hover", () => {
|
||||
],
|
||||
};
|
||||
const cdp = {
|
||||
send: vi.fn(
|
||||
async (
|
||||
_t: number,
|
||||
method: string,
|
||||
params?: { backendNodeId?: number; objectId?: string },
|
||||
) => {
|
||||
if (method === "DOMSnapshot.enable") return {};
|
||||
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return {
|
||||
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
|
||||
};
|
||||
}
|
||||
if (method === "DOM.getBoxModel")
|
||||
return {
|
||||
model: {
|
||||
content:
|
||||
params?.backendNodeId === 13
|
||||
? [10, 20, 310, 20, 310, 220, 10, 220]
|
||||
: [15, 26, 115, 26, 115, 106, 15, 106],
|
||||
},
|
||||
};
|
||||
if (method === "DOM.resolveNode")
|
||||
return { object: { objectId: String(params?.backendNodeId) } };
|
||||
if (method === "Runtime.callFunctionOn")
|
||||
return {
|
||||
result: {
|
||||
value:
|
||||
params?.objectId === "13"
|
||||
? { width: 300, height: 200 }
|
||||
: { width: 100, height: 80 },
|
||||
},
|
||||
};
|
||||
if (method === "Runtime.releaseObject") return {};
|
||||
throw new Error(method);
|
||||
},
|
||||
) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
send: vi.fn(async (_t: number, method: string) => {
|
||||
if (method === "DOMSnapshot.enable") return {};
|
||||
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return {
|
||||
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
|
||||
};
|
||||
}
|
||||
throw new Error(method);
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
|
||||
const { documents } = await captureObservationFacts(cdp, 4);
|
||||
const nestedIframe = documents
|
||||
.find((doc) => doc.frame.frameId === "child")
|
||||
?.domNodes?.find((n) => n.backendNodeId === 23);
|
||||
const input = documents
|
||||
.find((doc) => doc.frame.frameId === "nested")
|
||||
?.domNodes?.find((n) => n.backendNodeId === 31);
|
||||
const { iframeNodes } = await captureViewModel(cdp, 4);
|
||||
const nestedIframe = iframeNodes.get(13)?.find((n) => n.backendNodeId === 23);
|
||||
const input = iframeNodes.get(23)?.find((n) => n.backendNodeId === 31);
|
||||
|
||||
expect(nestedIframe?.localRect).toEqual({ x: 5, y: 6, w: 100, h: 80 });
|
||||
expect(nestedIframe?.rect).toEqual({ x: 15, y: 26, w: 100, h: 80 });
|
||||
@@ -1190,14 +1139,13 @@ describe("snapshot capture and hover", () => {
|
||||
expect(input?.rect).toEqual({ x: 16, y: 28, w: 40, h: 20 });
|
||||
});
|
||||
|
||||
it("subtracts CSS scroll once without dividing snapshot bounds by a metrics ratio", async () => {
|
||||
it("normalizes bounds then subtracts CSS scroll at dpr>1", async () => {
|
||||
const S = ["html", "body", "div", "position", "fixed", "static", "pointer-events", "auto"];
|
||||
const i = (s: string) => S.indexOf(s);
|
||||
const snapshot = {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
frameId: "root",
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1],
|
||||
nodeName: [i("html"), i("body"), i("div")],
|
||||
@@ -1232,8 +1180,8 @@ describe("snapshot capture and hover", () => {
|
||||
throw new Error(method);
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
const { nodes } = semanticCapture(await captureObservationFacts(cdp, 4)).captured;
|
||||
expect(nodes.find((n) => n.backendNodeId === 12)?.rect?.y).toBe(400 - 100);
|
||||
const { nodes } = await captureViewModel(cdp, 4);
|
||||
expect(nodes.find((n) => n.backendNodeId === 12)?.rect?.y).toBe(400 / 2 - 100);
|
||||
});
|
||||
|
||||
it("collectOverlayExcludedBackendIds walks the pierced overlay host subtree", async () => {
|
||||
|
||||
@@ -1,127 +0,0 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { OVERLAY_HOST_MARKER_ATTR } from "@/lib/overlay-bridge";
|
||||
import { createCaptureCheckpoint } from "../capture-abort";
|
||||
import { buildDocumentIndex, type DecodedNode } from "../facts";
|
||||
import { decodeDocument, REQUESTED_STYLES } from "../snapshot";
|
||||
|
||||
function node(id: number, parent: number | null): DecodedNode {
|
||||
return {
|
||||
backendNodeId: id,
|
||||
parentBackendNodeId: parent,
|
||||
tag: "div",
|
||||
attrs: {},
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
};
|
||||
}
|
||||
|
||||
describe("document facts", () => {
|
||||
it.each([
|
||||
1_000, 10_000, 100_000,
|
||||
])("indexes deep and wide inputs of %i without copying ancestors", async (count) => {
|
||||
for (const wide of [false, true]) {
|
||||
let reads = 0;
|
||||
const input = Array.from({ length: count }, (_, i) => {
|
||||
const item = node(i, i ? (wide ? 0 : i - 1) : null);
|
||||
if (i === 0) item.attrs[OVERLAY_HOST_MARKER_ATTR] = "";
|
||||
Object.defineProperty(item, "parentBackendNodeId", {
|
||||
get: () => {
|
||||
reads++;
|
||||
return i ? (wide ? 0 : i - 1) : null;
|
||||
},
|
||||
});
|
||||
return item;
|
||||
}).reverse(); // parent need not precede child
|
||||
const index = await buildDocumentIndex(input);
|
||||
expect(index.nodes.size).toBe(count);
|
||||
expect(index.excludedBackendNodeIds.size).toBe(count);
|
||||
expect(reads).toBeLessThanOrEqual(count * 8);
|
||||
expect(index.nodes.get(count - 1)).toBe(input[0]);
|
||||
}
|
||||
});
|
||||
|
||||
it("handles cycles and orphans and propagates overlay through shadow roots", async () => {
|
||||
const host = { ...node(1, null), attrs: { [OVERLAY_HOST_MARKER_ATTR]: "" } };
|
||||
const input = [
|
||||
node(3, 2),
|
||||
{ ...node(2, 1), tag: "#document-fragment" },
|
||||
host,
|
||||
node(4, 99),
|
||||
node(5, 6),
|
||||
node(6, 5),
|
||||
node(7, 6),
|
||||
];
|
||||
const index = await buildDocumentIndex(input);
|
||||
expect([...index.excludedBackendNodeIds].sort()).toEqual([1, 2, 3]);
|
||||
expect(index.nodes.size).toBe(input.length);
|
||||
});
|
||||
|
||||
it("does not propagate a descendant overlay backwards into a cycle", async () => {
|
||||
const overlay = { ...node(3, 1), attrs: { [OVERLAY_HOST_MARKER_ATTR]: "" } };
|
||||
const index = await buildDocumentIndex([overlay, node(1, 2), node(2, 1)]);
|
||||
expect([...index.excludedBackendNodeIds]).toEqual([3]);
|
||||
});
|
||||
|
||||
it("decodes current observation bounds, styles and attributes without projecting coordinates", async () => {
|
||||
const strings = ["div", "aria-hidden", "true", "inert", "", "visible", "0", "static", "auto"];
|
||||
const styles = REQUESTED_STYLES.map((style) =>
|
||||
style === "visibility" ? 5 : style === "opacity" ? 6 : style === "position" ? 7 : 8,
|
||||
);
|
||||
const { nodes } = await decodeDocument(
|
||||
{
|
||||
nodes: { backendNodeId: [1], parentIndex: [-1], nodeName: [0], attributes: [[1, 2, 3, 4]] },
|
||||
layout: { nodeIndex: [0], bounds: [[10, 20, 120, 40]], styles: [styles] },
|
||||
},
|
||||
strings,
|
||||
);
|
||||
expect(nodes[0].attrs).toEqual({ "aria-hidden": "true", inert: "" });
|
||||
expect(nodes[0].layout).toEqual({
|
||||
boundsSpace: "snapshot-document-css",
|
||||
bounds: [10, 20, 120, 40],
|
||||
styles: {
|
||||
position: "static",
|
||||
"pointer-events": "auto",
|
||||
cursor: "auto",
|
||||
visibility: "visible",
|
||||
opacity: "0",
|
||||
},
|
||||
});
|
||||
expect(nodes[0]).not.toHaveProperty("rect");
|
||||
expect(nodes[0]).not.toHaveProperty("excluded");
|
||||
});
|
||||
|
||||
it("keeps short blocks synchronous and checks cancellation after yielding", async () => {
|
||||
const clock = vi.spyOn(performance, "now").mockReturnValue(0);
|
||||
try {
|
||||
const controller = new AbortController();
|
||||
const checkpoint = createCaptureCheckpoint(controller.signal);
|
||||
for (let i = 0; i < 20; i++) expect(checkpoint()).toBeUndefined();
|
||||
clock.mockReturnValue(9);
|
||||
const pending = checkpoint();
|
||||
expect(pending).toBeInstanceOf(Promise);
|
||||
controller.abort();
|
||||
await expect(pending).rejects.toMatchObject({ name: "AbortError" });
|
||||
} finally {
|
||||
clock.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it("checks cancellation at bounded indexing blocks", async () => {
|
||||
const controller = new AbortController();
|
||||
const input = Array.from({ length: 100_000 }, (_, i) => node(i, i ? i - 1 : null));
|
||||
let reads = 0;
|
||||
for (const item of input)
|
||||
Object.defineProperty(item, "attrs", {
|
||||
get: () => {
|
||||
reads++;
|
||||
if (reads === 100) controller.abort();
|
||||
return {};
|
||||
},
|
||||
});
|
||||
await expect(buildDocumentIndex(input, controller.signal)).rejects.toMatchObject({
|
||||
name: "AbortError",
|
||||
});
|
||||
expect(reads).toBeLessThanOrEqual(356);
|
||||
});
|
||||
});
|
||||
@@ -1,298 +0,0 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { type CdpRunner, cdpRunnerForTarget } from "../../shared";
|
||||
import { enrichFormControlStates } from "../form-capture";
|
||||
import { decodeDocument } from "../snapshot";
|
||||
|
||||
interface Control {
|
||||
id: number;
|
||||
value: string;
|
||||
type?: string;
|
||||
}
|
||||
|
||||
function snapshot(groups: Control[][]) {
|
||||
const strings: string[] = [];
|
||||
const str = (value: string) => {
|
||||
let index = strings.indexOf(value);
|
||||
if (index < 0) index = strings.push(value) - 1;
|
||||
return index;
|
||||
};
|
||||
const documents = groups.map((controls, groupIndex) => ({
|
||||
frameId: `frame-${groupIndex}`,
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, ...controls.map(() => 1)],
|
||||
nodeName: [str("html"), str("body"), ...controls.map(() => str("input"))],
|
||||
backendNodeId: [8000 + groupIndex * 2, 8001 + groupIndex * 2, ...controls.map((c) => c.id)],
|
||||
attributes: [
|
||||
[],
|
||||
[],
|
||||
...controls.map((c) => [
|
||||
str("type"),
|
||||
str(c.type ?? "text"),
|
||||
str("placeholder"),
|
||||
str("snapshot placeholder"),
|
||||
]),
|
||||
],
|
||||
inputValue: {
|
||||
index: controls.map((_, i) => i + 2),
|
||||
value: controls.map((c) => str(c.value)),
|
||||
},
|
||||
contentDocumentIndex: { index: [] as number[], value: [] as number[] },
|
||||
},
|
||||
}));
|
||||
const root = documents[0].nodes;
|
||||
for (let i = 1; i < documents.length; i++) {
|
||||
root.contentDocumentIndex.index.push(root.backendNodeId.length);
|
||||
root.contentDocumentIndex.value.push(i);
|
||||
root.parentIndex.push(1);
|
||||
root.nodeName.push(str("iframe"));
|
||||
root.backendNodeId.push(9000 + i);
|
||||
root.attributes.push([]);
|
||||
}
|
||||
return { strings, documents };
|
||||
}
|
||||
|
||||
function state(value: string) {
|
||||
return {
|
||||
value,
|
||||
defaultValue: "",
|
||||
placeholder: "live placeholder",
|
||||
state: value ? "filled" : "empty",
|
||||
sensitive: false,
|
||||
};
|
||||
}
|
||||
|
||||
function reply(entries: Array<[number, unknown]>) {
|
||||
return {
|
||||
result: {
|
||||
deepSerializedValue: {
|
||||
type: "array",
|
||||
value: entries.map(([backendNodeId, fields]) => ({
|
||||
type: "array",
|
||||
value: [
|
||||
{ type: "node", value: { backendNodeId } },
|
||||
{ type: "string", value: JSON.stringify(fields) },
|
||||
],
|
||||
})),
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function fakeCdp(
|
||||
groups: Control[][],
|
||||
evaluate: (params: Record<string, unknown>) => unknown,
|
||||
) {
|
||||
const snap = snapshot(groups);
|
||||
const send = vi.fn(async (_tabId: number, method: string, params?: object) => {
|
||||
if (method === "Runtime.evaluate") return evaluate(params as Record<string, unknown>);
|
||||
return {};
|
||||
});
|
||||
const nodes = await decodeControls(snap);
|
||||
return { send, cdp: { send: send as CdpRunner["send"] }, nodes };
|
||||
}
|
||||
|
||||
async function decodeControls(snap: ReturnType<typeof snapshot>) {
|
||||
return Promise.all(
|
||||
snap.documents.map(async (doc) =>
|
||||
(await decodeDocument(doc, snap.strings)).nodes.map((node) => ({ ...node, rect: null })),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
describe("form state identity", () => {
|
||||
it("matches reordered controls by identity and preserves controls missing from the batch", async () => {
|
||||
// The snapshot includes a shadow input. The runtime query skips it,
|
||||
// encounters a new input, and sees the remaining inputs in a new order.
|
||||
const { cdp, nodes } = await fakeCdp(
|
||||
[
|
||||
[
|
||||
{ id: 10, value: "" },
|
||||
{ id: 11, value: "old email" },
|
||||
{ id: 12, value: "old city" },
|
||||
],
|
||||
],
|
||||
() =>
|
||||
reply([
|
||||
[99, state("new control")],
|
||||
[12, state("上海")],
|
||||
[11, state("alice@example.com")],
|
||||
]),
|
||||
);
|
||||
await enrichFormControlStates(cdp, 7, nodes);
|
||||
expect(nodes[0].find((n) => n.backendNodeId === 10)).toMatchObject({
|
||||
formValue: "",
|
||||
formState: "empty",
|
||||
formPlaceholder: "snapshot placeholder",
|
||||
});
|
||||
expect(nodes[0].find((n) => n.backendNodeId === 11)).toMatchObject({
|
||||
formValue: "alice@example.com",
|
||||
});
|
||||
expect(nodes[0].find((n) => n.backendNodeId === 12)).toMatchObject({ formValue: "上海" });
|
||||
expect(nodes[0].some((n) => n.backendNodeId === 99)).toBe(false);
|
||||
});
|
||||
|
||||
it("does not shift state between frames when one frame is absent from the runtime batch", async () => {
|
||||
const { cdp, nodes } = await fakeCdp(
|
||||
[
|
||||
[],
|
||||
[{ id: 11, value: "inaccessible frame" }],
|
||||
[{ id: 21, value: "old second" }],
|
||||
[{ id: 31, value: "old third" }],
|
||||
],
|
||||
() =>
|
||||
reply([
|
||||
[31, state("third frame")],
|
||||
[21, state("second frame")],
|
||||
]),
|
||||
);
|
||||
await enrichFormControlStates(cdp, 7, nodes);
|
||||
const controls = nodes.slice(1).flat();
|
||||
expect(controls.find((n) => n.backendNodeId === 11)?.formValue).toBe("inaccessible frame");
|
||||
expect(controls.find((n) => n.backendNodeId === 21)?.formValue).toBe("second frame");
|
||||
expect(controls.find((n) => n.backendNodeId === 31)?.formValue).toBe("third frame");
|
||||
});
|
||||
|
||||
it("keeps identical backend ids in different OOPIF targets separate", async () => {
|
||||
const sendToTarget = vi.fn(async (target, method) => {
|
||||
if (method === "Runtime.evaluate") return reply([[11, state(target.sessionId)]]);
|
||||
return {};
|
||||
});
|
||||
const send = vi.fn(async () => {
|
||||
throw new Error("unexpected top-level target");
|
||||
});
|
||||
const cdp = { send, sendToTarget } as unknown as CdpRunner;
|
||||
for (const sessionId of ["left-frame", "right-frame"]) {
|
||||
const nodes = await decodeControls(snapshot([[{ id: 11, value: "old" }]]));
|
||||
await enrichFormControlStates(cdpRunnerForTarget(cdp, { tabId: 7, sessionId }), 7, nodes);
|
||||
expect(nodes[0].find((n) => n.backendNodeId === 11)?.formValue).toBe(sessionId);
|
||||
expect(sendToTarget).toHaveBeenCalledWith(
|
||||
{ tabId: 7, sessionId },
|
||||
"Runtime.releaseObjectGroup",
|
||||
expect.anything(),
|
||||
);
|
||||
}
|
||||
expect(send).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each([
|
||||
"unsupported",
|
||||
"legacy",
|
||||
"malformed",
|
||||
"invalid-state",
|
||||
])("preserves snapshot data when enrichment is %s", async (mode) => {
|
||||
const { cdp, send, nodes } = await fakeCdp([[{ id: 11, value: "snapshot value" }]], () => {
|
||||
if (mode === "unsupported") throw new Error("serialization unsupported");
|
||||
if (mode === "legacy") return { result: { value: { controls: [state("wrong value")] } } };
|
||||
const result = reply([
|
||||
[11, mode === "invalid-state" ? { value: "wrong value" } : state("wrong value")],
|
||||
]);
|
||||
if (mode === "malformed")
|
||||
result.result.deepSerializedValue.value[0].value[1].value = "not json";
|
||||
return result;
|
||||
});
|
||||
expect(await enrichFormControlStates(cdp, 7, nodes)).toBe(mode !== "unsupported");
|
||||
expect(nodes[0].find((n) => n.backendNodeId === 11)).toMatchObject({
|
||||
formValue: "snapshot value",
|
||||
formPlaceholder: "snapshot placeholder",
|
||||
});
|
||||
expect(send).toHaveBeenCalledWith(7, "Runtime.releaseObjectGroup", expect.anything());
|
||||
});
|
||||
|
||||
it("releases the batch's remote objects when capture is cancelled", async () => {
|
||||
const controller = new AbortController();
|
||||
const { cdp, send, nodes } = await fakeCdp([[{ id: 11, value: "old" }]], () => {
|
||||
controller.abort();
|
||||
return reply([[11, state("new")]]);
|
||||
});
|
||||
await expect(enrichFormControlStates(cdp, 7, nodes, controller.signal)).rejects.toMatchObject({
|
||||
name: "AbortError",
|
||||
});
|
||||
expect(send).toHaveBeenCalledWith(7, "Runtime.releaseObjectGroup", expect.anything());
|
||||
});
|
||||
|
||||
it("does not report success when cancelled during object cleanup", async () => {
|
||||
const controller = new AbortController();
|
||||
const nodes = await decodeControls(snapshot([[{ id: 11, value: "old" }]]));
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async (_tabId, method) => {
|
||||
if (method === "Runtime.evaluate") return reply([[11, state("new")]]);
|
||||
if (method === "Runtime.releaseObjectGroup") controller.abort();
|
||||
return {};
|
||||
}) as CdpRunner["send"],
|
||||
};
|
||||
await expect(enrichFormControlStates(cdp, 7, nodes, controller.signal)).rejects.toMatchObject({
|
||||
name: "AbortError",
|
||||
});
|
||||
expect(cdp.send).toHaveBeenCalledWith(7, "Runtime.releaseObjectGroup", expect.anything());
|
||||
});
|
||||
|
||||
it("skips entries without node identity while still applying valid entries", async () => {
|
||||
const { cdp, nodes } = await fakeCdp(
|
||||
[
|
||||
[
|
||||
{ id: 11, value: "snapshot value" },
|
||||
{ id: 12, value: "old value" },
|
||||
],
|
||||
],
|
||||
() => {
|
||||
const result = reply([
|
||||
[11, state("wrong value")],
|
||||
[12, state("live value")],
|
||||
]);
|
||||
const element = result.result.deepSerializedValue.value[0].value[0];
|
||||
delete (element.value as { backendNodeId?: number }).backendNodeId;
|
||||
return result;
|
||||
},
|
||||
);
|
||||
await enrichFormControlStates(cdp, 7, nodes);
|
||||
expect(nodes[0].find((n) => n.backendNodeId === 11)?.formValue).toBe("snapshot value");
|
||||
expect(nodes[0].find((n) => n.backendNodeId === 12)?.formValue).toBe("live value");
|
||||
});
|
||||
|
||||
it.each([
|
||||
"password",
|
||||
"text",
|
||||
])("redacts a sensitive %s control after identity matching", async (type) => {
|
||||
const { cdp, nodes } = await fakeCdp([[{ id: 11, value: "secret", type }]], () =>
|
||||
reply([[11, { ...state("secret"), sensitive: true }]]),
|
||||
);
|
||||
await enrichFormControlStates(cdp, 7, nodes);
|
||||
const control = nodes[0].find((n) => n.backendNodeId === 11);
|
||||
expect(control?.formState).toBe("filled");
|
||||
expect(control?.formValue).toBeUndefined();
|
||||
expect(control?.formDefaultValue).toBeUndefined();
|
||||
expect(control?.attrs.value).toBeUndefined();
|
||||
});
|
||||
|
||||
it("keeps one bounded batch without per-control CDP queries", async () => {
|
||||
const controls = Array.from({ length: 251 }, (_, i) => ({ id: i + 1, value: "old" }));
|
||||
const elements = controls.map((c) => ({
|
||||
backendNodeId: c.id,
|
||||
tagName: "INPUT",
|
||||
type: "text",
|
||||
value: `live ${c.id}`,
|
||||
defaultValue: "",
|
||||
placeholder: "",
|
||||
}));
|
||||
const { cdp, send, nodes } = await fakeCdp([controls], (params) => {
|
||||
const doc = {
|
||||
querySelectorAll: (selector: string) =>
|
||||
selector === "input,textarea,select" ? elements : [],
|
||||
};
|
||||
const entries = new Function("document", `return ${params.expression}`)(doc) as Array<
|
||||
[{ backendNodeId: number }, string]
|
||||
>;
|
||||
expect(entries).toHaveLength(250);
|
||||
return reply(entries.map(([element, json]) => [element.backendNodeId, JSON.parse(json)]));
|
||||
});
|
||||
await enrichFormControlStates(cdp, 7, nodes);
|
||||
expect(nodes[0].find((n) => n.backendNodeId === 250)?.formValue).toBe("live 250");
|
||||
expect(nodes[0].find((n) => n.backendNodeId === 251)?.formValue).toBe("old");
|
||||
expect(send.mock.calls.filter(([, method]) => method === "Runtime.evaluate")).toHaveLength(1);
|
||||
expect(
|
||||
send.mock.calls.filter(([, method]) => method === "Runtime.releaseObjectGroup"),
|
||||
).toHaveLength(1);
|
||||
expect(send).not.toHaveBeenCalledWith(7, "DOM.resolveNode", expect.anything());
|
||||
expect(send).not.toHaveBeenCalledWith(7, "Runtime.callFunctionOn", expect.anything());
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,198 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import type { CdpRunner } from "../../shared";
|
||||
import type { CapturedNode, CapturedViewModel } from "../capture";
|
||||
import { captureFrameData } from "../frame-capture";
|
||||
|
||||
function ownerNode(backendNodeId: number, x: number): CapturedNode {
|
||||
return {
|
||||
backendNodeId,
|
||||
parentBackendNodeId: 1,
|
||||
frameId: "main",
|
||||
ownerFrameBackendNodeId: null,
|
||||
tag: "iframe",
|
||||
attrs: {},
|
||||
rect: { x, y: 100, w: 300, h: 200 },
|
||||
localRect: { x, y: 100, w: 300, h: 200 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
};
|
||||
}
|
||||
|
||||
function childSnapshot(frameId: string, backendNodeId: number) {
|
||||
const strings = [frameId, "body", "button", "static", "auto", "pointer"];
|
||||
return {
|
||||
strings,
|
||||
documents: [
|
||||
{
|
||||
frameId,
|
||||
nodes: {
|
||||
parentIndex: [-1, 0],
|
||||
nodeName: [1, 2],
|
||||
backendNodeId: [backendNodeId - 1, backendNodeId],
|
||||
attributes: [[], []],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [0, 1],
|
||||
styles: [
|
||||
[3, 4, 4],
|
||||
[3, 4, 5],
|
||||
],
|
||||
bounds: [
|
||||
[0, 0, 300, 200],
|
||||
[10, 20, 100, 40],
|
||||
],
|
||||
paintOrders: [0, 1],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
describe("captureFrameData", () => {
|
||||
it("captures and positions multiple OOPIF documents missing from the root snapshot", async () => {
|
||||
const leftOwner = ownerNode(10, 50);
|
||||
const rightOwner = ownerNode(20, 500);
|
||||
const captured: CapturedViewModel = {
|
||||
nodes: [leftOwner, rightOwner],
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
frameNodes: new Map([["main", [leftOwner, rightOwner]]]),
|
||||
frameOwnerBackendNodeIds: new Map(),
|
||||
rootFrameId: "main",
|
||||
excludedBackendNodeIds: new Set(),
|
||||
};
|
||||
const sendToTarget = vi.fn(async (target, method) => {
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 300, clientHeight: 200, pageX: 0, pageY: 0 } };
|
||||
}
|
||||
if (method === "DOMSnapshot.enable" || method === "Accessibility.enable") return {};
|
||||
if (method === "DOMSnapshot.captureSnapshot") {
|
||||
return target.sessionId === "left-session"
|
||||
? childSnapshot("left", 101)
|
||||
: childSnapshot("right", 201);
|
||||
}
|
||||
if (method === "Accessibility.getFullAXTree") return { nodes: [] };
|
||||
throw new Error(`unexpected ${method}`);
|
||||
});
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async (_tabId, method, params) => {
|
||||
if (method === "Accessibility.enable") return {};
|
||||
if (method === "Accessibility.getFullAXTree") return { nodes: [] };
|
||||
if (method === "DOM.getBoxModel") {
|
||||
const backendNodeId = (params as { backendNodeId?: number })?.backendNodeId;
|
||||
const x = backendNodeId === 10 ? 50 : 500;
|
||||
return { model: { content: [x, 100, x + 300, 100, x + 300, 300, x, 300] } };
|
||||
}
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } };
|
||||
}
|
||||
throw new Error(`unexpected root ${method}`);
|
||||
}) as CdpRunner["send"],
|
||||
sendToTarget: sendToTarget as unknown as NonNullable<CdpRunner["sendToTarget"]>,
|
||||
getFrameGraph: vi.fn(async () => ({
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "left",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 10,
|
||||
target: { tabId: 4, sessionId: "left-session" },
|
||||
},
|
||||
{
|
||||
frameId: "right",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 20,
|
||||
target: { tabId: 4, sessionId: "right-session" },
|
||||
},
|
||||
],
|
||||
})),
|
||||
};
|
||||
|
||||
const trees = await captureFrameData(cdp, 4, captured);
|
||||
|
||||
expect(trees.map((tree) => tree.frameId)).toEqual(["main", "left", "right"]);
|
||||
expect(
|
||||
captured.frameNodes?.get("left")?.find((node) => node.backendNodeId === 101)?.rect,
|
||||
).toEqual({ x: 60, y: 120, w: 100, h: 40 });
|
||||
expect(
|
||||
captured.frameNodes?.get("right")?.find((node) => node.backendNodeId === 201)?.rect,
|
||||
).toEqual({ x: 510, y: 120, w: 100, h: 40 });
|
||||
expect(captured.frameOwnerBackendNodeIds).toEqual(
|
||||
new Map([
|
||||
["left", 10],
|
||||
["right", 20],
|
||||
]),
|
||||
);
|
||||
expect(captured.frameParentIds).toEqual(
|
||||
new Map([
|
||||
["left", "main"],
|
||||
["right", "main"],
|
||||
]),
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps OOPIF semantics when viewport projection is unavailable", async () => {
|
||||
const owner = { ...ownerNode(10, 50), rect: null, localRect: null };
|
||||
const captured: CapturedViewModel = {
|
||||
nodes: [owner],
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
frameNodes: new Map([["main", [owner]]]),
|
||||
rootFrameId: "main",
|
||||
excludedBackendNodeIds: new Set(),
|
||||
};
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async (_tabId, method) => {
|
||||
if (method === "Accessibility.enable") return {};
|
||||
if (method === "Accessibility.getFullAXTree") return { nodes: [] };
|
||||
if (method === "DOM.getBoxModel") throw new Error("owner geometry unavailable");
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } };
|
||||
}
|
||||
throw new Error(`unexpected root ${method}`);
|
||||
}) as CdpRunner["send"],
|
||||
sendToTarget: vi.fn(async (_target, method) => {
|
||||
if (method === "Page.getLayoutMetrics") throw new Error("viewport unavailable");
|
||||
if (method === "DOMSnapshot.enable" || method === "Accessibility.enable") return {};
|
||||
if (method === "DOMSnapshot.captureSnapshot") return childSnapshot("child", 101);
|
||||
if (method === "Accessibility.getFullAXTree") {
|
||||
return {
|
||||
nodes: [
|
||||
{
|
||||
nodeId: "button",
|
||||
backendDOMNodeId: 101,
|
||||
role: { type: "role", value: "button" },
|
||||
name: { type: "computedString", value: "Continue" },
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
throw new Error(`unexpected ${method}`);
|
||||
}) as unknown as NonNullable<CdpRunner["sendToTarget"]>,
|
||||
getFrameGraph: vi.fn(async () => ({
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "child",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 10,
|
||||
target: { tabId: 4, sessionId: "child-session" },
|
||||
},
|
||||
],
|
||||
})),
|
||||
};
|
||||
|
||||
const documents = await captureFrameData(cdp, 4, captured);
|
||||
const childDocument = documents.find((document) => document.frameId === "child");
|
||||
|
||||
expect(childDocument?.axNodes).toEqual([
|
||||
expect.objectContaining({ backendDOMNodeId: 101, frameId: "child" }),
|
||||
]);
|
||||
expect(childDocument?.domNodes.find((node) => node.backendNodeId === 101)).toEqual(
|
||||
expect.objectContaining({ rect: null, localRect: { x: 10, y: 20, w: 100, h: 40 } }),
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -1,12 +1,7 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { CdpFrameGraph } from "@/browser-driver/frame-graph";
|
||||
import type { CapturedNode } from "../capture";
|
||||
import {
|
||||
buildFrameDocuments,
|
||||
type FrameAxBatch,
|
||||
type FrameDomInput,
|
||||
type FrameOwnedAxNode,
|
||||
} from "../frame-document";
|
||||
import type { CapturedNode, CapturedViewModel } from "../capture";
|
||||
import { buildFrameDocuments, type FrameAxBatch, type FrameOwnedAxNode } from "../frame-document";
|
||||
|
||||
function domNode(frameId: string, backendNodeId: number): CapturedNode {
|
||||
return {
|
||||
@@ -22,16 +17,19 @@ function domNode(frameId: string, backendNodeId: number): CapturedNode {
|
||||
};
|
||||
}
|
||||
|
||||
function captured(frameNodes: Map<string, CapturedNode[]>): FrameDomInput {
|
||||
function captured(frameNodes: Map<string, CapturedNode[]>): CapturedViewModel {
|
||||
return {
|
||||
nodes: frameNodes.get("main") ?? [],
|
||||
viewport: { width: 800, height: 600 },
|
||||
iframeNodes: new Map(),
|
||||
frameNodes,
|
||||
rootFrameId: "main",
|
||||
excludedBackendNodeIds: new Set(),
|
||||
};
|
||||
}
|
||||
|
||||
describe("buildFrameDocuments", () => {
|
||||
it("partitions mixed AX results by node ownership and cuts cross-frame parent edges", async () => {
|
||||
it("partitions mixed AX results by node ownership and cuts cross-frame parent edges", () => {
|
||||
const graph: CdpFrameGraph = {
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
@@ -56,7 +54,7 @@ describe("buildFrameDocuments", () => {
|
||||
},
|
||||
];
|
||||
|
||||
const documents = await buildFrameDocuments(
|
||||
const documents = buildFrameDocuments(
|
||||
graph,
|
||||
batches,
|
||||
captured(
|
||||
@@ -74,7 +72,7 @@ describe("buildFrameDocuments", () => {
|
||||
expect(childNodes.find((node) => node.nodeId === "virtual")?.frameId).toBe("child");
|
||||
});
|
||||
|
||||
it("deduplicates overlapping same-target AX results using the strongest ownership", async () => {
|
||||
it("deduplicates overlapping same-target AX results using the strongest ownership", () => {
|
||||
const graph: CdpFrameGraph = {
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
@@ -94,7 +92,7 @@ describe("buildFrameDocuments", () => {
|
||||
},
|
||||
];
|
||||
|
||||
const documents = await buildFrameDocuments(
|
||||
const documents = buildFrameDocuments(
|
||||
graph,
|
||||
batches,
|
||||
captured(
|
||||
@@ -109,7 +107,7 @@ describe("buildFrameDocuments", () => {
|
||||
expect(documents.find((document) => document.frameId === "child")?.axNodes).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("uses target-scoped backend ownership when OOPIF backend ids collide", async () => {
|
||||
it("uses target-scoped backend ownership when OOPIF backend ids collide", () => {
|
||||
const graph: CdpFrameGraph = {
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
@@ -131,7 +129,7 @@ describe("buildFrameDocuments", () => {
|
||||
nodes: [{ nodeId: `${frame.frameId}-node`, backendDOMNodeId: 7 }],
|
||||
}));
|
||||
|
||||
const documents = await buildFrameDocuments(
|
||||
const documents = buildFrameDocuments(
|
||||
graph,
|
||||
batches,
|
||||
captured(
|
||||
@@ -151,7 +149,7 @@ describe("buildFrameDocuments", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("uses captured frame parent identity instead of guessing from backend ids", async () => {
|
||||
it("uses captured frame parent identity instead of guessing from backend ids", () => {
|
||||
const frameNodes = new Map([
|
||||
["main", [domNode("main", 1)]],
|
||||
["left", [domNode("left", 7)]],
|
||||
@@ -159,51 +157,18 @@ describe("buildFrameDocuments", () => {
|
||||
["child", [domNode("child", 9)]],
|
||||
]);
|
||||
const capture = captured(frameNodes);
|
||||
capture.frameOwnerBackendNodeIds = new Map([["child", 7]]);
|
||||
capture.frameParentIds = new Map([["child", "left"]]);
|
||||
const frames = ["main", "left", "right", "child"].map((frameId) => ({
|
||||
frame: {
|
||||
frameId,
|
||||
target: { tabId: 4 },
|
||||
...(frameId === "child" ? { parentFrameId: "left", ownerBackendNodeId: 7 } : {}),
|
||||
},
|
||||
frame: { frameId, target: { tabId: 4 } },
|
||||
nodes: [],
|
||||
}));
|
||||
|
||||
const documents = await buildFrameDocuments(null, frames, capture);
|
||||
const documents = buildFrameDocuments(null, frames, capture);
|
||||
|
||||
expect(documents.find((document) => document.frameId === "child")).toMatchObject({
|
||||
parentFrameId: "left",
|
||||
ownerBackendNodeId: 7,
|
||||
});
|
||||
});
|
||||
it("rejects explicit AX frame ownership in a different target", async () => {
|
||||
const frames = [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{ frameId: "remote", target: { tabId: 4, sessionId: "remote" } },
|
||||
];
|
||||
const unresolved = vi.fn();
|
||||
const documents = await buildFrameDocuments(
|
||||
{ rootFrameId: "main", frames },
|
||||
[{ frame: frames[0], nodes: [{ nodeId: "wrong", frameId: "remote" }] }],
|
||||
captured(new Map()),
|
||||
undefined,
|
||||
unresolved,
|
||||
);
|
||||
expect(documents.every((doc) => doc.axNodes.length === 0)).toBe(true);
|
||||
expect(unresolved).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("resolves a reverse-ordered deep AX parent chain without recursion", async () => {
|
||||
const frame = { frameId: "main", target: { tabId: 4 } };
|
||||
const nodes: FrameOwnedAxNode[] = Array.from({ length: 10000 }, (_, i) => ({
|
||||
nodeId: String(i),
|
||||
...(i ? { parentId: String(i - 1) } : { frameId: "main" }),
|
||||
})).reverse();
|
||||
const documents = await buildFrameDocuments(
|
||||
{ rootFrameId: "main", frames: [frame] },
|
||||
[{ frame, nodes }],
|
||||
captured(new Map()),
|
||||
);
|
||||
expect(documents[0].axNodes).toHaveLength(10000);
|
||||
expect(documents[0].axNodes[0].frameId).toBe("main");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,40 +0,0 @@
|
||||
function captureAbortError(): Error {
|
||||
const error = new Error("observation aborted");
|
||||
error.name = "AbortError";
|
||||
return error;
|
||||
}
|
||||
|
||||
export function isAbortError(error: unknown): boolean {
|
||||
return (
|
||||
typeof error === "object" &&
|
||||
error !== null &&
|
||||
(error as { name?: string }).name === "AbortError"
|
||||
);
|
||||
}
|
||||
|
||||
export function throwIfAborted(signal: AbortSignal | undefined): void {
|
||||
if (signal?.aborted) throw captureAbortError();
|
||||
}
|
||||
|
||||
/** Check at bounded work blocks; yield only after a processing slice is spent.
|
||||
* Each caller owns its deadline, so unrelated captures do not share state. */
|
||||
export function createCaptureCheckpoint(signal?: AbortSignal): () => Promise<void> | undefined {
|
||||
throwIfAborted(signal);
|
||||
let deadline = performance.now() + 8;
|
||||
return () => {
|
||||
throwIfAborted(signal);
|
||||
if (performance.now() < deadline) return;
|
||||
return new Promise<void>((resolve) => {
|
||||
const channel = new MessageChannel();
|
||||
channel.port1.onmessage = () => {
|
||||
channel.port1.close();
|
||||
channel.port2.close();
|
||||
resolve();
|
||||
};
|
||||
channel.port2.postMessage(null);
|
||||
}).then(() => {
|
||||
throwIfAborted(signal);
|
||||
deadline = performance.now() + 8;
|
||||
});
|
||||
};
|
||||
}
|
||||
@@ -1,413 +0,0 @@
|
||||
import {
|
||||
type CdpFrame,
|
||||
type CdpFrameGraph,
|
||||
type CdpTarget,
|
||||
cdpTargetKey,
|
||||
} from "@/browser-driver/frame-graph";
|
||||
import { cssViewport, GeometryContext } from "../geometry/frame-context";
|
||||
import { type CdpRunner, cdpRunnerForTarget } from "../shared";
|
||||
import { collectOverlayExcludedBackendIds } from "./capture";
|
||||
import {
|
||||
isAbortError as isCaptureAbort,
|
||||
throwIfAborted as throwCaptureAborted,
|
||||
} from "./capture-abort";
|
||||
import { verifyDocumentIdentity } from "./document-identity";
|
||||
import type { CapturedSceneInput, DocumentIdentity } from "./facts";
|
||||
import { buildDocumentIndex, type CaptureIssue, type ObservationFacts } from "./facts";
|
||||
import { enrichFormControlStates } from "./form-capture";
|
||||
import {
|
||||
buildFrameDocuments,
|
||||
type FrameAxBatch,
|
||||
type FrameDocument,
|
||||
type FrameOwnedAxNode,
|
||||
} from "./frame-document";
|
||||
import { type NormalizedFrameDocument, normalizeSnapshot } from "./normalize";
|
||||
import { describeSnapshotFrames, REQUESTED_STYLES, type SnapshotReply } from "./snapshot";
|
||||
|
||||
interface TargetBatch<T extends FrameOwnedAxNode> {
|
||||
target: CdpTarget;
|
||||
frames: CdpFrame[];
|
||||
snapshotAttachmentId?: string;
|
||||
documents: NormalizedFrameDocument[];
|
||||
ax: FrameAxBatch<T>[];
|
||||
fallbackExcluded: Set<number>;
|
||||
snapshotFrameIds?: Set<string>;
|
||||
rootFrameId?: string;
|
||||
}
|
||||
|
||||
/** A fixed worker pool scoped to one capture; each worker issues at most one
|
||||
* collection request at a time. Geometry has its own measurement-phase bound. */
|
||||
async function collectTasks<T>(
|
||||
items: readonly T[],
|
||||
collect: (item: T) => Promise<void>,
|
||||
signal?: AbortSignal,
|
||||
): Promise<void> {
|
||||
let cursor = 0;
|
||||
let failed = false;
|
||||
let failure: unknown;
|
||||
await Promise.all(
|
||||
Array.from({ length: Math.min(4, items.length) }, async () => {
|
||||
try {
|
||||
while (cursor < items.length && !failed && !signal?.aborted) {
|
||||
const item = items[cursor++];
|
||||
await collect(item);
|
||||
}
|
||||
} catch (error) {
|
||||
if (!failed) failure = error;
|
||||
failed = true;
|
||||
}
|
||||
}),
|
||||
);
|
||||
// A worker's rejection must not let the capture outlive its own task scope.
|
||||
// In-flight reads and their finally blocks settle before cancellation/failure.
|
||||
throwCaptureAborted(signal);
|
||||
if (failed) throw failure;
|
||||
}
|
||||
|
||||
export async function captureObservationFacts<T extends FrameOwnedAxNode>(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
signal?: AbortSignal,
|
||||
pageUrl?: string,
|
||||
): Promise<ObservationFacts<T>> {
|
||||
const startedAt = Date.now();
|
||||
throwCaptureAborted(signal);
|
||||
let graph: CdpFrameGraph | undefined;
|
||||
try {
|
||||
graph = await cdp.getFrameGraph?.(tabId);
|
||||
} catch (error) {
|
||||
if (isCaptureAbort(error)) throw error;
|
||||
}
|
||||
throwCaptureAborted(signal);
|
||||
const geometry = new GeometryContext(cdp, tabId, graph, signal);
|
||||
const issues: CaptureIssue[] = [];
|
||||
const graphFrames = new Map(graph?.frames.map((frame) => [frame.frameId, frame]) ?? []);
|
||||
const groups = new Map<string, TargetBatch<T>>();
|
||||
for (const frame of graph?.frames ?? []) {
|
||||
const key = cdpTargetKey(frame.target);
|
||||
let group = groups.get(key);
|
||||
if (!group) {
|
||||
group = {
|
||||
target: frame.target,
|
||||
frames: [],
|
||||
documents: [],
|
||||
ax: [],
|
||||
fallbackExcluded: new Set(),
|
||||
};
|
||||
groups.set(key, group);
|
||||
}
|
||||
group.frames.push(
|
||||
!frame.target.sessionId && frame.frameId === graph?.rootFrameId && !frame.url && pageUrl
|
||||
? { ...frame, url: pageUrl }
|
||||
: frame,
|
||||
);
|
||||
}
|
||||
if (!groups.has(cdpTargetKey({ tabId })))
|
||||
groups.set(cdpTargetKey({ tabId }), {
|
||||
target: { tabId },
|
||||
frames: [],
|
||||
documents: [],
|
||||
ax: [],
|
||||
fallbackExcluded: new Set(),
|
||||
});
|
||||
const batches = [...groups.values()];
|
||||
const unavailable = (batch: TargetBatch<T>, stage: CaptureIssue["stage"], frameId?: string) =>
|
||||
issues.push({ target: batch.target, frameId, stage, reason: "capture-unavailable" });
|
||||
let firstFailure: unknown;
|
||||
const axReady = new Set<TargetBatch<T>>();
|
||||
await collectTasks(
|
||||
batches,
|
||||
async (batch) => {
|
||||
const scoped = cdpRunnerForTarget(cdp, batch.target);
|
||||
try {
|
||||
throwCaptureAborted(signal);
|
||||
await scoped.send(tabId, "DOMSnapshot.enable", {});
|
||||
throwCaptureAborted(signal);
|
||||
batch.snapshotAttachmentId = cdp.getAttachmentId?.(tabId);
|
||||
const snapshot = await scoped.send<SnapshotReply>(tabId, "DOMSnapshot.captureSnapshot", {
|
||||
computedStyles: REQUESTED_STYLES,
|
||||
includePaintOrder: true,
|
||||
includeDOMRects: true,
|
||||
});
|
||||
throwCaptureAborted(signal);
|
||||
const observed = describeSnapshotFrames(snapshot, batch.target, graphFrames, pageUrl);
|
||||
batch.snapshotFrameIds = observed.ids;
|
||||
batch.rootFrameId = observed.rootFrameId;
|
||||
// Keep graph-only frames for AX fallback, never in preference to observed membership.
|
||||
batch.frames = [
|
||||
...observed.frames,
|
||||
...batch.frames.filter((frame) => !observed.ids.has(frame.frameId)),
|
||||
];
|
||||
batch.documents = await normalizeSnapshot(
|
||||
snapshot,
|
||||
batch.target,
|
||||
batch.frames.filter((frame) => observed.ids.has(frame.frameId)),
|
||||
geometry,
|
||||
issues,
|
||||
signal,
|
||||
observed.rootFrameId,
|
||||
);
|
||||
const formsAvailable = await enrichFormControlStates(
|
||||
scoped,
|
||||
tabId,
|
||||
batch.documents.map((doc) => doc.nodes),
|
||||
signal,
|
||||
);
|
||||
throwCaptureAborted(signal);
|
||||
if (!formsAvailable) unavailable(batch, "forms");
|
||||
const present = new Set(batch.documents.map((doc) => doc.frame.frameId));
|
||||
for (const frame of batch.frames)
|
||||
if (!present.has(frame.frameId)) unavailable(batch, "dom", frame.frameId);
|
||||
} catch (error) {
|
||||
throwCaptureAborted(signal);
|
||||
if (isCaptureAbort(error)) throw error;
|
||||
firstFailure ??= error;
|
||||
unavailable(batch, "dom");
|
||||
batch.fallbackExcluded = await collectOverlayExcludedBackendIds(scoped, tabId, signal);
|
||||
}
|
||||
if (!batch.frames.length && !graph)
|
||||
batch.frames = [
|
||||
{
|
||||
frameId: "root",
|
||||
target: batch.target,
|
||||
...(!batch.target.sessionId && pageUrl ? { url: pageUrl } : {}),
|
||||
},
|
||||
];
|
||||
try {
|
||||
throwCaptureAborted(signal);
|
||||
await scoped.send(tabId, "Accessibility.enable", {});
|
||||
} catch (error) {
|
||||
throwCaptureAborted(signal);
|
||||
if (isCaptureAbort(error)) throw error;
|
||||
firstFailure ??= error;
|
||||
unavailable(batch, "ax");
|
||||
return;
|
||||
}
|
||||
axReady.add(batch);
|
||||
},
|
||||
signal,
|
||||
);
|
||||
|
||||
// Schedule AX by frame across all ready targets, rather than nesting pools.
|
||||
// Keep result slots in source order even when CDP replies arrive out of order.
|
||||
const axTasks: { batch: TargetBatch<T>; frame: CdpFrame; result?: FrameAxBatch<T> }[] =
|
||||
batches.flatMap((batch) =>
|
||||
axReady.has(batch) ? batch.frames.map((frame) => ({ batch, frame })) : [],
|
||||
);
|
||||
await collectTasks(
|
||||
axTasks,
|
||||
async (task) => {
|
||||
const { batch, frame } = task;
|
||||
try {
|
||||
throwCaptureAborted(signal);
|
||||
const result = await cdpRunnerForTarget(cdp, batch.target).send<{ nodes?: T[] }>(
|
||||
tabId,
|
||||
"Accessibility.getFullAXTree",
|
||||
frame.frameId === "root" ? {} : { frameId: frame.frameId },
|
||||
);
|
||||
throwCaptureAborted(signal);
|
||||
task.result = { frame, nodes: result.nodes ?? [] };
|
||||
} catch (error) {
|
||||
throwCaptureAborted(signal);
|
||||
if (isCaptureAbort(error)) throw error;
|
||||
firstFailure ??= error;
|
||||
unavailable(batch, "ax", frame.frameId);
|
||||
}
|
||||
},
|
||||
signal,
|
||||
);
|
||||
for (const task of axTasks) if (task.result) task.batch.ax.push(task.result);
|
||||
|
||||
let viewport = { width: 0, height: 0 };
|
||||
try {
|
||||
const measured = cssViewport(await geometry.layoutMetrics({ tabId }));
|
||||
viewport = { width: measured.width, height: measured.height };
|
||||
} catch (error) {
|
||||
if (isCaptureAbort(error)) throw error;
|
||||
}
|
||||
|
||||
// A snapshot claim outranks an old graph hint. Conflicting actual claims
|
||||
// cannot be resolved by completion order or by overwriting a frameId map.
|
||||
const claims = new Map<string, string>();
|
||||
const invalid = new Set<string>();
|
||||
for (const batch of batches) {
|
||||
const targetKey = cdpTargetKey(batch.target);
|
||||
for (const id of batch.snapshotFrameIds ?? []) {
|
||||
const previous = claims.get(id);
|
||||
if (previous !== undefined && previous !== targetKey) invalid.add(id);
|
||||
claims.set(id, targetKey);
|
||||
}
|
||||
const accepted = new Set(batch.frames.map((frame) => frame.frameId));
|
||||
for (const id of batch.snapshotFrameIds ?? []) if (!accepted.has(id)) invalid.add(id);
|
||||
}
|
||||
const currentFrames = batches.flatMap((batch) =>
|
||||
batch.frames.filter(
|
||||
(frame) =>
|
||||
!claims.has(frame.frameId) || claims.get(frame.frameId) === cdpTargetKey(batch.target),
|
||||
),
|
||||
);
|
||||
const rootBatch = batches.find((batch) => !batch.target.sessionId);
|
||||
const rootFrameId =
|
||||
rootBatch?.rootFrameId ?? graph?.rootFrameId ?? rootBatch?.frames[0]?.frameId ?? "root";
|
||||
const ownershipInvalid = new Set(invalid);
|
||||
const identities = new Map<string, DocumentIdentity>();
|
||||
const selected = new Map(currentFrames.map((frame) => [frame.frameId, frame]));
|
||||
const checks = batches.flatMap((batch) => {
|
||||
const docs = new Map(batch.documents.map((doc) => [doc.frame.frameId, doc]));
|
||||
return batch.frames
|
||||
.filter(
|
||||
(frame) =>
|
||||
!invalid.has(frame.frameId) &&
|
||||
cdpTargetKey(selected.get(frame.frameId)!.target) === cdpTargetKey(batch.target),
|
||||
)
|
||||
.map((frame) => ({ batch, frame, doc: docs.get(frame.frameId) }));
|
||||
});
|
||||
await collectTasks(
|
||||
checks,
|
||||
async ({ batch, frame, doc }) => {
|
||||
throwCaptureAborted(signal);
|
||||
if (!batch.snapshotAttachmentId || doc?.documentElementBackendNodeId === undefined) {
|
||||
issues.push({
|
||||
target: frame.target,
|
||||
frameId: frame.frameId,
|
||||
stage: "identity",
|
||||
reason: "identity-unverified",
|
||||
});
|
||||
return;
|
||||
}
|
||||
const identity: DocumentIdentity = {
|
||||
attachmentId: batch.snapshotAttachmentId,
|
||||
target: frame.target,
|
||||
frameId: frame.frameId,
|
||||
documentElementBackendNodeId: doc.documentElementBackendNodeId,
|
||||
};
|
||||
const verified = await verifyDocumentIdentity(cdp, identity, signal);
|
||||
const status = cdp.getAttachmentId?.(tabId) === identity.attachmentId ? verified : "changed";
|
||||
if (status === "current") identities.set(frame.frameId, identity);
|
||||
else {
|
||||
invalid.add(frame.frameId);
|
||||
issues.push({
|
||||
target: frame.target,
|
||||
frameId: frame.frameId,
|
||||
stage: "identity",
|
||||
reason: status === "changed" ? "document-changed" : "identity-unavailable",
|
||||
});
|
||||
}
|
||||
},
|
||||
signal,
|
||||
);
|
||||
const children = new Map<string, string[]>();
|
||||
for (const frame of currentFrames) {
|
||||
if (!frame.parentFrameId) continue;
|
||||
const siblings = children.get(frame.parentFrameId);
|
||||
if (siblings) siblings.push(frame.frameId);
|
||||
else children.set(frame.parentFrameId, [frame.frameId]);
|
||||
}
|
||||
const blocked = [...invalid];
|
||||
for (let i = 0; i < blocked.length; i++) {
|
||||
for (const child of children.get(blocked[i]) ?? []) {
|
||||
if (!invalid.has(child)) {
|
||||
invalid.add(child);
|
||||
blocked.push(child);
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const frame of currentFrames) {
|
||||
if (ownershipInvalid.has(frame.frameId))
|
||||
issues.push({
|
||||
target: frame.target,
|
||||
frameId: frame.frameId,
|
||||
stage: "ownership",
|
||||
reason: "frame-ownership-unresolved",
|
||||
});
|
||||
}
|
||||
if (invalid.has(rootFrameId))
|
||||
throw new Error(
|
||||
ownershipInvalid.has(rootFrameId)
|
||||
? "observation root document ownership is ambiguous"
|
||||
: "observation document identity changed or could not be verified; observe again",
|
||||
);
|
||||
const frames = currentFrames.filter((frame) => !invalid.has(frame.frameId));
|
||||
const frameById = new Map(frames.map((frame) => [frame.frameId, frame]));
|
||||
const belongs = (frame: CdpFrame) => {
|
||||
const current = frameById.get(frame.frameId);
|
||||
return current && cdpTargetKey(current.target) === cdpTargetKey(frame.target);
|
||||
};
|
||||
const decoded = new Map(
|
||||
batches
|
||||
.flatMap((batch) => batch.documents)
|
||||
.filter((doc) => belongs(doc.frame))
|
||||
.map((doc) => [doc.frame.frameId, doc]),
|
||||
);
|
||||
const ax = batches.flatMap((batch) => batch.ax).filter((batch) => belongs(batch.frame));
|
||||
const documents = await buildFrameDocuments(
|
||||
{ rootFrameId, frames },
|
||||
ax,
|
||||
{
|
||||
nodes: [],
|
||||
rootFrameId,
|
||||
frameNodes: new Map([...decoded].map(([id, doc]) => [id, doc.nodes])),
|
||||
frameExcludedBackendNodeIds: new Map(
|
||||
[...decoded].map(([id, doc]) => [id, doc.index.excludedBackendNodeIds]),
|
||||
),
|
||||
},
|
||||
signal,
|
||||
(frame) =>
|
||||
issues.push({
|
||||
target: frame.target,
|
||||
frameId: frame.frameId,
|
||||
stage: "ownership",
|
||||
reason: "frame-ownership-unresolved",
|
||||
}),
|
||||
);
|
||||
const facts: ObservationFacts<T>["documents"][number][] = [];
|
||||
for (const document of documents) {
|
||||
const doc = decoded.get(document.frameId);
|
||||
const index = doc?.index ?? (await buildDocumentIndex([], signal));
|
||||
const fallback = groups.get(cdpTargetKey(document.target))?.fallbackExcluded;
|
||||
const {
|
||||
domNodes,
|
||||
axNodes,
|
||||
contextScopeId: _scope,
|
||||
excludedBackendNodeIds: _excluded,
|
||||
...frame
|
||||
} = document;
|
||||
facts.push({
|
||||
frame,
|
||||
identity: identities.get(document.frameId),
|
||||
index: fallback?.size ? { ...index, excludedBackendNodeIds: fallback } : index,
|
||||
domNodes,
|
||||
axNodes,
|
||||
});
|
||||
}
|
||||
throwCaptureAborted(signal);
|
||||
if (firstFailure && facts.every((doc) => !doc.domNodes.length && !doc.axNodes.length))
|
||||
throw firstFailure;
|
||||
return { rootFrameId, viewport, documents: facts, issues, startedAt, finishedAt: Date.now() };
|
||||
}
|
||||
|
||||
/** Existing semantic consumers get a narrow view; no raw snapshot
|
||||
* data can leak through recording's allowlisted output DTO. */
|
||||
export function semanticCapture<T extends FrameOwnedAxNode>(
|
||||
facts: ObservationFacts<T>,
|
||||
): { captured: CapturedSceneInput; documents: FrameDocument<T>[] } {
|
||||
const documents = facts.documents.map((doc) => ({
|
||||
...doc.frame,
|
||||
contextScopeId: doc.frame.frameId,
|
||||
domNodes: doc.domNodes.filter(
|
||||
(node) => !doc.index.excludedBackendNodeIds.has(node.backendNodeId),
|
||||
),
|
||||
axNodes: doc.axNodes,
|
||||
excludedBackendNodeIds: doc.index.excludedBackendNodeIds,
|
||||
}));
|
||||
const root = documents.find((doc) => doc.frameId === facts.rootFrameId);
|
||||
return {
|
||||
documents,
|
||||
captured: {
|
||||
nodes: root?.domNodes ?? [],
|
||||
viewport: facts.viewport,
|
||||
rootFrameId: facts.rootFrameId,
|
||||
excludedBackendNodeIds: root?.excludedBackendNodeIds ?? new Set(),
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -1,37 +0,0 @@
|
||||
import type { Rect } from "@browser-skill/vom";
|
||||
|
||||
export interface CapturedNode {
|
||||
backendNodeId: number;
|
||||
parentBackendNodeId: number | null;
|
||||
frameId?: string;
|
||||
/** Owning iframe backend node id; `null` for the top-level document. */
|
||||
ownerFrameBackendNodeId?: number | null;
|
||||
tag: string;
|
||||
attrs: Record<string, string>;
|
||||
/** Top-level viewport-relative CSS px, clipped to the owning frame viewport. */
|
||||
rect: Rect | null;
|
||||
/** Frame-local viewport-relative CSS px before top-level projection. */
|
||||
localRect?: Rect | null;
|
||||
paintOrder: number;
|
||||
position: string;
|
||||
pointerEvents: string;
|
||||
/**
|
||||
* computed `cursor`. `cursor: pointer` is the strongest CDP-free signal
|
||||
* that a non-semantic element (a `<div>`/`<span>` with a click handler)
|
||||
* is actually an interactive control — used by the adapter to surface
|
||||
* custom buttons/checkboxes the AX tree drops as `generic`. Optional like
|
||||
* `textContent`: the live parser always sets it, hand-built fixtures may not.
|
||||
*/
|
||||
cursor?: string;
|
||||
/**
|
||||
* Whether the live DOM snapshot provides a painted, non-hidden box for this
|
||||
* node. Semantic resolution uses this only for DOM fallback nodes; AX-backed
|
||||
* nodes remain authoritative even when they are outside the viewport.
|
||||
*/
|
||||
rendered?: boolean;
|
||||
textContent?: string;
|
||||
formState?: "empty" | "filled" | "default";
|
||||
formValue?: string;
|
||||
formDefaultValue?: string;
|
||||
formPlaceholder?: string;
|
||||
}
|
||||
@@ -1,19 +1,298 @@
|
||||
// Browser-side hover probes and overlay exclusion fallback. Static collection
|
||||
// lives in capture-coordinator; snapshot decoding has no browser dependency.
|
||||
// CDP capture adapter: DOMSnapshot.captureSnapshot + Page.getLayoutMetrics
|
||||
// → CapturedNode[] + Viewport. This is the ONLY VOM module that touches
|
||||
// raw CDP. The captureSnapshot reply is columnar (parallel arrays + a
|
||||
// shared string table); requested computed styles are decoded through
|
||||
// STYLE_COL so capture and visibility policy share one explicit contract.
|
||||
|
||||
import type { Rect, Viewport } from "@browser-skill/vom";
|
||||
import { evaluateHoverTrigger } from "@/lib/hover-trigger-policy";
|
||||
import { OVERLAY_HOST_SELECTOR } from "../../lib/overlay-bridge";
|
||||
import { isOverlayHostNode, OVERLAY_HOST_SELECTOR } from "../../lib/overlay-bridge";
|
||||
import { childFrameProjection, type GeometryProjection, projectRectToViewport } from "../geometry";
|
||||
import type { CdpRunner } from "../shared";
|
||||
import type { CapturedNode } from "./capture-types";
|
||||
import type { CapturedSurfaceProbe } from "./facts";
|
||||
|
||||
export type {
|
||||
CapturedNode,
|
||||
CapturedSurfaceProbe,
|
||||
} from "./facts";
|
||||
|
||||
import { isAbortError, throwIfAborted } from "./capture-abort";
|
||||
import { clearHover, ProbeBudget, waitForHover } from "./hover-perception";
|
||||
|
||||
const REQUESTED_STYLES = ["position", "pointer-events", "cursor", "visibility", "opacity"] as const;
|
||||
const STYLE_COL = Object.fromEntries(
|
||||
REQUESTED_STYLES.map((name, index) => [name, index]),
|
||||
) as Record<(typeof REQUESTED_STYLES)[number], number>;
|
||||
|
||||
export interface CapturedNode {
|
||||
backendNodeId: number;
|
||||
parentBackendNodeId: number | null;
|
||||
frameId?: string;
|
||||
/** Owning iframe backend node id; `null` for the top-level document. */
|
||||
ownerFrameBackendNodeId?: number | null;
|
||||
tag: string;
|
||||
attrs: Record<string, string>;
|
||||
/** Top-level viewport-relative CSS px, clipped to the owning frame viewport. */
|
||||
rect: Rect | null;
|
||||
/** Frame-local viewport-relative CSS px before top-level projection. */
|
||||
localRect?: Rect | null;
|
||||
paintOrder: number;
|
||||
position: string;
|
||||
pointerEvents: string;
|
||||
/**
|
||||
* computed `cursor`. `cursor: pointer` is the strongest CDP-free signal
|
||||
* that a non-semantic element (a `<div>`/`<span>` with a click handler)
|
||||
* is actually an interactive control — used by the adapter to surface
|
||||
* custom buttons/checkboxes the AX tree drops as `generic`. Optional like
|
||||
* `textContent`: the live parser always sets it, hand-built fixtures may not.
|
||||
*/
|
||||
cursor?: string;
|
||||
/**
|
||||
* Whether the live DOM snapshot provides a painted, non-hidden box for this
|
||||
* node. Semantic resolution uses this only for DOM fallback nodes; AX-backed
|
||||
* nodes remain authoritative even when they are outside the viewport.
|
||||
*/
|
||||
rendered?: boolean;
|
||||
textContent?: string;
|
||||
formState?: "empty" | "filled" | "default";
|
||||
formValue?: string;
|
||||
formDefaultValue?: string;
|
||||
formPlaceholder?: string;
|
||||
}
|
||||
|
||||
export type CapturedIframeNodes = Map<number, CapturedNode[]>;
|
||||
|
||||
export interface CapturedSurfaceProbe {
|
||||
triggerBackendNodeId: number;
|
||||
triggerPoint?: { x: number; y: number };
|
||||
triggerAction: "hover" | "focus" | string;
|
||||
subItems: string[];
|
||||
confidence?: "high" | "medium" | "low";
|
||||
}
|
||||
|
||||
export interface CapturedViewModel {
|
||||
nodes: CapturedNode[];
|
||||
viewport: Viewport;
|
||||
iframeNodes: CapturedIframeNodes;
|
||||
frameNodes?: Map<string, CapturedNode[]>;
|
||||
frameOwnerBackendNodeIds?: Map<string, number>;
|
||||
/** Explicit DOMSnapshot frame ancestry; never inferred from backend node ids. */
|
||||
frameParentIds?: Map<string, string>;
|
||||
rootFrameId?: string;
|
||||
/** Backend node ids belonging to the agent overlay host + its shadow subtree. */
|
||||
excludedBackendNodeIds: Set<number>;
|
||||
}
|
||||
|
||||
export interface CaptureViewModelOptions {
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
function captureAbortError(): Error {
|
||||
const error = new Error("observation aborted");
|
||||
error.name = "AbortError";
|
||||
return error;
|
||||
}
|
||||
|
||||
function isAbortError(error: unknown): boolean {
|
||||
return (
|
||||
typeof error === "object" &&
|
||||
error !== null &&
|
||||
(error as { name?: string }).name === "AbortError"
|
||||
);
|
||||
}
|
||||
|
||||
function throwIfAborted(signal: AbortSignal | undefined): void {
|
||||
if (signal?.aborted) throw captureAbortError();
|
||||
}
|
||||
|
||||
/** Sparse array format Chrome uses for infrequently-set per-node fields. */
|
||||
interface SparseArray {
|
||||
index: number[];
|
||||
value: number[];
|
||||
}
|
||||
|
||||
interface RareBooleanData {
|
||||
index: number[];
|
||||
}
|
||||
|
||||
interface SnapshotDocument {
|
||||
frameId?: string | number;
|
||||
scrollOffsetX?: number;
|
||||
scrollOffsetY?: number;
|
||||
nodes?: {
|
||||
parentIndex?: number[];
|
||||
nodeName?: number[];
|
||||
backendNodeId?: number[];
|
||||
attributes?: number[][];
|
||||
/**
|
||||
* Per-node text value (index into strings), set for `#text` / CDATA nodes.
|
||||
* Element nodes carry -1. Same length as `backendNodeId`.
|
||||
*/
|
||||
nodeValue?: number[];
|
||||
/** Maps node array index → index into `documents[]` for frame content. */
|
||||
contentDocumentIndex?: SparseArray;
|
||||
inputValue?: SparseArray;
|
||||
textValue?: SparseArray;
|
||||
inputChecked?: RareBooleanData;
|
||||
optionSelected?: RareBooleanData;
|
||||
};
|
||||
layout?: {
|
||||
nodeIndex?: number[];
|
||||
styles?: number[][];
|
||||
bounds?: number[][];
|
||||
paintOrders?: number[];
|
||||
};
|
||||
}
|
||||
|
||||
function snapshotFrameId(document: SnapshotDocument, strings: string[]): string | undefined {
|
||||
if (typeof document.frameId === "string") return document.frameId || undefined;
|
||||
if (typeof document.frameId === "number") return str(strings, document.frameId) || undefined;
|
||||
return undefined;
|
||||
}
|
||||
|
||||
interface SnapshotReply {
|
||||
strings?: string[];
|
||||
documents?: SnapshotDocument[];
|
||||
}
|
||||
|
||||
interface LayoutMetricsReply {
|
||||
cssLayoutViewport?: {
|
||||
clientWidth?: number;
|
||||
clientHeight?: number;
|
||||
pageX?: number;
|
||||
pageY?: number;
|
||||
};
|
||||
layoutViewport?: { clientWidth?: number; clientHeight?: number; pageX?: number; pageY?: number };
|
||||
}
|
||||
|
||||
function isFormControlTag(tag: string): boolean {
|
||||
return tag === "input" || tag === "textarea" || tag === "select";
|
||||
}
|
||||
|
||||
function isSensitiveFormControl(node: Pick<CapturedNode, "tag" | "attrs">): boolean {
|
||||
return node.tag === "input" && (node.attrs.type ?? "").toLowerCase() === "password";
|
||||
}
|
||||
|
||||
function snapshotFormState(
|
||||
value: string | undefined,
|
||||
defaultValue: string,
|
||||
hasDefaultValue: boolean,
|
||||
sensitive: boolean,
|
||||
): CapturedNode["formState"] {
|
||||
if (sensitive) {
|
||||
if (value === undefined && !hasDefaultValue) return undefined;
|
||||
return (value ?? defaultValue) === "" ? "empty" : "filled";
|
||||
}
|
||||
if (value === undefined) return undefined;
|
||||
if (value === "") return "empty";
|
||||
return value === defaultValue ? "default" : "filled";
|
||||
}
|
||||
|
||||
const MAX_FORM_ENRICH_CONTROLS = 250;
|
||||
|
||||
interface CapturedFormState {
|
||||
value?: string;
|
||||
defaultValue?: string;
|
||||
placeholder?: string;
|
||||
state?: "empty" | "filled" | "default";
|
||||
sensitive?: boolean;
|
||||
}
|
||||
|
||||
interface CapturedFrameFormState {
|
||||
controls: CapturedFormState[];
|
||||
childFrames: CapturedFrameFormState[];
|
||||
}
|
||||
|
||||
function formStateBatchExpression(maxControls: number): string {
|
||||
return `(() => {
|
||||
const maxControls = ${JSON.stringify(maxControls)};
|
||||
let remaining = maxControls;
|
||||
const controlSelector = "input,textarea,select";
|
||||
const controlState = (el) => {
|
||||
const tag = el.tagName.toLowerCase();
|
||||
const type = tag === "input" ? String(el.type || "text").toLowerCase() : tag;
|
||||
const sensitive = type === "password" || type === "credit-card";
|
||||
const rawValue = typeof el.value === "string" ? el.value : "";
|
||||
const defaultValue = typeof el.defaultValue === "string" ? el.defaultValue : "";
|
||||
const placeholder = typeof el.placeholder === "string" ? el.placeholder : "";
|
||||
const state = rawValue === "" ? "empty" : rawValue === defaultValue ? "default" : "filled";
|
||||
return {
|
||||
state,
|
||||
sensitive,
|
||||
placeholder,
|
||||
...(sensitive ? {} : { value: rawValue, defaultValue }),
|
||||
};
|
||||
};
|
||||
const collect = (doc) => {
|
||||
const controls = [];
|
||||
for (const el of Array.from(doc.querySelectorAll(controlSelector))) {
|
||||
if (remaining <= 0) break;
|
||||
controls.push(controlState(el));
|
||||
remaining -= 1;
|
||||
}
|
||||
const childFrames = [];
|
||||
for (const frame of Array.from(doc.querySelectorAll("iframe"))) {
|
||||
let childDoc = null;
|
||||
try { childDoc = frame.contentDocument; } catch { childDoc = null; }
|
||||
childFrames.push(childDoc && remaining > 0 ? collect(childDoc) : { controls: [], childFrames: [] });
|
||||
}
|
||||
return { controls, childFrames };
|
||||
};
|
||||
return collect(document);
|
||||
})()`;
|
||||
}
|
||||
|
||||
function flattenFrameFormStates(
|
||||
frame: CapturedFrameFormState | undefined,
|
||||
out: CapturedFormState[][] = [],
|
||||
): CapturedFormState[][] {
|
||||
if (!frame) return out;
|
||||
out.push(Array.isArray(frame.controls) ? frame.controls : []);
|
||||
for (const child of Array.isArray(frame.childFrames) ? frame.childFrames : []) {
|
||||
flattenFrameFormStates(child, out);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function applyFormStates(nodes: CapturedNode[], states: CapturedFormState[]): void {
|
||||
let index = 0;
|
||||
for (const node of nodes) {
|
||||
if (!isFormControlTag(node.tag)) continue;
|
||||
const state = states[index];
|
||||
index += 1;
|
||||
if (!state) continue;
|
||||
node.formState = state.state;
|
||||
node.formPlaceholder = state.placeholder ?? "";
|
||||
if (state.sensitive || isSensitiveFormControl(node)) {
|
||||
delete node.formValue;
|
||||
delete node.formDefaultValue;
|
||||
delete node.attrs.value;
|
||||
} else {
|
||||
node.formDefaultValue = state.defaultValue ?? "";
|
||||
if (state.value !== undefined) node.formValue = state.value;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function enrichFormControlStates(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
frameNodeGroups: CapturedNode[][],
|
||||
signal?: AbortSignal,
|
||||
): Promise<void> {
|
||||
const hasControls = frameNodeGroups.some((nodes) =>
|
||||
nodes.some((node) => isFormControlTag(node.tag)),
|
||||
);
|
||||
if (!hasControls) return;
|
||||
try {
|
||||
throwIfAborted(signal);
|
||||
const result = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
|
||||
expression: formStateBatchExpression(MAX_FORM_ENRICH_CONTROLS),
|
||||
returnByValue: true,
|
||||
});
|
||||
throwIfAborted(signal);
|
||||
const frameStates = flattenFrameFormStates(runtimeValue<CapturedFrameFormState>(result));
|
||||
for (let i = 0; i < frameNodeGroups.length; i += 1) {
|
||||
applyFormStates(frameNodeGroups[i], frameStates[i] ?? []);
|
||||
}
|
||||
} catch (error) {
|
||||
if (isAbortError(error)) throw error;
|
||||
// Best-effort enrichment. DOMSnapshot/AX data still carries the nodes.
|
||||
}
|
||||
}
|
||||
|
||||
interface RuntimeEvaluateReply {
|
||||
result?: {
|
||||
value?: unknown;
|
||||
@@ -35,6 +314,44 @@ interface CdpDomNode {
|
||||
shadowRoots?: CdpDomNode[];
|
||||
}
|
||||
|
||||
interface ParseDocumentResult {
|
||||
nodes: CapturedNode[];
|
||||
excludedBackendNodeIds: Set<number>;
|
||||
}
|
||||
|
||||
interface FrameContext {
|
||||
frameId?: string;
|
||||
ownerFrameBackendNodeId: number | null;
|
||||
projection: GeometryProjection | null;
|
||||
scrollX: number;
|
||||
scrollY: number;
|
||||
}
|
||||
|
||||
function str(strings: string[], idx: number | undefined): string {
|
||||
if (idx === undefined || idx < 0) return "";
|
||||
return strings[idx] ?? "";
|
||||
}
|
||||
|
||||
function devicePixelRatio(metrics: LayoutMetricsReply): number {
|
||||
const layoutW = metrics.layoutViewport?.clientWidth ?? 0;
|
||||
const cssW = metrics.cssLayoutViewport?.clientWidth ?? 0;
|
||||
if (!layoutW || !cssW) return 1;
|
||||
const dpr = layoutW / cssW;
|
||||
if (!Number.isFinite(dpr) || dpr <= 0) return 1;
|
||||
return dpr >= 1 ? dpr : 1;
|
||||
}
|
||||
|
||||
function sparseIndexMap(sparse: SparseArray | undefined): Map<number, number> {
|
||||
const out = new Map<number, number>();
|
||||
if (!sparse?.index || !sparse.value) return out;
|
||||
for (let i = 0; i < sparse.index.length; i++) {
|
||||
const nodeIndex = sparse.index[i];
|
||||
const docIndex = sparse.value[i];
|
||||
if (nodeIndex !== undefined && docIndex !== undefined) out.set(nodeIndex, docIndex);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function collectBackendIdsFromDomNode(node: CdpDomNode | undefined, out: Set<number>): void {
|
||||
if (!node) return;
|
||||
if (typeof node.backendNodeId === "number") {
|
||||
@@ -443,3 +760,358 @@ export async function collectOverlayExcludedBackendIds(
|
||||
}
|
||||
return excluded;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse one DOMSnapshot document into CapturedNode[].
|
||||
*
|
||||
* @param doc - the raw DOMSnapshot document object
|
||||
* @param strings - the shared string table for the whole snapshot
|
||||
* @param dpr - device-pixel-ratio from Page.getLayoutMetrics
|
||||
* @param context - frame coordinate context; output rects are top viewport-relative
|
||||
*/
|
||||
function parseDocumentNodes(
|
||||
doc: SnapshotDocument,
|
||||
strings: string[],
|
||||
dpr: number,
|
||||
context: FrameContext,
|
||||
): ParseDocumentResult {
|
||||
const dn = doc.nodes;
|
||||
const dl = doc.layout;
|
||||
if (!dn?.backendNodeId) {
|
||||
return { nodes: [], excludedBackendNodeIds: new Set() };
|
||||
}
|
||||
|
||||
const count = dn.backendNodeId.length;
|
||||
const layoutByNode = new Map<number, number>();
|
||||
(dl?.nodeIndex ?? []).forEach((nodeIdx, layoutIdx) => layoutByNode.set(nodeIdx, layoutIdx));
|
||||
|
||||
// The agent's own overlay (WXT shadow host, marked with OVERLAY_HOST_MARKER_ATTR)
|
||||
// is injected into the page and its open shadow root
|
||||
// is inlined by DOMSnapshot as descendants of the host. Its fixed
|
||||
// full-viewport click-blocker would otherwise dominate occlusion detection
|
||||
// and hide the real page, so mark the host + its whole subtree excluded.
|
||||
// parentIndex always references an earlier node, so one forward pass suffices.
|
||||
const excluded = new Array<boolean>(count).fill(false);
|
||||
const excludedBackendNodeIds = new Set<number>();
|
||||
for (let n = 0; n < count; n++) {
|
||||
const parentIdx = dn.parentIndex?.[n] ?? -1;
|
||||
const inherited = parentIdx >= 0 && excluded[parentIdx];
|
||||
let isOverlayHost = inherited;
|
||||
if (!isOverlayHost) {
|
||||
const pairs = dn.attributes?.[n] ?? [];
|
||||
const attrNames: string[] = [];
|
||||
for (let a = 0; a + 1 < pairs.length; a += 2) {
|
||||
attrNames.push(str(strings, pairs[a]));
|
||||
}
|
||||
isOverlayHost = isOverlayHostNode(str(strings, dn.nodeName?.[n]), attrNames);
|
||||
}
|
||||
if (!isOverlayHost) continue;
|
||||
excluded[n] = true;
|
||||
const backendNodeId = dn.backendNodeId[n];
|
||||
if (backendNodeId !== undefined) {
|
||||
excludedBackendNodeIds.add(backendNodeId);
|
||||
}
|
||||
}
|
||||
|
||||
const posCol = STYLE_COL.position;
|
||||
const peCol = STYLE_COL["pointer-events"];
|
||||
const cursorCol = STYLE_COL.cursor;
|
||||
const visibilityCol = STYLE_COL.visibility;
|
||||
const opacityCol = STYLE_COL.opacity;
|
||||
const inputValues = sparseIndexMap(dn.inputValue);
|
||||
const textValues = sparseIndexMap(dn.textValue);
|
||||
const checkedInputs = new Set(dn.inputChecked?.index ?? []);
|
||||
const selectedOptions = new Set(dn.optionSelected?.index ?? []);
|
||||
|
||||
// Collect visible text from #text child nodes so element CapturedNodes
|
||||
// carry a textContent value usable as a button/link label fallback.
|
||||
// nodeValue is a parallel array: string index for text nodes, -1 otherwise.
|
||||
const nodeTextContent = new Map<number, string>();
|
||||
if (dn.nodeValue) {
|
||||
for (let n = 0; n < count; n++) {
|
||||
const nvIdx = dn.nodeValue[n] ?? -1;
|
||||
if (nvIdx < 0) continue;
|
||||
const text = str(strings, nvIdx).trim();
|
||||
if (!text) continue;
|
||||
const parentIdx = dn.parentIndex?.[n] ?? -1;
|
||||
if (parentIdx >= 0) {
|
||||
const existing = nodeTextContent.get(parentIdx);
|
||||
nodeTextContent.set(parentIdx, existing ? `${existing} ${text}` : text);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Build CapturedNodes.
|
||||
const nodes: CapturedNode[] = [];
|
||||
for (let n = 0; n < count; n++) {
|
||||
if (excluded[n]) continue;
|
||||
const backendNodeId = dn.backendNodeId[n];
|
||||
const parentIdx = dn.parentIndex?.[n] ?? -1;
|
||||
const parentBackendNodeId = parentIdx >= 0 ? (dn.backendNodeId[parentIdx] ?? null) : null;
|
||||
const tag = str(strings, dn.nodeName?.[n]).toLowerCase();
|
||||
|
||||
const attrs: Record<string, string> = {};
|
||||
const pairs = dn.attributes?.[n] ?? [];
|
||||
for (let a = 0; a + 1 < pairs.length; a += 2) {
|
||||
attrs[str(strings, pairs[a]).toLowerCase()] = str(strings, pairs[a + 1]);
|
||||
}
|
||||
|
||||
let rect: Rect | null = null;
|
||||
let localRect: Rect | null = null;
|
||||
let paintOrder = 0;
|
||||
let position = "static";
|
||||
let pointerEvents = "auto";
|
||||
let cursor = "auto";
|
||||
let rendered = false;
|
||||
const li = layoutByNode.get(n);
|
||||
if (li !== undefined) {
|
||||
const b = dl?.bounds?.[li];
|
||||
if (b && b.length >= 4 && b[2] > 0 && b[3] > 0) {
|
||||
localRect = {
|
||||
x: b[0] / dpr - context.scrollX,
|
||||
y: b[1] / dpr - context.scrollY,
|
||||
w: b[2] / dpr,
|
||||
h: b[3] / dpr,
|
||||
};
|
||||
const bounds = context.projection
|
||||
? projectRectToViewport(localRect, context.projection)
|
||||
: null;
|
||||
rect = bounds ? { x: bounds.x, y: bounds.y, w: bounds.width, h: bounds.height } : null;
|
||||
}
|
||||
paintOrder = dl?.paintOrders?.[li] ?? 0;
|
||||
const styleRow = dl?.styles?.[li] ?? [];
|
||||
position = str(strings, styleRow[posCol]) || "static";
|
||||
pointerEvents = str(strings, styleRow[peCol]) || "auto";
|
||||
cursor = str(strings, styleRow[cursorCol]) || "auto";
|
||||
const visibility = str(strings, styleRow[visibilityCol]) || "visible";
|
||||
const opacity = str(strings, styleRow[opacityCol]) || "1";
|
||||
rendered =
|
||||
localRect !== null &&
|
||||
visibility !== "hidden" &&
|
||||
visibility !== "collapse" &&
|
||||
(Number.parseFloat(opacity) || 0) > 0;
|
||||
}
|
||||
|
||||
// Skip non-element nodes (#text, #cdata-section, etc.) — they carry no
|
||||
// geometry and are never queried by callers.
|
||||
if (tag.startsWith("#")) continue;
|
||||
|
||||
const textContent = nodeTextContent.get(n);
|
||||
|
||||
const rawFormValueIndex = tag === "textarea" ? textValues.get(n) : inputValues.get(n);
|
||||
const rawFormValue =
|
||||
rawFormValueIndex !== undefined ? str(strings, rawFormValueIndex) : undefined;
|
||||
const sensitive = isSensitiveFormControl({ tag, attrs });
|
||||
const formDefaultValue = attrs.value ?? "";
|
||||
const formValue = sensitive ? undefined : rawFormValue;
|
||||
const formState = snapshotFormState(
|
||||
rawFormValue,
|
||||
formDefaultValue,
|
||||
Object.prototype.hasOwnProperty.call(attrs, "value"),
|
||||
sensitive,
|
||||
);
|
||||
if (sensitive) delete attrs.value;
|
||||
|
||||
nodes.push({
|
||||
backendNodeId,
|
||||
parentBackendNodeId,
|
||||
...(context.frameId ? { frameId: context.frameId } : {}),
|
||||
ownerFrameBackendNodeId: context.ownerFrameBackendNodeId,
|
||||
tag,
|
||||
attrs,
|
||||
rect,
|
||||
localRect,
|
||||
paintOrder,
|
||||
position,
|
||||
pointerEvents,
|
||||
cursor,
|
||||
rendered,
|
||||
textContent,
|
||||
...(formValue !== undefined ? { formValue } : {}),
|
||||
...(tag === "input" || tag === "textarea"
|
||||
? {
|
||||
formPlaceholder: attrs.placeholder ?? "",
|
||||
...(!sensitive ? { formDefaultValue } : {}),
|
||||
...(formState ? { formState } : {}),
|
||||
}
|
||||
: {}),
|
||||
...(checkedInputs.has(n) ? { formValue: "true", formState: "filled" } : {}),
|
||||
...(selectedOptions.has(n) ? { formValue: attrs.value ?? textContent ?? "" } : {}),
|
||||
});
|
||||
}
|
||||
return { nodes, excludedBackendNodeIds };
|
||||
}
|
||||
|
||||
interface ParseFrameDocumentsResult {
|
||||
iframeNodes: CapturedIframeNodes;
|
||||
frameOwnerBackendNodeIds: Map<string, number>;
|
||||
frameParentIds: Map<string, string>;
|
||||
excludedBackendNodeIds: Set<number>;
|
||||
}
|
||||
|
||||
function parseChildFrameDocuments(
|
||||
documents: SnapshotDocument[],
|
||||
strings: string[],
|
||||
dpr: number,
|
||||
parentDocIndex: number,
|
||||
parentNodes: CapturedNode[],
|
||||
parentContext: FrameContext,
|
||||
visited = new Set<number>(),
|
||||
): ParseFrameDocumentsResult {
|
||||
const iframeNodes: CapturedIframeNodes = new Map();
|
||||
const frameOwnerBackendNodeIds = new Map<string, number>();
|
||||
const frameParentIds = new Map<string, string>();
|
||||
const excludedBackendNodeIds = new Set<number>();
|
||||
const parentDoc = documents[parentDocIndex];
|
||||
const cdi = sparseIndexMap(parentDoc?.nodes?.contentDocumentIndex);
|
||||
if (cdi.size === 0) {
|
||||
return { iframeNodes, frameOwnerBackendNodeIds, frameParentIds, excludedBackendNodeIds };
|
||||
}
|
||||
|
||||
const parentBackendIds = parentDoc?.nodes?.backendNodeId ?? [];
|
||||
const parentNodeByBackendId = new Map(parentNodes.map((node) => [node.backendNodeId, node]));
|
||||
|
||||
for (const [nodeArrayIdx, childDocIndex] of cdi) {
|
||||
if (visited.has(childDocIndex)) continue;
|
||||
const childDoc = documents[childDocIndex];
|
||||
if (!childDoc) continue;
|
||||
const iframeBackendId = parentBackendIds[nodeArrayIdx];
|
||||
if (iframeBackendId === undefined) continue;
|
||||
const iframeNode = parentNodeByBackendId.get(iframeBackendId);
|
||||
const projection =
|
||||
parentContext.projection && iframeNode?.localRect
|
||||
? childFrameProjection(parentContext.projection, iframeNode.localRect)
|
||||
: null;
|
||||
|
||||
const childContext: FrameContext = {
|
||||
frameId: snapshotFrameId(childDoc, strings),
|
||||
ownerFrameBackendNodeId: iframeBackendId,
|
||||
projection,
|
||||
scrollX: childDoc.scrollOffsetX ?? 0,
|
||||
scrollY: childDoc.scrollOffsetY ?? 0,
|
||||
};
|
||||
const nextVisited = new Set(visited);
|
||||
nextVisited.add(childDocIndex);
|
||||
const parsed = parseDocumentNodes(childDoc, strings, dpr, childContext);
|
||||
iframeNodes.set(iframeBackendId, parsed.nodes);
|
||||
if (childContext.frameId) {
|
||||
frameOwnerBackendNodeIds.set(childContext.frameId, iframeBackendId);
|
||||
if (parentContext.frameId) frameParentIds.set(childContext.frameId, parentContext.frameId);
|
||||
}
|
||||
for (const id of parsed.excludedBackendNodeIds) excludedBackendNodeIds.add(id);
|
||||
|
||||
const nested = parseChildFrameDocuments(
|
||||
documents,
|
||||
strings,
|
||||
dpr,
|
||||
childDocIndex,
|
||||
parsed.nodes,
|
||||
childContext,
|
||||
nextVisited,
|
||||
);
|
||||
for (const [nestedFrameId, nestedNodes] of nested.iframeNodes) {
|
||||
iframeNodes.set(nestedFrameId, nestedNodes);
|
||||
}
|
||||
for (const [frameId, ownerBackendNodeId] of nested.frameOwnerBackendNodeIds) {
|
||||
frameOwnerBackendNodeIds.set(frameId, ownerBackendNodeId);
|
||||
}
|
||||
for (const [frameId, parentFrameId] of nested.frameParentIds) {
|
||||
frameParentIds.set(frameId, parentFrameId);
|
||||
}
|
||||
for (const id of nested.excludedBackendNodeIds) excludedBackendNodeIds.add(id);
|
||||
}
|
||||
|
||||
return { iframeNodes, frameOwnerBackendNodeIds, frameParentIds, excludedBackendNodeIds };
|
||||
}
|
||||
|
||||
export async function captureViewModel(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
options: CaptureViewModelOptions = {},
|
||||
): Promise<CapturedViewModel> {
|
||||
throwIfAborted(options.signal);
|
||||
let metrics: LayoutMetricsReply = {};
|
||||
try {
|
||||
metrics = await cdp.send<LayoutMetricsReply>(tabId, "Page.getLayoutMetrics", {});
|
||||
} catch (error) {
|
||||
if (isAbortError(error)) throw error;
|
||||
console.debug("[bsk capture] layout metrics unavailable", error);
|
||||
}
|
||||
throwIfAborted(options.signal);
|
||||
const dpr = devicePixelRatio(metrics);
|
||||
const vpSrc = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
|
||||
const viewport: Viewport = {
|
||||
width: vpSrc.clientWidth ?? 0,
|
||||
height: vpSrc.clientHeight ?? 0,
|
||||
};
|
||||
const scrollX = vpSrc.pageX ?? 0;
|
||||
const scrollY = vpSrc.pageY ?? 0;
|
||||
|
||||
await cdp.send(tabId, "DOMSnapshot.enable", {});
|
||||
throwIfAborted(options.signal);
|
||||
const snap = await cdp.send<SnapshotReply>(tabId, "DOMSnapshot.captureSnapshot", {
|
||||
computedStyles: REQUESTED_STYLES,
|
||||
includePaintOrder: true,
|
||||
includeDOMRects: true,
|
||||
});
|
||||
throwIfAborted(options.signal);
|
||||
|
||||
const strings = snap.strings ?? [];
|
||||
const documents = snap.documents ?? [];
|
||||
const doc0 = documents[0];
|
||||
if (!doc0?.nodes?.backendNodeId) {
|
||||
return {
|
||||
nodes: [],
|
||||
viewport,
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
};
|
||||
}
|
||||
|
||||
const topContext: FrameContext = {
|
||||
frameId: snapshotFrameId(doc0, strings),
|
||||
ownerFrameBackendNodeId: null,
|
||||
projection: {
|
||||
sourceClips: [],
|
||||
edges: [],
|
||||
topViewport: viewport,
|
||||
},
|
||||
scrollX,
|
||||
scrollY,
|
||||
};
|
||||
const mainParsed = parseDocumentNodes(doc0, strings, dpr, topContext);
|
||||
const nodes = mainParsed.nodes;
|
||||
const excludedBackendNodeIds = new Set(mainParsed.excludedBackendNodeIds);
|
||||
|
||||
const frameParsed = parseChildFrameDocuments(documents, strings, dpr, 0, nodes, topContext);
|
||||
const iframeNodes = frameParsed.iframeNodes;
|
||||
for (const id of frameParsed.excludedBackendNodeIds) {
|
||||
excludedBackendNodeIds.add(id);
|
||||
}
|
||||
|
||||
await enrichFormControlStates(cdp, tabId, [nodes, ...iframeNodes.values()], options.signal);
|
||||
throwIfAborted(options.signal);
|
||||
|
||||
const frameNodes = new Map<string, CapturedNode[]>();
|
||||
if (topContext.frameId) frameNodes.set(topContext.frameId, nodes);
|
||||
for (const iframeFrameNodes of iframeNodes.values()) {
|
||||
const frameId = iframeFrameNodes.find((node) => node.frameId)?.frameId;
|
||||
if (frameId) frameNodes.set(frameId, iframeFrameNodes);
|
||||
}
|
||||
for (const [frameId, ownerBackendNodeId] of frameParsed.frameOwnerBackendNodeIds) {
|
||||
if (!frameNodes.has(frameId)) {
|
||||
frameNodes.set(frameId, iframeNodes.get(ownerBackendNodeId) ?? []);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
nodes,
|
||||
viewport,
|
||||
iframeNodes,
|
||||
frameNodes,
|
||||
frameOwnerBackendNodeIds: frameParsed.frameOwnerBackendNodeIds,
|
||||
frameParentIds: frameParsed.frameParentIds,
|
||||
...(topContext.frameId ? { rootFrameId: topContext.frameId } : {}),
|
||||
excludedBackendNodeIds,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1,53 +0,0 @@
|
||||
import type { CdpRunner } from "../shared";
|
||||
import { sendToCdpTarget } from "../shared";
|
||||
import { isAbortError, throwIfAborted } from "./capture-abort";
|
||||
import type { DocumentIdentity } from "./facts";
|
||||
|
||||
/** Compare the snapshot root with the current root in that exact frame.
|
||||
* Deep serialization supplies the backend ID without a separate describeNode. */
|
||||
export async function verifyDocumentIdentity(
|
||||
cdp: CdpRunner,
|
||||
identity: DocumentIdentity,
|
||||
signal?: AbortSignal,
|
||||
): Promise<"current" | "changed" | "unavailable"> {
|
||||
const attached = () => cdp.getAttachmentId?.(identity.target.tabId) === identity.attachmentId;
|
||||
throwIfAborted(signal);
|
||||
if (!attached()) return "changed";
|
||||
const objectGroup = `bsk-document-identity-${crypto.randomUUID()}`;
|
||||
const send = <T>(method: string, params: object) => {
|
||||
throwIfAborted(signal);
|
||||
return sendToCdpTarget<T>(cdp, identity.target, method, params);
|
||||
};
|
||||
try {
|
||||
const world = await send<{ executionContextId: number }>("Page.createIsolatedWorld", {
|
||||
frameId: identity.frameId,
|
||||
worldName: "bsk-document-identity",
|
||||
});
|
||||
const reply = await send<{
|
||||
result?: { deepSerializedValue?: { type: string; value?: { backendNodeId?: number } } };
|
||||
}>("Runtime.evaluate", {
|
||||
expression: "document.documentElement",
|
||||
contextId: world.executionContextId,
|
||||
objectGroup,
|
||||
serializationOptions: {
|
||||
serialization: "deep",
|
||||
additionalParameters: { maxNodeDepth: 0, includeShadowTree: "none" },
|
||||
},
|
||||
});
|
||||
if (!attached()) return "changed";
|
||||
const root = reply.result?.deepSerializedValue;
|
||||
if (root?.type === "null") return "changed";
|
||||
const id = root?.type === "node" ? root.value?.backendNodeId : undefined;
|
||||
if (id === undefined) return "unavailable";
|
||||
return id === identity.documentElementBackendNodeId ? "current" : "changed";
|
||||
} catch (error) {
|
||||
throwIfAborted(signal);
|
||||
if (isAbortError(error)) throw error;
|
||||
return attached() ? "unavailable" : "changed";
|
||||
} finally {
|
||||
await sendToCdpTarget(cdp, identity.target, "Runtime.releaseObjectGroup", {
|
||||
objectGroup,
|
||||
}).catch(() => {});
|
||||
throwIfAborted(signal);
|
||||
}
|
||||
}
|
||||
@@ -1,155 +0,0 @@
|
||||
import type { Viewport } from "@browser-skill/vom";
|
||||
import type { CdpFrame, CdpTarget } from "@/browser-driver/frame-graph";
|
||||
import { isOverlayHostNode } from "@/lib/overlay-bridge";
|
||||
import type { FrameProjectionIssue } from "../geometry/coordinate-types";
|
||||
import { createCaptureCheckpoint } from "./capture-abort";
|
||||
import type { CapturedNode } from "./capture-types";
|
||||
import type { FrameOwnedAxNode } from "./frame-document";
|
||||
|
||||
/** Bounds retain their snapshot document CSS units until normalization. */
|
||||
export interface SnapshotLayout {
|
||||
readonly boundsSpace: "snapshot-document-css";
|
||||
bounds?: number[];
|
||||
styles: Readonly<Record<string, string>>;
|
||||
}
|
||||
|
||||
export interface DecodedNode extends Omit<CapturedNode, "rect" | "localRect" | "rendered"> {
|
||||
nodeType?: number;
|
||||
layout?: SnapshotLayout;
|
||||
}
|
||||
|
||||
export interface NodeFacts extends CapturedNode {
|
||||
nodeType?: number;
|
||||
layout?: SnapshotLayout;
|
||||
}
|
||||
|
||||
export interface DocumentIndex<T extends DecodedNode = NodeFacts> {
|
||||
readonly nodes: ReadonlyMap<number, T>;
|
||||
readonly excludedBackendNodeIds: ReadonlySet<number>;
|
||||
}
|
||||
|
||||
export interface DecodedDocument {
|
||||
nodes: DecodedNode[];
|
||||
}
|
||||
|
||||
/** All snapshot nodes participate, including document and shadow roots. This
|
||||
* preserves overlay propagation when the semantic adapter omits those nodes. */
|
||||
export async function buildDocumentIndex<T extends DecodedNode>(
|
||||
input: readonly T[],
|
||||
signal?: AbortSignal,
|
||||
): Promise<DocumentIndex<T>> {
|
||||
const checkpoint = createCaptureCheckpoint(signal);
|
||||
const nodes = new Map<number, T>();
|
||||
const overlayByNode = new Map<number, boolean>();
|
||||
const excludedBackendNodeIds = new Set<number>();
|
||||
for (let i = 0; i < input.length; i++) {
|
||||
if (i % 256 === 0) {
|
||||
const pending = checkpoint();
|
||||
if (pending) await pending;
|
||||
}
|
||||
const node = input[i];
|
||||
nodes.set(node.backendNodeId, node);
|
||||
}
|
||||
let work = 0;
|
||||
for (const node of input) {
|
||||
if (work++ % 256 === 0) {
|
||||
const pending = checkpoint();
|
||||
if (pending) await pending;
|
||||
}
|
||||
if (overlayByNode.has(node.backendNodeId)) continue;
|
||||
const path: DecodedNode[] = [];
|
||||
const visiting = new Set<number>();
|
||||
let current: DecodedNode | undefined = node;
|
||||
let overlay = false;
|
||||
while (current && !overlayByNode.has(current.backendNodeId)) {
|
||||
if (work++ % 256 === 0) {
|
||||
const pending = checkpoint();
|
||||
if (pending) await pending;
|
||||
}
|
||||
if (visiting.has(current.backendNodeId)) {
|
||||
// Only an overlay inside the cycle may mark the cycle and its descendants.
|
||||
overlay = path
|
||||
.slice(path.findIndex((item) => item.backendNodeId === current!.backendNodeId))
|
||||
.some((item) => isOverlayHostNode(item.tag, Object.keys(item.attrs)));
|
||||
break;
|
||||
}
|
||||
visiting.add(current.backendNodeId);
|
||||
path.push(current);
|
||||
if (current.parentBackendNodeId === null) {
|
||||
current = undefined;
|
||||
break;
|
||||
}
|
||||
current = nodes.get(current.parentBackendNodeId);
|
||||
}
|
||||
if (current && overlayByNode.has(current.backendNodeId))
|
||||
overlay = overlayByNode.get(current.backendNodeId)!;
|
||||
for (let i = path.length - 1; i >= 0; i--) {
|
||||
if (work++ % 256 === 0) {
|
||||
const pending = checkpoint();
|
||||
if (pending) await pending;
|
||||
}
|
||||
const item = path[i];
|
||||
overlay = overlay || isOverlayHostNode(item.tag, Object.keys(item.attrs));
|
||||
overlayByNode.set(item.backendNodeId, overlay);
|
||||
if (overlay) excludedBackendNodeIds.add(item.backendNodeId);
|
||||
}
|
||||
}
|
||||
return { nodes, excludedBackendNodeIds };
|
||||
}
|
||||
|
||||
export interface DocumentIdentity {
|
||||
attachmentId: string;
|
||||
target: CdpTarget;
|
||||
frameId: string;
|
||||
documentElementBackendNodeId: number;
|
||||
}
|
||||
|
||||
/** Narrow input shared by the existing semantic scene/hover consumers. */
|
||||
export interface CapturedSceneInput {
|
||||
nodes: CapturedNode[];
|
||||
viewport: Viewport;
|
||||
rootFrameId?: string;
|
||||
excludedBackendNodeIds: ReadonlySet<number>;
|
||||
}
|
||||
|
||||
export interface CaptureIssue {
|
||||
/** Owner failure or inherited blockage; absent for other geometry failures. */
|
||||
projectionIssue?: FrameProjectionIssue;
|
||||
target: CdpTarget;
|
||||
frameId?: string;
|
||||
stage: "dom" | "ax" | "identity" | "ownership" | "geometry" | "forms";
|
||||
reason:
|
||||
| "document-changed"
|
||||
| "identity-unavailable"
|
||||
| "identity-unverified"
|
||||
| "capture-unavailable"
|
||||
| "frame-ownership-unresolved"
|
||||
| "geometry-unavailable";
|
||||
}
|
||||
|
||||
export interface DocumentFacts<T extends FrameOwnedAxNode> {
|
||||
readonly frame: CdpFrame;
|
||||
readonly identity?: DocumentIdentity;
|
||||
readonly index: DocumentIndex;
|
||||
readonly domNodes: CapturedNode[];
|
||||
readonly axNodes: T[];
|
||||
}
|
||||
|
||||
export interface ObservationFacts<T extends FrameOwnedAxNode> {
|
||||
readonly rootFrameId: string;
|
||||
readonly viewport: Viewport;
|
||||
readonly documents: readonly DocumentFacts<T>[];
|
||||
readonly issues: readonly CaptureIssue[];
|
||||
readonly startedAt: number;
|
||||
readonly finishedAt: number;
|
||||
}
|
||||
|
||||
export interface CapturedSurfaceProbe {
|
||||
triggerBackendNodeId: number;
|
||||
triggerPoint?: { x: number; y: number };
|
||||
triggerAction: "hover" | "focus" | string;
|
||||
subItems: string[];
|
||||
confidence?: "high" | "medium" | "low";
|
||||
}
|
||||
|
||||
export type { CapturedNode } from "./capture-types";
|
||||
@@ -1,155 +0,0 @@
|
||||
import type { CdpRunner } from "../shared";
|
||||
import { isAbortError, throwIfAborted } from "./capture-abort";
|
||||
import type { CapturedNode } from "./capture-types";
|
||||
|
||||
interface RuntimeEvaluateReply {
|
||||
result?: { deepSerializedValue?: DeepSerializedValue };
|
||||
}
|
||||
const MAX_FORM_ENRICH_CONTROLS = 250;
|
||||
|
||||
interface CapturedFormState {
|
||||
value?: string;
|
||||
defaultValue?: string;
|
||||
placeholder?: string;
|
||||
state?: "empty" | "filled" | "default";
|
||||
sensitive?: boolean;
|
||||
}
|
||||
|
||||
interface DeepSerializedValue {
|
||||
type: string;
|
||||
value?: unknown;
|
||||
}
|
||||
|
||||
function formStateBatchExpression(maxControls: number): string {
|
||||
return `(() => {
|
||||
const maxControls = ${JSON.stringify(maxControls)};
|
||||
let remaining = maxControls;
|
||||
const controlSelector = "input,textarea,select";
|
||||
const controlState = (el) => {
|
||||
const tag = el.tagName.toLowerCase();
|
||||
const type = tag === "input" ? String(el.type || "text").toLowerCase() : tag;
|
||||
const sensitive = type === "password" || type === "credit-card";
|
||||
const rawValue = typeof el.value === "string" ? el.value : "";
|
||||
const defaultValue = typeof el.defaultValue === "string" ? el.defaultValue : "";
|
||||
const placeholder = typeof el.placeholder === "string" ? el.placeholder : "";
|
||||
const state = rawValue === "" ? "empty" : rawValue === defaultValue ? "default" : "filled";
|
||||
return {
|
||||
state,
|
||||
sensitive,
|
||||
placeholder,
|
||||
...(sensitive ? {} : { value: rawValue, defaultValue }),
|
||||
};
|
||||
};
|
||||
const controls = [];
|
||||
const collect = (doc) => {
|
||||
for (const el of Array.from(doc.querySelectorAll(controlSelector))) {
|
||||
if (remaining <= 0) break;
|
||||
// Deep serialization supplies the node's backend id. Keep the
|
||||
// state as JSON so decoding needs no general-purpose V8 deserializer.
|
||||
controls.push([el, JSON.stringify(controlState(el))]);
|
||||
remaining -= 1;
|
||||
}
|
||||
for (const frame of Array.from(doc.querySelectorAll("iframe"))) {
|
||||
if (remaining <= 0) break;
|
||||
let childDoc = null;
|
||||
try { childDoc = frame.contentDocument; } catch { childDoc = null; }
|
||||
if (childDoc) collect(childDoc);
|
||||
}
|
||||
};
|
||||
collect(document);
|
||||
return controls;
|
||||
})()`;
|
||||
}
|
||||
|
||||
function formStatesByBackendId(result: RuntimeEvaluateReply): Map<number, CapturedFormState> {
|
||||
const states = new Map<number, CapturedFormState>();
|
||||
const serialized = result.result?.deepSerializedValue;
|
||||
if (serialized?.type !== "array" || !Array.isArray(serialized.value)) return states;
|
||||
for (const entry of serialized.value as DeepSerializedValue[]) {
|
||||
if (entry?.type !== "array" || !Array.isArray(entry.value)) continue;
|
||||
const [element, json] = entry.value as DeepSerializedValue[];
|
||||
if (element?.type !== "node" || json?.type !== "string" || typeof json.value !== "string") {
|
||||
continue;
|
||||
}
|
||||
const backendNodeId = (element.value as { backendNodeId?: number } | undefined)?.backendNodeId;
|
||||
if (typeof backendNodeId !== "number" || !Number.isSafeInteger(backendNodeId)) continue;
|
||||
try {
|
||||
const state = JSON.parse(json.value) as CapturedFormState | null;
|
||||
if (
|
||||
!state ||
|
||||
!["empty", "filled", "default"].includes(state.state ?? "") ||
|
||||
typeof state.sensitive !== "boolean" ||
|
||||
typeof state.placeholder !== "string" ||
|
||||
(!state.sensitive &&
|
||||
(typeof state.value !== "string" || typeof state.defaultValue !== "string"))
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
states.set(backendNodeId, state);
|
||||
} catch {
|
||||
// A malformed entry must not overwrite the snapshot's own state.
|
||||
}
|
||||
}
|
||||
return states;
|
||||
}
|
||||
|
||||
function applyFormStates(nodes: CapturedNode[], states: Map<number, CapturedFormState>): void {
|
||||
for (const node of nodes) {
|
||||
if (!["input", "textarea", "select"].includes(node.tag)) continue;
|
||||
const state = states.get(node.backendNodeId);
|
||||
if (!state) continue;
|
||||
node.formState = state.state;
|
||||
node.formPlaceholder = state.placeholder ?? "";
|
||||
if (
|
||||
state.sensitive ||
|
||||
(node.tag === "input" && (node.attrs.type ?? "").toLowerCase() === "password")
|
||||
) {
|
||||
delete node.formValue;
|
||||
delete node.formDefaultValue;
|
||||
delete node.attrs.value;
|
||||
} else {
|
||||
node.formDefaultValue = state.defaultValue ?? "";
|
||||
if (state.value !== undefined) node.formValue = state.value;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function enrichFormControlStates(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
frameNodeGroups: CapturedNode[][],
|
||||
signal?: AbortSignal,
|
||||
): Promise<boolean> {
|
||||
const hasControls = frameNodeGroups.some((nodes) =>
|
||||
nodes.some((node) => ["input", "textarea", "select"].includes(node.tag)),
|
||||
);
|
||||
if (!hasControls) return true;
|
||||
const objectGroup = `bsk-vom-forms-${crypto.randomUUID()}`;
|
||||
try {
|
||||
throwIfAborted(signal);
|
||||
const result = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
|
||||
expression: formStateBatchExpression(MAX_FORM_ENRICH_CONTROLS),
|
||||
objectGroup,
|
||||
serializationOptions: {
|
||||
serialization: "deep",
|
||||
maxDepth: 3,
|
||||
additionalParameters: { maxNodeDepth: 0, includeShadowTree: "none" },
|
||||
},
|
||||
});
|
||||
throwIfAborted(signal);
|
||||
// Backend ids are scoped to this capture's CDP target. OOPIF targets
|
||||
// have separate enrichment calls and never share this lookup.
|
||||
const states = formStatesByBackendId(result);
|
||||
for (const nodes of frameNodeGroups) {
|
||||
applyFormStates(nodes, states);
|
||||
}
|
||||
return true;
|
||||
} catch (error) {
|
||||
if (isAbortError(error)) throw error;
|
||||
// Best-effort enrichment. DOMSnapshot/AX data still carries the nodes.
|
||||
return false;
|
||||
} finally {
|
||||
await cdp.send(tabId, "Runtime.releaseObjectGroup", { objectGroup }).catch(() => undefined);
|
||||
throwIfAborted(signal);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,181 @@
|
||||
import type { CdpFrame, CdpFrameGraph } from "@/browser-driver/frame-graph";
|
||||
import { resolveFrameProjection } from "../frame-geometry";
|
||||
import { type GeometryProjection, projectRectToViewport } from "../geometry";
|
||||
import { type CdpRunner, cdpRunnerForTarget, sendToCdpTarget } from "../shared";
|
||||
import { type CapturedNode, type CapturedViewModel, captureViewModel } from "./capture";
|
||||
import {
|
||||
buildFrameDocuments,
|
||||
type FrameAxBatch,
|
||||
type FrameDocument,
|
||||
type FrameOwnedAxNode,
|
||||
} from "./frame-document";
|
||||
|
||||
export type FrameAxNode = FrameOwnedAxNode;
|
||||
export type CapturedFrameDocument<T extends FrameAxNode> = FrameDocument<T>;
|
||||
|
||||
function throwIfAborted(signal?: AbortSignal): void {
|
||||
if (signal?.aborted) throw new DOMException("observation aborted", "AbortError");
|
||||
}
|
||||
|
||||
function isAbortError(err: unknown): boolean {
|
||||
return err instanceof Error && err.name === "AbortError";
|
||||
}
|
||||
|
||||
async function discoverFrameGraph(cdp: CdpRunner, tabId: number): Promise<CdpFrameGraph | null> {
|
||||
if (!cdp.getFrameGraph) return null;
|
||||
try {
|
||||
return await cdp.getFrameGraph(tabId);
|
||||
} catch (err) {
|
||||
console.debug("[bsk observation] frame graph capture failed", err);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function transformFrameNodes(
|
||||
nodes: CapturedNode[],
|
||||
projection: GeometryProjection | null,
|
||||
): CapturedNode[] {
|
||||
return nodes.map((node) => ({
|
||||
...node,
|
||||
rect: projection
|
||||
? (() => {
|
||||
const bounds = projectRectToViewport(node.rect, projection);
|
||||
return bounds ? { x: bounds.x, y: bounds.y, w: bounds.width, h: bounds.height } : null;
|
||||
})()
|
||||
: null,
|
||||
}));
|
||||
}
|
||||
|
||||
async function captureMissingFrameDocuments(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
graph: CdpFrameGraph,
|
||||
captured: CapturedViewModel,
|
||||
signal?: AbortSignal,
|
||||
): Promise<void> {
|
||||
if (!captured.frameNodes) captured.frameNodes = new Map();
|
||||
if (!captured.frameOwnerBackendNodeIds) captured.frameOwnerBackendNodeIds = new Map();
|
||||
if (!captured.frameParentIds) captured.frameParentIds = new Map();
|
||||
|
||||
for (const frame of graph.frames) {
|
||||
if (!frame.target.sessionId || captured.frameNodes.has(frame.frameId)) continue;
|
||||
try {
|
||||
const child = await captureViewModel(cdpRunnerForTarget(cdp, frame.target), tabId, {
|
||||
signal,
|
||||
});
|
||||
let projection: GeometryProjection | null = null;
|
||||
try {
|
||||
projection = await resolveFrameProjection(cdp, graph, frame.frameId);
|
||||
} catch (err) {
|
||||
if (isAbortError(err)) throw err;
|
||||
console.debug("[bsk observation] child frame geometry projection failed", {
|
||||
frameId: frame.frameId,
|
||||
err,
|
||||
});
|
||||
}
|
||||
const rootFrameId = child.rootFrameId ?? frame.frameId;
|
||||
const childFrames = child.frameNodes ?? new Map<string, CapturedNode[]>();
|
||||
if (!childFrames.has(rootFrameId)) childFrames.set(rootFrameId, child.nodes);
|
||||
if (rootFrameId !== frame.frameId && !childFrames.has(frame.frameId)) {
|
||||
childFrames.set(
|
||||
frame.frameId,
|
||||
child.nodes.map((node) => ({ ...node, frameId: frame.frameId })),
|
||||
);
|
||||
}
|
||||
for (const [childFrameId, nodes] of childFrames) {
|
||||
captured.frameNodes.set(childFrameId, transformFrameNodes(nodes, projection));
|
||||
}
|
||||
if (frame.ownerBackendNodeId !== undefined) {
|
||||
captured.iframeNodes.set(
|
||||
frame.ownerBackendNodeId,
|
||||
captured.frameNodes.get(frame.frameId) ?? [],
|
||||
);
|
||||
captured.frameOwnerBackendNodeIds.set(frame.frameId, frame.ownerBackendNodeId);
|
||||
}
|
||||
if (frame.parentFrameId) captured.frameParentIds.set(frame.frameId, frame.parentFrameId);
|
||||
for (const [childFrameId, ownerBackendNodeId] of child.frameOwnerBackendNodeIds ?? []) {
|
||||
captured.frameOwnerBackendNodeIds.set(childFrameId, ownerBackendNodeId);
|
||||
}
|
||||
for (const [childFrameId, parentFrameId] of child.frameParentIds ?? []) {
|
||||
captured.frameParentIds.set(childFrameId, parentFrameId);
|
||||
}
|
||||
} catch (err) {
|
||||
if (isAbortError(err)) throw err;
|
||||
console.debug("[bsk observation] child frame DOM capture failed", {
|
||||
frameId: frame.frameId,
|
||||
err,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function captureAxTrees<T extends FrameAxNode>(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
discoveredFrames: CdpFrame[],
|
||||
captured: CapturedViewModel,
|
||||
signal?: AbortSignal,
|
||||
): Promise<FrameAxBatch<T>[]> {
|
||||
let frames = [...discoveredFrames];
|
||||
const capturedFrameIds = [...(captured.frameNodes?.keys() ?? [])];
|
||||
if (frames.length === 0) {
|
||||
const rootFrameId = captured.rootFrameId ?? capturedFrameIds[0] ?? "root";
|
||||
frames = [{ frameId: rootFrameId, target: { tabId } }];
|
||||
}
|
||||
for (const frameId of capturedFrameIds) {
|
||||
if (!frames.some((frame) => frame.frameId === frameId)) {
|
||||
frames.push({ frameId, target: { tabId } });
|
||||
}
|
||||
}
|
||||
|
||||
let firstFailure: unknown;
|
||||
const trees = await Promise.all(
|
||||
frames.map(async (frame): Promise<FrameAxBatch<T>> => {
|
||||
try {
|
||||
throwIfAborted(signal);
|
||||
await sendToCdpTarget(cdp, frame.target, "Accessibility.enable", {});
|
||||
const result = await sendToCdpTarget<{ nodes?: T[] }>(
|
||||
cdp,
|
||||
frame.target,
|
||||
"Accessibility.getFullAXTree",
|
||||
frame.frameId === "root" ? {} : { frameId: frame.frameId },
|
||||
);
|
||||
throwIfAborted(signal);
|
||||
return {
|
||||
frame,
|
||||
nodes: result.nodes ?? [],
|
||||
};
|
||||
} catch (err) {
|
||||
if (isAbortError(err)) throw err;
|
||||
firstFailure ??= err;
|
||||
console.debug("[bsk observation] frame AX capture failed", {
|
||||
frameId: frame.frameId,
|
||||
err,
|
||||
});
|
||||
return { frame, nodes: [] };
|
||||
}
|
||||
}),
|
||||
);
|
||||
const capturedNodeCount = [...captured.iframeNodes.values()].reduce(
|
||||
(count, nodes) => count + nodes.length,
|
||||
captured.nodes.length,
|
||||
);
|
||||
if (firstFailure && capturedNodeCount === 0 && trees.every((tree) => tree.nodes.length === 0)) {
|
||||
throw firstFailure;
|
||||
}
|
||||
return trees;
|
||||
}
|
||||
|
||||
export async function captureFrameData<T extends FrameAxNode>(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
captured: CapturedViewModel,
|
||||
signal?: AbortSignal,
|
||||
): Promise<CapturedFrameDocument<T>[]> {
|
||||
const graph = await discoverFrameGraph(cdp, tabId);
|
||||
const frames = graph?.frames ?? [];
|
||||
if (graph) await captureMissingFrameDocuments(cdp, tabId, graph, captured, signal);
|
||||
throwIfAborted(signal);
|
||||
const batches = await captureAxTrees<T>(cdp, tabId, frames, captured, signal);
|
||||
return buildFrameDocuments(graph, batches, captured);
|
||||
}
|
||||
@@ -4,8 +4,7 @@ import {
|
||||
type CdpTarget,
|
||||
cdpTargetKey,
|
||||
} from "@/browser-driver/frame-graph";
|
||||
import { createCaptureCheckpoint } from "./capture-abort";
|
||||
import type { CapturedNode } from "./capture-types";
|
||||
import type { CapturedNode, CapturedViewModel } from "./capture";
|
||||
|
||||
export interface FrameOwnedAxNode {
|
||||
nodeId: string;
|
||||
@@ -24,15 +23,6 @@ export interface FrameDocument<T extends FrameOwnedAxNode> extends CdpFrame {
|
||||
contextScopeId: string;
|
||||
axNodes: T[];
|
||||
domNodes: CapturedNode[];
|
||||
excludedBackendNodeIds?: ReadonlySet<number>;
|
||||
}
|
||||
|
||||
/** Explicit snapshot document data used to join DOM and AX ownership. */
|
||||
export interface FrameDomInput {
|
||||
nodes: CapturedNode[];
|
||||
rootFrameId?: string;
|
||||
frameNodes?: ReadonlyMap<string, CapturedNode[]>;
|
||||
frameExcludedBackendNodeIds?: ReadonlyMap<string, ReadonlySet<number>>;
|
||||
}
|
||||
|
||||
interface Ownership {
|
||||
@@ -57,25 +47,35 @@ function targetBackendKey(target: CdpTarget, backendNodeId: number): string {
|
||||
function frameList<T extends FrameOwnedAxNode>(
|
||||
graph: CdpFrameGraph | null,
|
||||
batches: FrameAxBatch<T>[],
|
||||
captured: CapturedViewModel,
|
||||
): CdpFrame[] {
|
||||
const frames = new Map<string, CdpFrame>();
|
||||
for (const frame of graph?.frames ?? []) frames.set(frame.frameId, frame);
|
||||
for (const batch of batches) {
|
||||
if (!frames.has(batch.frame.frameId)) frames.set(batch.frame.frameId, batch.frame);
|
||||
}
|
||||
return [...frames.values()];
|
||||
for (const frameId of captured.frameNodes?.keys() ?? []) {
|
||||
if (!frames.has(frameId))
|
||||
frames.set(frameId, { frameId, target: { tabId: batches[0]?.frame.target.tabId ?? 0 } });
|
||||
}
|
||||
return [...frames.values()].map((frame) => {
|
||||
const ownerBackendNodeId =
|
||||
frame.ownerBackendNodeId ?? captured.frameOwnerBackendNodeIds?.get(frame.frameId);
|
||||
const parentFrameId = frame.parentFrameId ?? captured.frameParentIds?.get(frame.frameId);
|
||||
return {
|
||||
...frame,
|
||||
...(ownerBackendNodeId !== undefined ? { ownerBackendNodeId } : {}),
|
||||
...(parentFrameId ? { parentFrameId } : {}),
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
export async function buildFrameDocuments<T extends FrameOwnedAxNode>(
|
||||
export function buildFrameDocuments<T extends FrameOwnedAxNode>(
|
||||
graph: CdpFrameGraph | null,
|
||||
batches: FrameAxBatch<T>[],
|
||||
captured: FrameDomInput,
|
||||
signal?: AbortSignal,
|
||||
unresolved?: (frame: CdpFrame) => void,
|
||||
): Promise<FrameDocument<T>[]> {
|
||||
const checkpoint = createCaptureCheckpoint(signal);
|
||||
let work = 0;
|
||||
const frames = frameList(graph, batches);
|
||||
captured: CapturedViewModel,
|
||||
): FrameDocument<T>[] {
|
||||
const frames = frameList(graph, batches, captured);
|
||||
const frameById = new Map(frames.map((frame) => [frame.frameId, frame]));
|
||||
const rootFrameId = graph?.rootFrameId ?? captured.rootFrameId ?? frames[0]?.frameId;
|
||||
const domNodesForFrame = (frameId: string): CapturedNode[] =>
|
||||
@@ -83,80 +83,54 @@ export async function buildFrameDocuments<T extends FrameOwnedAxNode>(
|
||||
const backendOwner = new Map<string, string>();
|
||||
for (const frame of frames) {
|
||||
for (const node of domNodesForFrame(frame.frameId)) {
|
||||
if (work++ % 256 === 0) {
|
||||
const pending = checkpoint();
|
||||
if (pending) await pending;
|
||||
}
|
||||
backendOwner.set(targetBackendKey(frame.target, node.backendNodeId), frame.frameId);
|
||||
}
|
||||
}
|
||||
|
||||
const candidates = new Map<string, OwnedCandidate<T>>();
|
||||
for (const batch of batches) {
|
||||
const nodeById = new Map<string, T>();
|
||||
for (const node of batch.nodes) {
|
||||
if (work++ % 256 === 0) {
|
||||
const pending = checkpoint();
|
||||
if (pending) await pending;
|
||||
}
|
||||
nodeById.set(node.nodeId, node);
|
||||
}
|
||||
let reported = false;
|
||||
const nodeById = new Map(batch.nodes.map((node) => [node.nodeId, node]));
|
||||
const ownershipByNodeId = new Map<string, Ownership>();
|
||||
for (const node of batch.nodes) {
|
||||
const path: T[] = [];
|
||||
const visiting = new Set<string>();
|
||||
let current: T | undefined = node;
|
||||
let ownership: Ownership = { frameId: batch.frame.frameId, strength: 1 };
|
||||
while (current) {
|
||||
if (work++ % 256 === 0) {
|
||||
const pending = checkpoint();
|
||||
if (pending) await pending;
|
||||
}
|
||||
const cached = ownershipByNodeId.get(current.nodeId);
|
||||
if (cached) {
|
||||
ownership = cached;
|
||||
break;
|
||||
}
|
||||
if (visiting.has(current.nodeId)) break;
|
||||
visiting.add(current.nodeId);
|
||||
if (current.frameId) {
|
||||
const frame = frameById.get(current.frameId);
|
||||
// An explicit contradiction must not fall through to a guessed owner.
|
||||
if (!frame || cdpTargetKey(frame.target) !== cdpTargetKey(batch.frame.target)) {
|
||||
if (!reported) {
|
||||
unresolved?.(batch.frame);
|
||||
reported = true;
|
||||
}
|
||||
ownership = { frameId: "", strength: 0 };
|
||||
break;
|
||||
}
|
||||
ownership = { frameId: frame.frameId, strength: 4 };
|
||||
ownershipByNodeId.set(current.nodeId, ownership);
|
||||
break;
|
||||
}
|
||||
const frameId =
|
||||
typeof current.backendDOMNodeId === "number"
|
||||
? backendOwner.get(targetBackendKey(batch.frame.target, current.backendDOMNodeId))
|
||||
: undefined;
|
||||
const resolving = new Set<string>();
|
||||
const resolveOwnership = (node: T): Ownership => {
|
||||
const cached = ownershipByNodeId.get(node.nodeId);
|
||||
if (cached) return cached;
|
||||
if (node.frameId && frameById.has(node.frameId)) {
|
||||
const ownership = { frameId: node.frameId, strength: 4 };
|
||||
ownershipByNodeId.set(node.nodeId, ownership);
|
||||
return ownership;
|
||||
}
|
||||
if (typeof node.backendDOMNodeId === "number") {
|
||||
const frameId = backendOwner.get(
|
||||
targetBackendKey(batch.frame.target, node.backendDOMNodeId),
|
||||
);
|
||||
if (frameId) {
|
||||
ownership = { frameId, strength: 3 };
|
||||
ownershipByNodeId.set(current.nodeId, ownership);
|
||||
break;
|
||||
const ownership = { frameId, strength: 3 };
|
||||
ownershipByNodeId.set(node.nodeId, ownership);
|
||||
return ownership;
|
||||
}
|
||||
path.push(current);
|
||||
current = current.parentId ? nodeById.get(current.parentId) : undefined;
|
||||
}
|
||||
for (let i = path.length - 1; i >= 0; i--) {
|
||||
if (work++ % 256 === 0) {
|
||||
const pending = checkpoint();
|
||||
if (pending) await pending;
|
||||
if (node.parentId && !resolving.has(node.nodeId)) {
|
||||
const parent = nodeById.get(node.parentId);
|
||||
if (parent) {
|
||||
resolving.add(node.nodeId);
|
||||
const inherited = resolveOwnership(parent);
|
||||
resolving.delete(node.nodeId);
|
||||
const ownership = {
|
||||
frameId: inherited.frameId,
|
||||
strength: Math.min(2, inherited.strength),
|
||||
};
|
||||
ownershipByNodeId.set(node.nodeId, ownership);
|
||||
return ownership;
|
||||
}
|
||||
ownership = { frameId: ownership.frameId, strength: Math.min(2, ownership.strength) };
|
||||
ownershipByNodeId.set(path[i].nodeId, ownership);
|
||||
}
|
||||
if (!ownershipByNodeId.has(node.nodeId)) ownershipByNodeId.set(node.nodeId, ownership);
|
||||
if (!ownership.frameId) continue;
|
||||
const ownership = { frameId: batch.frame.frameId, strength: 1 };
|
||||
ownershipByNodeId.set(node.nodeId, ownership);
|
||||
return ownership;
|
||||
};
|
||||
|
||||
for (const node of batch.nodes) {
|
||||
const ownership = resolveOwnership(node);
|
||||
const key = targetNodeKey(batch.frame.target, node.nodeId);
|
||||
const existing = candidates.get(key);
|
||||
if (!existing || ownership.strength > existing.ownership.strength) {
|
||||
@@ -170,10 +144,6 @@ export async function buildFrameDocuments<T extends FrameOwnedAxNode>(
|
||||
);
|
||||
const axNodesByFrame = new Map<string, T[]>();
|
||||
for (const candidate of candidates.values()) {
|
||||
if (work++ % 256 === 0) {
|
||||
const pending = checkpoint();
|
||||
if (pending) await pending;
|
||||
}
|
||||
const { node, target, ownership } = candidate;
|
||||
const parentFrameId = node.parentId
|
||||
? ownershipByTargetNode.get(targetNodeKey(target, node.parentId))
|
||||
@@ -199,6 +169,5 @@ export async function buildFrameDocuments<T extends FrameOwnedAxNode>(
|
||||
contextScopeId: frame.frameId,
|
||||
axNodes: axNodesByFrame.get(frame.frameId) ?? [],
|
||||
domNodes: domNodesForFrame(frame.frameId),
|
||||
excludedBackendNodeIds: captured.frameExcludedBackendNodeIds?.get(frame.frameId),
|
||||
}));
|
||||
}
|
||||
|
||||
@@ -1,292 +0,0 @@
|
||||
import type { Viewport } from "@browser-skill/vom";
|
||||
import { type CdpFrame, type CdpTarget, cdpTargetKey } from "@/browser-driver/frame-graph";
|
||||
import { type GeometryProjection, projectRectToViewport } from "../geometry";
|
||||
import {
|
||||
type FrameProjectionState,
|
||||
projectSnapshotRect,
|
||||
snapshotViewportRect,
|
||||
} from "../geometry/coordinate-types";
|
||||
import { cssViewport, GeometryContext } from "../geometry/frame-context";
|
||||
import {
|
||||
createCaptureCheckpoint,
|
||||
isAbortError as isCaptureAbort,
|
||||
throwIfAborted as throwCaptureAborted,
|
||||
} from "./capture-abort";
|
||||
import { buildDocumentIndex, type CaptureIssue, type DocumentIndex, type NodeFacts } from "./facts";
|
||||
import {
|
||||
decodeDocument,
|
||||
type SnapshotDocument,
|
||||
type SnapshotReply,
|
||||
snapshotFrameId,
|
||||
} from "./snapshot";
|
||||
|
||||
export interface FrameContext {
|
||||
frameId?: string;
|
||||
ownerFrameBackendNodeId: number | null;
|
||||
projection: FrameProjectionState | null;
|
||||
targetProjection?: GeometryProjection | null;
|
||||
target: CdpTarget;
|
||||
scrollX: number;
|
||||
scrollY: number;
|
||||
}
|
||||
|
||||
export interface NormalizedDocument {
|
||||
nodes: NodeFacts[];
|
||||
index: DocumentIndex;
|
||||
documentElementBackendNodeId?: number;
|
||||
}
|
||||
|
||||
/** Interpret one document using a supplied projection. No live reads or frame scheduling. */
|
||||
export async function normalizeDocument(
|
||||
doc: SnapshotDocument,
|
||||
strings: string[],
|
||||
context: FrameContext,
|
||||
signal?: AbortSignal,
|
||||
): Promise<NormalizedDocument> {
|
||||
const decoded = await decodeDocument(doc, strings, signal);
|
||||
const checkpoint = createCaptureCheckpoint(signal);
|
||||
const nodes: NodeFacts[] = [];
|
||||
for (let i = 0; i < decoded.nodes.length; i++) {
|
||||
if (i % 256 === 0) {
|
||||
const pending = checkpoint();
|
||||
if (pending) await pending;
|
||||
}
|
||||
const node = decoded.nodes[i];
|
||||
const input = snapshotViewportRect(
|
||||
node.layout?.bounds ?? [],
|
||||
{ target: context.target, frameId: context.frameId },
|
||||
{ x: context.scrollX, y: context.scrollY },
|
||||
);
|
||||
const local = input?.rect;
|
||||
let rect =
|
||||
input && context.projection?.status === "available"
|
||||
? projectSnapshotRect(input, context.projection.projection)
|
||||
: null;
|
||||
if (rect && context.targetProjection !== undefined)
|
||||
rect = context.targetProjection
|
||||
? projectRectToViewport(
|
||||
{ x: rect.x, y: rect.y, w: rect.width, h: rect.height },
|
||||
context.targetProjection,
|
||||
)
|
||||
: null;
|
||||
const visibility = node.layout?.styles.visibility || "visible";
|
||||
const opacity = node.layout?.styles.opacity || "1";
|
||||
nodes.push({
|
||||
...node,
|
||||
...(context.frameId ? { frameId: context.frameId } : {}),
|
||||
ownerFrameBackendNodeId: context.ownerFrameBackendNodeId,
|
||||
localRect: local ? { x: local.x, y: local.y, w: local.width, h: local.height } : null,
|
||||
rect: rect ? { x: rect.x, y: rect.y, w: rect.width, h: rect.height } : null,
|
||||
rendered:
|
||||
!!local &&
|
||||
visibility !== "hidden" &&
|
||||
visibility !== "collapse" &&
|
||||
(Number.parseFloat(opacity) || 0) > 0,
|
||||
});
|
||||
}
|
||||
const index = await buildDocumentIndex(nodes, signal);
|
||||
return {
|
||||
nodes: nodes.filter(
|
||||
(node) => !node.tag.startsWith("#") && !index.excludedBackendNodeIds.has(node.backendNodeId),
|
||||
),
|
||||
index,
|
||||
documentElementBackendNodeId: nodes.find(
|
||||
(node) =>
|
||||
node.nodeType === 1 &&
|
||||
node.parentBackendNodeId !== null &&
|
||||
index.nodes.get(node.parentBackendNodeId)?.nodeType === 9,
|
||||
)?.backendNodeId,
|
||||
};
|
||||
}
|
||||
|
||||
export interface NormalizedFrameDocument extends NormalizedDocument {
|
||||
frame: CdpFrame;
|
||||
}
|
||||
|
||||
/** One target's snapshot. Source ownership is checked before interpreting any
|
||||
* coordinates. Snapshot and live quad coordinates are deliberately not conflated. */
|
||||
export async function normalizeSnapshot(
|
||||
snapshot: SnapshotReply,
|
||||
target: CdpTarget,
|
||||
frames: readonly CdpFrame[],
|
||||
geometry: GeometryContext,
|
||||
issues: CaptureIssue[],
|
||||
signal?: AbortSignal,
|
||||
rootFrameId?: string,
|
||||
): Promise<NormalizedFrameDocument[]> {
|
||||
const strings = snapshot.strings ?? [];
|
||||
const raw = snapshot.documents ?? [];
|
||||
const ownerIds = new Set<number>();
|
||||
for (const document of raw) {
|
||||
for (const index of document.nodes?.contentDocumentIndex?.index ?? []) {
|
||||
const id = document.nodes?.backendNodeId?.[index];
|
||||
if (id !== undefined) ownerIds.add(id);
|
||||
}
|
||||
}
|
||||
geometry.registerSnapshotOwners(target, ownerIds);
|
||||
const frameById = new Map(frames.map((frame) => [frame.frameId, frame]));
|
||||
const sources = new Map<string, SnapshotDocument>();
|
||||
for (const doc of raw) {
|
||||
const id = snapshotFrameId(doc, strings);
|
||||
if (!id || !frameById.has(id) || sources.has(id)) {
|
||||
issues.push({
|
||||
target,
|
||||
frameId: id,
|
||||
stage: "ownership",
|
||||
reason: "frame-ownership-unresolved",
|
||||
});
|
||||
continue;
|
||||
}
|
||||
sources.set(id, doc);
|
||||
}
|
||||
let viewport: Viewport = { width: 0, height: 0 };
|
||||
let scrollX = 0,
|
||||
scrollY = 0;
|
||||
try {
|
||||
const measured = cssViewport(await geometry.layoutMetrics(target));
|
||||
viewport = { width: measured.width, height: measured.height };
|
||||
scrollX = measured.scrollX;
|
||||
scrollY = measured.scrollY;
|
||||
} catch (error) {
|
||||
if (isCaptureAbort(error)) throw error;
|
||||
}
|
||||
|
||||
const children = new Map<string, CdpFrame[]>();
|
||||
const pending: CdpFrame[] = [];
|
||||
for (const frame of frames) {
|
||||
if (!frame.parentFrameId || !frameById.has(frame.parentFrameId)) pending.push(frame);
|
||||
else {
|
||||
const siblings = children.get(frame.parentFrameId);
|
||||
if (siblings) siblings.push(frame);
|
||||
else children.set(frame.parentFrameId, [frame]);
|
||||
}
|
||||
}
|
||||
const projections = new Map<string, FrameProjectionState | null>();
|
||||
const result: NormalizedFrameDocument[] = [];
|
||||
let targetProjection: GeometryProjection | null = null;
|
||||
const targetRoot = rootFrameId ? frameById.get(rootFrameId) : undefined;
|
||||
if (target.sessionId && targetRoot) {
|
||||
try {
|
||||
const known = await geometry.frame(targetRoot.frameId);
|
||||
if (known && cdpTargetKey(known.target) === cdpTargetKey(target))
|
||||
targetProjection = await geometry.targetProjection(targetRoot.frameId);
|
||||
} catch (error) {
|
||||
if (isCaptureAbort(error)) throw error;
|
||||
}
|
||||
}
|
||||
const checkpoint = createCaptureCheckpoint(signal);
|
||||
const visited = new Set<string>();
|
||||
const remaining = frames[Symbol.iterator]();
|
||||
for (let cursor = 0; visited.size < frames.length; cursor++) {
|
||||
if (cursor >= pending.length) {
|
||||
let next = remaining.next();
|
||||
while (!next.done && visited.has(next.value.frameId)) next = remaining.next();
|
||||
if (next.done) break;
|
||||
pending.push(next.value);
|
||||
}
|
||||
if (cursor % 256 === 0) {
|
||||
const pending = checkpoint();
|
||||
if (pending) await pending;
|
||||
}
|
||||
const frame = pending[cursor];
|
||||
if (visited.has(frame.frameId)) continue;
|
||||
visited.add(frame.frameId);
|
||||
for (const child of children.get(frame.frameId) ?? []) pending.push(child);
|
||||
const doc = sources.get(frame.frameId);
|
||||
if (!doc) continue;
|
||||
const source = { target, frameId: frame.frameId };
|
||||
const state: FrameProjectionState | null =
|
||||
frame.frameId === rootFrameId
|
||||
? {
|
||||
status: "available",
|
||||
projection: { source, geometry: { sourceClips: [], edges: [], topViewport: viewport } },
|
||||
}
|
||||
: (projections.get(frame.frameId) ?? null);
|
||||
if (!state && frame.frameId !== rootFrameId)
|
||||
issues.push({
|
||||
target,
|
||||
frameId: frame.frameId,
|
||||
stage: "ownership",
|
||||
reason: "frame-ownership-unresolved",
|
||||
});
|
||||
projections.set(frame.frameId, state);
|
||||
const projection = state?.status === "available" ? state.projection : null;
|
||||
if (!projection || (target.sessionId && !targetProjection))
|
||||
issues.push({
|
||||
target,
|
||||
frameId: frame.frameId,
|
||||
stage: "geometry",
|
||||
reason: "geometry-unavailable",
|
||||
...(state && state.status !== "available" ? { projectionIssue: state } : {}),
|
||||
});
|
||||
|
||||
// Only direct siblings share this pool. Keep normalization in its existing
|
||||
// breadth-first order, independent of owner read completion order.
|
||||
const siblings = (children.get(frame.frameId) ?? []).filter((child) =>
|
||||
sources.has(child.frameId),
|
||||
);
|
||||
const edge = projection?.geometry.edges[0];
|
||||
const clips = edge
|
||||
? [edge.destinationQuad, ...(edge.destinationClips ?? [])]
|
||||
: (projection?.geometry.sourceClips ?? []);
|
||||
let next = 0;
|
||||
let failure: unknown;
|
||||
await Promise.all(
|
||||
Array.from({ length: Math.min(4, siblings.length) }, async () => {
|
||||
while (next < siblings.length && !signal?.aborted && !failure) {
|
||||
const child = siblings[next++];
|
||||
const childSource = { target, frameId: child.frameId };
|
||||
if (!projection) {
|
||||
projections.set(
|
||||
child.frameId,
|
||||
state && state.status !== "available"
|
||||
? {
|
||||
status: "blocked",
|
||||
source: childSource,
|
||||
cause: state.status === "blocked" ? state.cause : state,
|
||||
}
|
||||
: null,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
if (child.ownerBackendNodeId === undefined) {
|
||||
projections.set(child.frameId, null);
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
projections.set(
|
||||
child.frameId,
|
||||
await geometry.snapshotProjection(
|
||||
childSource,
|
||||
child.ownerBackendNodeId,
|
||||
clips,
|
||||
viewport,
|
||||
),
|
||||
);
|
||||
} catch (error) {
|
||||
failure ??= error;
|
||||
}
|
||||
}
|
||||
}),
|
||||
);
|
||||
// Join active reads and their cleanup before returning cancellation.
|
||||
if (failure) throw failure;
|
||||
throwCaptureAborted(signal);
|
||||
const normalized = await normalizeDocument(
|
||||
doc,
|
||||
strings,
|
||||
{
|
||||
frameId: frame.frameId,
|
||||
ownerFrameBackendNodeId: frame.ownerBackendNodeId ?? null,
|
||||
projection: state,
|
||||
target,
|
||||
...(target.sessionId ? { targetProjection } : {}),
|
||||
scrollX: doc.scrollOffsetX ?? (frame.frameId === rootFrameId ? scrollX : 0),
|
||||
scrollY: doc.scrollOffsetY ?? (frame.frameId === rootFrameId ? scrollY : 0),
|
||||
},
|
||||
signal,
|
||||
);
|
||||
result.push({ ...normalized, frame });
|
||||
}
|
||||
return result;
|
||||
}
|
||||
@@ -2,10 +2,7 @@ import type { Rect, RenderedRef, VomResult } from "@browser-skill/vom";
|
||||
import type { CdpTarget } from "@/browser-driver/frame-graph";
|
||||
import { readRecordingDocumentIdentity } from "@/shared/recording-document-identity";
|
||||
import type { CapturedSurfaceProbe } from "./capture";
|
||||
import type {
|
||||
FrameDocument as CapturedFrameDocument,
|
||||
FrameOwnedAxNode as FrameAxNode,
|
||||
} from "./frame-document";
|
||||
import type { CapturedFrameDocument, FrameAxNode } from "./frame-capture";
|
||||
|
||||
/** Frame identity needed to resolve an `@eN` that lives in an iframe. */
|
||||
export interface CaptureVomFrame {
|
||||
|
||||
@@ -61,9 +61,8 @@ export function buildSemanticGraph(input: BuildSemanticGraphInput): SemanticGrap
|
||||
dom,
|
||||
sourceOrder: index,
|
||||
excluded:
|
||||
document.excludedBackendNodeIds?.has(dom.backendNodeId) === true ||
|
||||
(document.frameId === rootFrameId &&
|
||||
input.excludedBackendNodeIds?.has(dom.backendNodeId) === true),
|
||||
document.frameId === rootFrameId &&
|
||||
input.excludedBackendNodeIds?.has(dom.backendNodeId) === true,
|
||||
};
|
||||
nodes.set(id, node);
|
||||
frameNodeIds.push(id);
|
||||
@@ -91,10 +90,9 @@ export function buildSemanticGraph(input: BuildSemanticGraphInput): SemanticGrap
|
||||
ax,
|
||||
sourceOrder: document.domNodes.length + index,
|
||||
excluded:
|
||||
document.frameId === rootFrameId &&
|
||||
typeof ax.backendDOMNodeId === "number" &&
|
||||
(document.excludedBackendNodeIds?.has(ax.backendDOMNodeId) === true ||
|
||||
(document.frameId === rootFrameId &&
|
||||
input.excludedBackendNodeIds?.has(ax.backendDOMNodeId) === true)),
|
||||
input.excludedBackendNodeIds?.has(ax.backendDOMNodeId) === true,
|
||||
});
|
||||
frameNodeIds.push(id);
|
||||
}
|
||||
|
||||
@@ -1,291 +0,0 @@
|
||||
import { type CdpFrame, type CdpTarget, cdpTargetKey } from "@/browser-driver/frame-graph";
|
||||
import { createCaptureCheckpoint } from "./capture-abort";
|
||||
import type { CapturedNode } from "./capture-types";
|
||||
import type { DecodedDocument, DecodedNode } from "./facts";
|
||||
export const REQUESTED_STYLES = [
|
||||
"position",
|
||||
"pointer-events",
|
||||
"cursor",
|
||||
"visibility",
|
||||
"opacity",
|
||||
] as const;
|
||||
const STYLE_COL = Object.fromEntries(
|
||||
REQUESTED_STYLES.map((name, index) => [name, index]),
|
||||
) as Record<(typeof REQUESTED_STYLES)[number], number>;
|
||||
/** Sparse array format Chrome uses for infrequently-set per-node fields. */
|
||||
interface SparseArray {
|
||||
index: number[];
|
||||
value: number[];
|
||||
}
|
||||
|
||||
interface RareBooleanData {
|
||||
index: number[];
|
||||
}
|
||||
|
||||
export interface SnapshotDocument {
|
||||
frameId?: string | number;
|
||||
documentURL?: number;
|
||||
scrollOffsetX?: number;
|
||||
scrollOffsetY?: number;
|
||||
nodes?: {
|
||||
parentIndex?: number[];
|
||||
nodeName?: number[];
|
||||
nodeType?: number[];
|
||||
backendNodeId?: number[];
|
||||
attributes?: number[][];
|
||||
/**
|
||||
* Per-node text value (index into strings), set for `#text` / CDATA nodes.
|
||||
* Element nodes carry -1. Same length as `backendNodeId`.
|
||||
*/
|
||||
nodeValue?: number[];
|
||||
/** Maps node array index → index into `documents[]` for frame content. */
|
||||
contentDocumentIndex?: SparseArray;
|
||||
inputValue?: SparseArray;
|
||||
textValue?: SparseArray;
|
||||
inputChecked?: RareBooleanData;
|
||||
optionSelected?: RareBooleanData;
|
||||
};
|
||||
layout?: {
|
||||
nodeIndex?: number[];
|
||||
styles?: number[][];
|
||||
bounds?: number[][];
|
||||
paintOrders?: number[];
|
||||
};
|
||||
}
|
||||
|
||||
export function snapshotFrameId(document: SnapshotDocument, strings: string[]): string | undefined {
|
||||
if (typeof document.frameId === "string") return document.frameId || undefined;
|
||||
if (typeof document.frameId === "number") return str(strings, document.frameId) || undefined;
|
||||
return undefined;
|
||||
}
|
||||
|
||||
export interface SnapshotReply {
|
||||
strings?: string[];
|
||||
documents?: SnapshotDocument[];
|
||||
}
|
||||
|
||||
function isSensitiveFormControl(node: Pick<CapturedNode, "tag" | "attrs">): boolean {
|
||||
return node.tag === "input" && (node.attrs.type ?? "").toLowerCase() === "password";
|
||||
}
|
||||
|
||||
function snapshotFormState(
|
||||
value: string | undefined,
|
||||
defaultValue: string,
|
||||
hasDefaultValue: boolean,
|
||||
sensitive: boolean,
|
||||
): CapturedNode["formState"] {
|
||||
if (sensitive) {
|
||||
if (value === undefined && !hasDefaultValue) return undefined;
|
||||
return (value ?? defaultValue) === "" ? "empty" : "filled";
|
||||
}
|
||||
if (value === undefined) return undefined;
|
||||
if (value === "") return "empty";
|
||||
return value === defaultValue ? "default" : "filled";
|
||||
}
|
||||
|
||||
function str(strings: string[], idx: number | undefined): string {
|
||||
if (idx === undefined || idx < 0) return "";
|
||||
return strings[idx] ?? "";
|
||||
}
|
||||
|
||||
export function sparseIndexMap(sparse: SparseArray | undefined): Map<number, number> {
|
||||
const out = new Map<number, number>();
|
||||
if (!sparse?.index || !sparse.value) return out;
|
||||
for (let i = 0; i < sparse.index.length; i++) {
|
||||
const nodeIndex = sparse.index[i];
|
||||
const docIndex = sparse.value[i];
|
||||
if (nodeIndex !== undefined && docIndex !== undefined) out.set(nodeIndex, docIndex);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
export async function decodeDocument(
|
||||
doc: SnapshotDocument,
|
||||
strings: string[],
|
||||
signal?: AbortSignal,
|
||||
): Promise<DecodedDocument> {
|
||||
const checkpoint = createCaptureCheckpoint(signal);
|
||||
const dn = doc.nodes;
|
||||
const dl = doc.layout;
|
||||
if (!dn?.backendNodeId) {
|
||||
return { nodes: [] };
|
||||
}
|
||||
|
||||
const count = dn.backendNodeId.length;
|
||||
const layoutByNode = new Map<number, number>();
|
||||
for (let i = 0; i < (dl?.nodeIndex?.length ?? 0); i++) {
|
||||
if (i % 256 === 0) {
|
||||
const pending = checkpoint();
|
||||
if (pending) await pending;
|
||||
}
|
||||
layoutByNode.set(dl!.nodeIndex![i], i);
|
||||
}
|
||||
|
||||
const inputValues = sparseIndexMap(dn.inputValue);
|
||||
const textValues = sparseIndexMap(dn.textValue);
|
||||
const checkedInputs = new Set(dn.inputChecked?.index ?? []);
|
||||
const selectedOptions = new Set(dn.optionSelected?.index ?? []);
|
||||
|
||||
// Collect visible text from #text child nodes so element CapturedNodes
|
||||
// carry a textContent value usable as a button/link label fallback.
|
||||
// nodeValue is a parallel array: string index for text nodes, -1 otherwise.
|
||||
const nodeTextContent = new Map<number, string[]>();
|
||||
if (dn.nodeValue) {
|
||||
for (let n = 0; n < count; n++) {
|
||||
if (n % 256 === 0) {
|
||||
const pending = checkpoint();
|
||||
if (pending) await pending;
|
||||
}
|
||||
const nvIdx = dn.nodeValue[n] ?? -1;
|
||||
if (nvIdx < 0) continue;
|
||||
const text = str(strings, nvIdx).trim();
|
||||
if (!text) continue;
|
||||
const parentIdx = dn.parentIndex?.[n] ?? -1;
|
||||
if (parentIdx >= 0) {
|
||||
const existing = nodeTextContent.get(parentIdx);
|
||||
if (existing) existing.push(text);
|
||||
else nodeTextContent.set(parentIdx, [text]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Decode fields without assigning a coordinate projection or semantic policy.
|
||||
const nodes: DecodedNode[] = [];
|
||||
for (let n = 0; n < count; n++) {
|
||||
if (n % 256 === 0) {
|
||||
const pending = checkpoint();
|
||||
if (pending) await pending;
|
||||
}
|
||||
const backendNodeId = dn.backendNodeId[n];
|
||||
const parentIdx = dn.parentIndex?.[n] ?? -1;
|
||||
const parentBackendNodeId = parentIdx >= 0 ? (dn.backendNodeId[parentIdx] ?? null) : null;
|
||||
const tag = str(strings, dn.nodeName?.[n]).toLowerCase();
|
||||
|
||||
const attrs: Record<string, string> = {};
|
||||
const pairs = dn.attributes?.[n] ?? [];
|
||||
for (let a = 0; a + 1 < pairs.length; a += 2) {
|
||||
attrs[str(strings, pairs[a]).toLowerCase()] = str(strings, pairs[a + 1]);
|
||||
}
|
||||
|
||||
const li = layoutByNode.get(n);
|
||||
const styleRow = li === undefined ? [] : (dl?.styles?.[li] ?? []);
|
||||
const styles: Record<string, string> = {};
|
||||
for (const name of REQUESTED_STYLES) styles[name] = str(strings, styleRow[STYLE_COL[name]]);
|
||||
const layout =
|
||||
li === undefined
|
||||
? undefined
|
||||
: {
|
||||
boundsSpace: "snapshot-document-css" as const,
|
||||
bounds: dl?.bounds?.[li],
|
||||
styles,
|
||||
};
|
||||
|
||||
const textContent = nodeTextContent.get(n)?.join(" ");
|
||||
|
||||
const rawFormValueIndex = tag === "textarea" ? textValues.get(n) : inputValues.get(n);
|
||||
const rawFormValue =
|
||||
rawFormValueIndex !== undefined ? str(strings, rawFormValueIndex) : undefined;
|
||||
const sensitive = isSensitiveFormControl({ tag, attrs });
|
||||
const formDefaultValue = attrs.value ?? "";
|
||||
const formValue = sensitive ? undefined : rawFormValue;
|
||||
const formState = snapshotFormState(
|
||||
rawFormValue,
|
||||
formDefaultValue,
|
||||
Object.prototype.hasOwnProperty.call(attrs, "value"),
|
||||
sensitive,
|
||||
);
|
||||
if (sensitive) delete attrs.value;
|
||||
|
||||
nodes.push({
|
||||
backendNodeId,
|
||||
nodeType: dn.nodeType?.[n],
|
||||
parentBackendNodeId,
|
||||
|
||||
tag,
|
||||
attrs,
|
||||
layout,
|
||||
paintOrder: li === undefined ? 0 : (dl?.paintOrders?.[li] ?? 0),
|
||||
position: styles.position || "static",
|
||||
pointerEvents: styles["pointer-events"] || "auto",
|
||||
cursor: styles.cursor || "auto",
|
||||
|
||||
textContent,
|
||||
...(formValue !== undefined ? { formValue } : {}),
|
||||
...(tag === "input" || tag === "textarea"
|
||||
? {
|
||||
formPlaceholder: attrs.placeholder ?? "",
|
||||
...(!sensitive ? { formDefaultValue } : {}),
|
||||
...(formState ? { formState } : {}),
|
||||
}
|
||||
: {}),
|
||||
...(checkedInputs.has(n) ? { formValue: "true", formState: "filled" } : {}),
|
||||
...(selectedOptions.has(n) ? { formValue: attrs.value ?? textContent ?? "" } : {}),
|
||||
});
|
||||
}
|
||||
return { nodes };
|
||||
}
|
||||
|
||||
/** Snapshot membership and same-target edges are current response evidence.
|
||||
* The earlier graph only supplies compatible metadata and cross-target edges. */
|
||||
export function describeSnapshotFrames(
|
||||
snapshot: SnapshotReply,
|
||||
target: CdpTarget,
|
||||
hintById: ReadonlyMap<string, CdpFrame>,
|
||||
pageUrl?: string,
|
||||
): { frames: CdpFrame[]; ids: Set<string>; rootFrameId?: string } {
|
||||
const raw = snapshot.documents ?? [];
|
||||
const strings = snapshot.strings ?? [];
|
||||
const ids = new Set<string>();
|
||||
const invalid = new Set<string>();
|
||||
const frames = raw.map((doc) => {
|
||||
const frameId = snapshotFrameId(doc, strings);
|
||||
if (!frameId) return undefined;
|
||||
if (ids.has(frameId)) invalid.add(frameId);
|
||||
ids.add(frameId);
|
||||
const hint = hintById.get(frameId);
|
||||
const compatible =
|
||||
hint && cdpTargetKey(hint.target) === cdpTargetKey(target) ? hint : undefined;
|
||||
const url =
|
||||
(doc.documentURL === undefined ? undefined : strings[doc.documentURL]) || compatible?.url;
|
||||
return { frameId, target, ...(url ? { url } : {}) } as CdpFrame;
|
||||
});
|
||||
const root = frames[0];
|
||||
if (root) {
|
||||
const hint = hintById.get(root.frameId);
|
||||
const parent = hint?.parentFrameId ? hintById.get(hint.parentFrameId) : undefined;
|
||||
if (
|
||||
target.sessionId &&
|
||||
hint &&
|
||||
cdpTargetKey(hint.target) === cdpTargetKey(target) &&
|
||||
parent &&
|
||||
cdpTargetKey(parent.target) !== cdpTargetKey(target)
|
||||
) {
|
||||
root.parentFrameId = parent.frameId;
|
||||
root.ownerBackendNodeId = hint.ownerBackendNodeId;
|
||||
}
|
||||
if (!target.sessionId && !root.url && pageUrl) root.url = pageUrl;
|
||||
}
|
||||
const owned = new Set<string>();
|
||||
for (let i = 0; i < raw.length; i++) {
|
||||
const edges = raw[i].nodes?.contentDocumentIndex;
|
||||
const parent = frames[i];
|
||||
if (!parent || !edges) continue;
|
||||
for (let e = 0; e < edges.index.length; e++) {
|
||||
const child = frames[edges.value[e]];
|
||||
const owner = raw[i].nodes?.backendNodeId?.[edges.index[e]];
|
||||
if (!child || owner === undefined) continue;
|
||||
if (child === root || child === parent || owned.has(child.frameId)) {
|
||||
invalid.add(child.frameId);
|
||||
continue;
|
||||
}
|
||||
owned.add(child.frameId);
|
||||
child.parentFrameId = parent.frameId;
|
||||
child.ownerBackendNodeId = owner;
|
||||
}
|
||||
}
|
||||
return {
|
||||
frames: frames.filter((frame): frame is CdpFrame => !!frame && !invalid.has(frame.frameId)),
|
||||
ids,
|
||||
rootFrameId: root?.frameId,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
DEFAULT_DAEMON_PORT,
|
||||
normalizeDaemonPort,
|
||||
parseDaemonPortInput,
|
||||
resolveDaemonWsUrl,
|
||||
} from "../daemon-endpoint";
|
||||
|
||||
describe("daemon-endpoint", () => {
|
||||
it("resolveDaemonWsUrl(DEFAULT) matches the build-time constant", () => {
|
||||
expect(resolveDaemonWsUrl(DEFAULT_DAEMON_PORT)).toBe(__BSK_DAEMON_WS_URL__);
|
||||
});
|
||||
|
||||
it("resolveDaemonWsUrl replaces only the port", () => {
|
||||
expect(resolveDaemonWsUrl(53200)).toBe("ws://127.0.0.1:53200");
|
||||
});
|
||||
|
||||
it("normalizeDaemonPort falls back to default for invalid values", () => {
|
||||
expect(normalizeDaemonPort(undefined)).toBe(DEFAULT_DAEMON_PORT);
|
||||
expect(normalizeDaemonPort(0)).toBe(DEFAULT_DAEMON_PORT);
|
||||
expect(normalizeDaemonPort(65536)).toBe(DEFAULT_DAEMON_PORT);
|
||||
expect(normalizeDaemonPort("abc")).toBe(DEFAULT_DAEMON_PORT);
|
||||
expect(normalizeDaemonPort(1.5)).toBe(DEFAULT_DAEMON_PORT);
|
||||
});
|
||||
|
||||
it("normalizeDaemonPort accepts valid numbers and numeric strings", () => {
|
||||
expect(normalizeDaemonPort(53200)).toBe(53200);
|
||||
expect(normalizeDaemonPort("53200")).toBe(53200);
|
||||
});
|
||||
|
||||
it("parseDaemonPortInput treats empty as default and rejects invalid", () => {
|
||||
expect(parseDaemonPortInput("")).toBe(DEFAULT_DAEMON_PORT);
|
||||
expect(parseDaemonPortInput(" ")).toBe(DEFAULT_DAEMON_PORT);
|
||||
expect(parseDaemonPortInput("abc")).toBeNull();
|
||||
expect(parseDaemonPortInput("0")).toBeNull();
|
||||
expect(parseDaemonPortInput("65536")).toBeNull();
|
||||
});
|
||||
|
||||
it("parseDaemonPortInput accepts valid ports", () => {
|
||||
expect(parseDaemonPortInput("53200")).toBe(53200);
|
||||
expect(parseDaemonPortInput(" 52800 ")).toBe(52800);
|
||||
});
|
||||
});
|
||||
@@ -270,4 +270,19 @@ describe("WSTransport", () => {
|
||||
lastSocket().receive({ id: "2", result: 2 });
|
||||
expect(handler).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("setUrl returns false for the same URL and uses the new URL on the next connect", async () => {
|
||||
const t = new WSTransport({
|
||||
url: "ws://127.0.0.1:52800",
|
||||
webSocketFactory: (url) => new FakeSocket(url) as unknown as WebSocket,
|
||||
});
|
||||
expect(t.setUrl("ws://127.0.0.1:52800")).toBe(false);
|
||||
expect(t.setUrl("ws://127.0.0.1:53200")).toBe(true);
|
||||
|
||||
const p = t.connect();
|
||||
expect(lastSocket().url).toBe("ws://127.0.0.1:53200");
|
||||
lastSocket().open();
|
||||
await p;
|
||||
expect(t.state).toBe("connected");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
/** Build-time default WebSocket URL (see wxt.config.ts). */
|
||||
const base = new URL(__BSK_DAEMON_WS_URL__);
|
||||
|
||||
export const DEFAULT_DAEMON_PORT = Number(base.port) || 52800;
|
||||
|
||||
const MIN_PORT = 1;
|
||||
const MAX_PORT = 65535;
|
||||
|
||||
function isValidPort(n: number): boolean {
|
||||
return Number.isInteger(n) && n >= MIN_PORT && n <= MAX_PORT;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compose the daemon WebSocket URL for a loopback port, preserving the
|
||||
* build-time host and path from {@link __BSK_DAEMON_WS_URL__}.
|
||||
*/
|
||||
export function resolveDaemonWsUrl(port: number): string {
|
||||
const path = base.pathname === "/" ? "" : base.pathname;
|
||||
return `${base.protocol}//${base.hostname}:${port}${path}`;
|
||||
}
|
||||
|
||||
/** Non-integer / out-of-range / unset → {@link DEFAULT_DAEMON_PORT}. */
|
||||
export function normalizeDaemonPort(raw: unknown): number {
|
||||
if (typeof raw === "number" && isValidPort(raw)) return raw;
|
||||
if (typeof raw === "string" && raw.trim() !== "") {
|
||||
const parsed = Number.parseInt(raw.trim(), 10);
|
||||
if (isValidPort(parsed)) return parsed;
|
||||
}
|
||||
return DEFAULT_DAEMON_PORT;
|
||||
}
|
||||
|
||||
/** Input semantics: empty string → default port; invalid → null. */
|
||||
export function parseDaemonPortInput(text: string): number | null {
|
||||
const trimmed = text.trim();
|
||||
if (trimmed === "") return DEFAULT_DAEMON_PORT;
|
||||
if (!/^\d+$/.test(trimmed)) return null;
|
||||
const parsed = Number.parseInt(trimmed, 10);
|
||||
if (!isValidPort(parsed)) return null;
|
||||
return parsed;
|
||||
}
|
||||
@@ -26,25 +26,16 @@ export type RpcErrorReason =
|
||||
| "ref_not_found"
|
||||
| "selector_not_found"
|
||||
| "target_not_fillable"
|
||||
| "fill_value_invalid"
|
||||
| "fill_target_changed"
|
||||
| "fill_focus_lost"
|
||||
| "fill_value_mismatch"
|
||||
| "fill_failed"
|
||||
| "target_not_select"
|
||||
| "option_not_found"
|
||||
| "single_select_value_count"
|
||||
| "tab_not_active"
|
||||
| "restricted_tab_url"
|
||||
| "cdp_extension_access_denied"
|
||||
| "borrow_conflict"
|
||||
| "screenshot_capture_failed"
|
||||
| "file_input_probe_failed"
|
||||
| "file_input_not_activated"
|
||||
| "set_file_input_failed"
|
||||
| "upload_mechanism_unsupported"
|
||||
| "file_drop_target_unavailable"
|
||||
| "file_drop_failed"
|
||||
| "download_capture_failed"
|
||||
| "transfer_outcome_unknown"
|
||||
| "transfer_timeout"
|
||||
@@ -532,15 +523,12 @@ export interface UploadFile {
|
||||
staged_path?: string;
|
||||
}
|
||||
|
||||
export type UploadMode = "input" | "drop";
|
||||
|
||||
export interface UploadParams {
|
||||
session_id: string;
|
||||
ref?: string;
|
||||
selector?: string;
|
||||
tab_id?: number;
|
||||
files: UploadFile[];
|
||||
mode?: UploadMode;
|
||||
timeout_ms?: number;
|
||||
}
|
||||
|
||||
|
||||
@@ -40,7 +40,7 @@ interface MessageLikeEvent {
|
||||
* (1s, 2s, 4s, …, capped at 5s) until `disconnect()` is called.
|
||||
*/
|
||||
export class WSTransport implements Transport {
|
||||
private readonly url: string;
|
||||
private url: string;
|
||||
private readonly factory: WebSocketFactory;
|
||||
private readonly initialDelayMs: number;
|
||||
private readonly maxDelayMs: number;
|
||||
@@ -69,6 +69,13 @@ export class WSTransport implements Transport {
|
||||
return this.currentState;
|
||||
}
|
||||
|
||||
/** Returns whether the URL changed. Does not reconnect. */
|
||||
setUrl(url: string): boolean {
|
||||
if (url === this.url) return false;
|
||||
this.url = url;
|
||||
return true;
|
||||
}
|
||||
|
||||
connect(): Promise<void> {
|
||||
if (this.currentState === "connected") return Promise.resolve();
|
||||
|
||||
|
||||
+146
-37
@@ -116,15 +116,156 @@ navigate navigate-back navigate-forward reload wait-for-navigation wai
|
||||
observe snapshot get-html screenshot console network
|
||||
click hover fill select press evaluate
|
||||
tab list|create|close|select|borrow|return window resize emulate
|
||||
upload download request-help record start|stop
|
||||
request-help record start|stop
|
||||
```
|
||||
|
||||
Required flags that are easy to get wrong:
|
||||
|
||||
```text
|
||||
bsk fill <ref> --value <text> bsk select <ref> --value <option-value>
|
||||
bsk screenshot --out <path> bsk emulate --device <preset-id>
|
||||
bsk upload <ref> --file <path> bsk download <ref> --out <path>
|
||||
### Tabs (require `--session <id>`)
|
||||
|
||||
| Command | Summary |
|
||||
|---------|---------|
|
||||
| `bsk tab list` | List tabs (`--scope user\|agent\|all`, default `all`) |
|
||||
| `bsk tab create` | New tab in Agent Window (`--url`, `--no-active`, `--index`) |
|
||||
| `bsk tab close <tab-id>` | Close an agent tab |
|
||||
| `bsk tab select <tab-id>` | Focus an agent tab |
|
||||
| `bsk tab borrow <tab-id>` | Move a user tab into the Agent Window |
|
||||
| `bsk tab return <tab-id>` | Return a borrowed tab to its original window |
|
||||
|
||||
### Observation (require `--session` unless noted)
|
||||
|
||||
| Command | Summary |
|
||||
|---------|---------|
|
||||
| `bsk snapshot` | First-choice static page understanding: accessibility tree with `@eN` element refs |
|
||||
| `bsk observe` | Semantic VOM observation with bounded perception probes for conditional surfaces |
|
||||
| `bsk get-html` | Raw HTML dump after snapshot is insufficient (high token cost) |
|
||||
| `bsk screenshot` | PNG capture after snapshot is insufficient: full visible tab, or `--ref @eN` to crop to one element (`--out` path optional) |
|
||||
|
||||
### Console & network debugging (read-only; require `--session`)
|
||||
|
||||
| Command | Summary |
|
||||
|---------|---------|
|
||||
| `bsk console` | Buffered page console messages, JS exceptions, and browser log entries (`--include-stack` for stack traces) |
|
||||
| `bsk network` | Buffered network responses (status, method, URL, MIME/resource type) and failures (`net::ERR_*` reason) |
|
||||
|
||||
Both capture from the moment the tab is attached and read a bounded per-tab buffer: `--since <seq>` pages from a cursor (`next_since` in the result), `--limit` (default 50, max 200), `--max-text-chars` (default 1000, max 4096), `--tab-id` to target a non-active tab. Both are strictly read-only — they never intercept or modify traffic, and request/response headers, bodies, and timings are not captured.
|
||||
|
||||
### Navigation
|
||||
|
||||
| Command | Summary |
|
||||
|---------|---------|
|
||||
| `bsk navigate <url>` | Go to URL in agent tab (`--wait-until`, `--timeout`) |
|
||||
| `bsk navigate-back` | History back one step |
|
||||
| `bsk navigate-forward` | History forward one step |
|
||||
| `bsk reload` | Reload current tab (`--hard` bypass cache) |
|
||||
|
||||
(`bsk navigate back` / `bsk navigate forward` are equivalent subcommands.)
|
||||
|
||||
### Interaction
|
||||
|
||||
| Command | Summary |
|
||||
|---------|---------|
|
||||
| `bsk click <ref-or-selector>` | Click element (`--button`, `--click-count`, `--modifiers`) |
|
||||
| `bsk hover <ref-or-selector>` | Move the mouse to an element and wait for hover UI to settle (`--settle`, `--modifiers`) |
|
||||
| `bsk fill <ref-or-selector> --value <text>` | Clear and type into input |
|
||||
| `bsk select <ref-or-selector> --value <v>` | Set `<select>` option(s) by `value` (repeat `--value` for multi-select) |
|
||||
| `bsk press <key>` | Key/combo (`Enter`, `Ctrl+A`, …; optional `--ref` to focus first) |
|
||||
|
||||
### File transfer (require `--session`)
|
||||
|
||||
| Command | Summary |
|
||||
|---------|---------|
|
||||
| `bsk upload <ref-or-selector> --file <path>` | Click one upload trigger and attach an agent-readable local file (`--file` is repeatable) |
|
||||
| `bsk download <ref-or-selector> --out <path>` | Click one download trigger and copy the single completed file to an exact local path (`--overwrite` is opt-in) |
|
||||
|
||||
The agent/harness decides whether a file transfer is appropriate and which local path belongs to the task. Treat upload as disclosure of that file to the current website, and download as accepting website-controlled bytes onto the local filesystem. Use only paths that are necessary for the user's bounded goal.
|
||||
|
||||
BrowserSkill enforces the mechanical boundary: files are staged under a session-scoped opaque transfer, only daemon-minted capabilities reach the extension, upload/download still obey Agent Window tab checks, and transfers are chunk/size bounded. Upload intercepts the native chooser for one transaction, locates the input activated in the resolved target's document, and assigns only the staged file paths. Download uniquely correlates one exact-target browser intent with one Chrome download in either event order, routes it through a daemon-minted relative directory, and lets the daemon validate and import it. Upload staging remains available for a later form submission and is removed when the session ends. Downloads cannot overwrite an existing destination unless `--overwrite` is explicit. BrowserSkill does not inspect file content or decide whether its meaning is sensitive.
|
||||
|
||||
Do not use `request-help` merely because a native file chooser or browser download is involved; try these commands first. For transfer failures, use the structured error instead of retrying blindly:
|
||||
|
||||
- `reason=file_input_not_activated` means the requested click did not activate exactly one `<input type="file">`; the page may use a non-input picker such as `window.showOpenFilePicker()`. Do not retry blindly. If human help is available, call `request-help` and tell the user the exact original local path to choose. The staged daemon path is internal and must not be shown to the user.
|
||||
- `reason=file_input_probe_failed` means BrowserSkill could not safely establish the browser-side upload transaction. Do not repeat the same action; use `request-help` when available.
|
||||
- `reason=set_file_input_failed` means BrowserSkill found the activated file input but Chrome rejected the staged path or assignment. Check the extension's file-URL access permission; otherwise use `request-help`.
|
||||
- `reason=download_capture_failed` means BrowserSkill could not attribute exactly one completed download to the requested target. Do not retry blindly or accept an unrelated browser download; use `request-help` when available.
|
||||
- `effect_state=none` means BrowserSkill confirmed that no file-transfer effect was committed. Follow the accompanying reason; a corrected target or explicit human fallback may be attempted.
|
||||
- `effect_state=unknown` means the browser may already have attached or created the file. Do not repeat the transfer. Observe the page if that can establish the result; otherwise stop and report the uncertainty.
|
||||
- `effect_state=committed` means the browser-side effect occurred even if later completion or cleanup failed. Do not repeat it; continue only after verifying the resulting page/download state.
|
||||
|
||||
If `request-help` returns `outcome="disabled"`, do not retry it. Stop gracefully and report the transfer mechanism that requires human intervention.
|
||||
|
||||
### Scripting & timing
|
||||
|
||||
| Command | Summary |
|
||||
|---------|---------|
|
||||
| `bsk evaluate <expression>` | Run JS in agent tab (see red lines). JS throw → stderr, **exit 0** (RPC success); use `--json` and check `.ok` to detect JS errors |
|
||||
| `bsk wait-for-navigation` | Block until load/DOM idle/etc. (`--wait-until`, `--timeout`) |
|
||||
| `bsk wait-ms <duration>` | Sleep (`500ms`, `2s`, `1m`; **no** `--session`) |
|
||||
|
||||
### Ask the human for help — `bsk request-help`
|
||||
|
||||
When a step needs a human (captcha, login, OTP) or you want the user to
|
||||
confirm an important action, pause and ask:
|
||||
|
||||
bsk request-help --session <id> --prompt "Solve the captcha, then click Done only after the site accepts it" \
|
||||
--title "Captcha required" --target @e7 --target "#submit" --timeout 5m
|
||||
|
||||
- `--prompt` (required): what the user should do.
|
||||
- `--title` (optional): custom title for the overlay panel. When omitted,
|
||||
the extension shows its default localized title.
|
||||
- `--target` (repeatable): a snapshot ref (`@e7`) or CSS selector
|
||||
(`#submit`) to scroll to and flash-highlight. **Strongly recommended** —
|
||||
whenever the prompt refers to a concrete element (a button to click, a
|
||||
field to fill, a checkbox to toggle), pass its `@eN` ref / selector so the
|
||||
user is guided straight to the right spot instead of hunting for it. For
|
||||
interaction scenarios, always include the relevant target(s); reserve a
|
||||
prompt with no `--target` for cases where there is genuinely no specific
|
||||
element to point at (e.g. "wait for the page to finish loading").
|
||||
- `--timeout` (default `5m`): how long to wait.
|
||||
- `--completion-criteria` (optional): JSON success detector. Use it only
|
||||
when there is a concrete post-help success signal, e.g.
|
||||
`{"any":[{"url_contains":"/dashboard"},{"selector_exists":"[data-testid='account-menu']"}],"stable_for_ms":1000}`.
|
||||
|
||||
The target tab is brought to the foreground; the page stays interactive
|
||||
while the agent control mask is hidden. The call blocks until the user
|
||||
explicitly acts, the timeout expires, cancellation arrives, or explicit
|
||||
completion criteria match. Page reloads, SPA route changes, and captcha
|
||||
refreshes do not return control by themselves. The result `outcome` is one of:
|
||||
|
||||
- `continued` — the user finished and clicked Done / return control (treat as confirm).
|
||||
- `cancelled` — the user clicked Cancel (treat as reject/abort).
|
||||
- `timed_out` — nobody acted within the timeout.
|
||||
- `completed` — the explicit `--completion-criteria` matched while the user had control.
|
||||
- `navigated` — deprecated legacy outcome. Do not rely on navigation as a completion signal.
|
||||
|
||||
`note` carries any text the user typed back. `resolved_targets` reports
|
||||
which refs/selectors matched a live element.
|
||||
|
||||
`request-help` does not refresh the page model after the user returns
|
||||
control. After a `continued` or `completed` result, issue a separate
|
||||
observation tool call (usually `bsk snapshot --session <id>`) before using
|
||||
new refs or reasoning about the post-help page state.
|
||||
#### Disabling request-help (unattended mode)
|
||||
Set `BSK_REQUEST_HELP=off` on unattended servers: `bsk request-help` then
|
||||
returns immediately with `outcome="disabled"` (no overlay, no waiting,
|
||||
exit 0). Any other value keeps it enabled. If you get `disabled`, do not
|
||||
retry — complete the task autonomously or stop gracefully.
|
||||
|
||||
### Recording — `bsk record`
|
||||
|
||||
Capture the user's own actions in the Agent Window to a **trace bundle**, for later LLM-driven automation. New CLI builds request **trace v3** (page observations + action chain); older extensions may still return **trace v2** (actions only), which the CLI exports as a single `trace.json` without `states/`.
|
||||
|
||||
```bash
|
||||
bsk record start --browser <instance-id-or-label> \
|
||||
[--url https://…] [--purpose "publish a wiki doc"] \
|
||||
[--max-page-tokens 3000] [--redact-values] \
|
||||
[--output trace]
|
||||
# `--url` is optional; default https://example.com/ when omitted (must be http(s)).
|
||||
# Blocks until the user clicks Finish in the recording panel, then writes:
|
||||
# trace/trace.json — action chain (+ state index when v3)
|
||||
# trace/states/ — v3 only: one `sN.txt` observe snapshot per settled page state
|
||||
|
||||
bsk record stop [--output trace] # terminal fallback if the browser panel is unavailable
|
||||
```
|
||||
|
||||
`select` matches an option's `value` attribute, not its visible label. Device preset ids are
|
||||
@@ -139,43 +280,11 @@ lowercase and hyphenated, such as `iphone-14`.
|
||||
- `record` captures a user's actions for later replay. Read `bsk record start --help` before use,
|
||||
and never record banking, SSO, password-manager, or other sensitive pages.
|
||||
|
||||
## File transfer
|
||||
|
||||
`upload` and `download` stage files through the daemon; the agent never touches browser-internal
|
||||
paths. Treat upload as disclosure to the website, download as accepting website-controlled bytes.
|
||||
|
||||
Upload has two independent mechanisms — choose explicitly, never rely on automatic fallback:
|
||||
|
||||
- **Default (input mode):** for upload buttons, file-input labels, or "upload from computer"
|
||||
actions. The command clicks the target and intercepts the native file chooser.
|
||||
- **`--mode drop`:** for reliably identified attachment-receiving areas — an explicit drop zone,
|
||||
chat composer, email editor, or form attachment area. Do not target page whitespace, generic
|
||||
containers, or areas whose attachment ownership is ambiguous.
|
||||
|
||||
Decision sequence when uploading:
|
||||
|
||||
1. Try input mode (the default).
|
||||
2. If it returns `reason=file_input_not_activated` with `effect_state=none`, re-observe. When a
|
||||
reliable attachment target exists, try `--mode drop` once against that target.
|
||||
3. Otherwise fall back to `request-help`.
|
||||
4. **Never** switch mechanisms or repeat when `effect_state` is `unknown` or `committed` — the
|
||||
browser may already have applied the file.
|
||||
|
||||
A successful drop means Chrome dispatched the native file-drop event; it does not prove the site
|
||||
accepted the attachment. Observe the page once after the command.
|
||||
|
||||
Download default-refuses to overwrite; pass `--overwrite` when replacing an existing file is
|
||||
intended. Read `bsk upload --help` and `bsk download --help` for all flags and error details.
|
||||
|
||||
## Recover without wandering
|
||||
|
||||
- Stale ref: observe again and retry the intended action once.
|
||||
- Unknown tab or session: list current tabs/sessions; never guess identifiers.
|
||||
- Timeout: inspect current page state before deciding whether one longer purposeful wait is useful.
|
||||
- Fill result unconfirmed (`fill_value_mismatch`): observe the field first; the page may have
|
||||
formatted the value. Continue if the visible result satisfies the user's intent. Otherwise correct
|
||||
the remaining difference; do not blindly repeat fill or immediately request human help. For other
|
||||
fill errors, follow the returned hint and inspect current state before retrying.
|
||||
- Unsupported command: continue with available capabilities; suggest updating only when the missing
|
||||
command is necessary.
|
||||
- Unrecoverable failure: report the blocker and stop the session in a finally-style path.
|
||||
|
||||
@@ -73,14 +73,6 @@ where
|
||||
P: Serialize + Send + 'static,
|
||||
R: DeserializeOwned + Send + 'static,
|
||||
{
|
||||
let parent_cancel = stdin_cancellation();
|
||||
if parent_cancel.as_ref().is_some_and(|rx| *rx.borrow()) {
|
||||
return Err(CliError::from_rpc(RpcError {
|
||||
code: ErrorCode::Cancelled,
|
||||
message: "parent closed the cancellation pipe".into(),
|
||||
data: Some(serde_json::json!({"reason": "cancelled_before_dispatch"})),
|
||||
}));
|
||||
}
|
||||
let rpc_id: RpcId = format!("{}-{}", rpc_id_prefix, random_short_id());
|
||||
let mut client = IpcClient::connect(sock.clone()).await?;
|
||||
let rpc_id_for_call = rpc_id.clone();
|
||||
@@ -96,16 +88,16 @@ where
|
||||
let outcome = tokio::select! {
|
||||
biased;
|
||||
res = &mut main_fut => res?,
|
||||
sig = wait_for_cancel(parent_cancel) => {
|
||||
sig.context("wait for cancellation").map_err(CliError::Local)?;
|
||||
debug!(rpc_id = %rpc_id_for_cancel, "forwarding cancel to daemon");
|
||||
sig = wait_for_sigint() => {
|
||||
sig.context("install SIGINT handler").map_err(CliError::Local)?;
|
||||
debug!(rpc_id = %rpc_id_for_cancel, "SIGINT: forwarding cancel to daemon");
|
||||
let _ = send_cancel(&sock, &rpc_id_for_cancel).await;
|
||||
match tokio::time::timeout(CANCEL_RESPONSE_GRACE, &mut main_fut).await {
|
||||
Ok(res) => res?,
|
||||
Err(_) => {
|
||||
debug!(
|
||||
rpc_id = %rpc_id_for_cancel,
|
||||
"cancel grace elapsed; synthesising cancelled error"
|
||||
"SIGINT: cancel grace elapsed; synthesising cancelled error"
|
||||
);
|
||||
return Err(CliError::from_rpc(RpcError {
|
||||
code: ErrorCode::Cancelled,
|
||||
@@ -120,46 +112,6 @@ where
|
||||
outcome.map_err(CliError::from_rpc)
|
||||
}
|
||||
|
||||
/// Release caller-owned staging even after cancellation. This deliberately
|
||||
/// exposes only transfer.release, not a general bypass for business RPCs.
|
||||
/// The entire batch, including connection setup, shares one bounded budget.
|
||||
pub fn release_transfers<'a>(
|
||||
sock: &Path,
|
||||
transfer_ids: impl IntoIterator<Item = &'a str>,
|
||||
) -> Result<(), CliError> {
|
||||
use bsk_protocol::tools::{TransferIdParams, TransferReleaseResult};
|
||||
const CLEANUP_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
let ids: Vec<_> = transfer_ids.into_iter().collect();
|
||||
if ids.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
let rt = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.context("build runtime for transfer cleanup")?;
|
||||
rt.block_on(async {
|
||||
tokio::time::timeout(CLEANUP_TIMEOUT, async {
|
||||
let mut client = IpcClient::connect(sock).await?;
|
||||
for id in ids {
|
||||
client
|
||||
.call::<_, TransferReleaseResult>(
|
||||
"transfer-release",
|
||||
Method::TransferRelease,
|
||||
Some(TransferIdParams {
|
||||
transfer_id: id.to_string(),
|
||||
}),
|
||||
CLEANUP_TIMEOUT,
|
||||
)
|
||||
.await?
|
||||
.map_err(CliError::from_rpc)?;
|
||||
}
|
||||
Ok::<_, CliError>(())
|
||||
})
|
||||
.await
|
||||
.context("transfer cleanup exceeded its total time budget")?
|
||||
})
|
||||
}
|
||||
|
||||
/// Send a `cancel { rpc_id }` frame over a fresh connection so it
|
||||
/// lands on the daemon while the original call is still parked.
|
||||
///
|
||||
@@ -194,57 +146,8 @@ fn random_short_id() -> String {
|
||||
bytes.iter().map(|b| format!("{b:02x}")).collect()
|
||||
}
|
||||
|
||||
/// Opt-in control pipe for parent processes that cannot send Ctrl-C (Node on
|
||||
/// Windows). One reader survives the short-lived runtimes used by multi-RPC
|
||||
/// commands such as upload. Ordinary CLI stdin and terminal Ctrl-C are unchanged.
|
||||
fn stdin_cancellation() -> Option<tokio::sync::watch::Receiver<bool>> {
|
||||
if std::env::var("BSK_CANCEL_ON_STDIN_CLOSE").as_deref() != Ok("1") {
|
||||
return None;
|
||||
}
|
||||
static CANCEL: std::sync::OnceLock<tokio::sync::watch::Receiver<bool>> =
|
||||
std::sync::OnceLock::new();
|
||||
Some(
|
||||
CANCEL
|
||||
.get_or_init(|| {
|
||||
let (tx, rx) = tokio::sync::watch::channel(false);
|
||||
std::thread::spawn(move || {
|
||||
use std::io::Read;
|
||||
let mut stdin = std::io::stdin().lock();
|
||||
let mut buf = [0; 64];
|
||||
loop {
|
||||
match stdin.read(&mut buf) {
|
||||
Ok(0) => break,
|
||||
Ok(_) => {}
|
||||
Err(err) if err.kind() == std::io::ErrorKind::Interrupted => continue,
|
||||
Err(_) => break,
|
||||
}
|
||||
}
|
||||
let _ = tx.send(true);
|
||||
});
|
||||
rx
|
||||
})
|
||||
.clone(),
|
||||
)
|
||||
}
|
||||
|
||||
async fn wait_for_cancel(
|
||||
parent_cancel: Option<tokio::sync::watch::Receiver<bool>>,
|
||||
) -> anyhow::Result<()> {
|
||||
let parent_closed = async move {
|
||||
match parent_cancel {
|
||||
Some(mut rx) => {
|
||||
let _ = rx.wait_for(|closed| *closed).await;
|
||||
}
|
||||
None => std::future::pending::<()>().await,
|
||||
}
|
||||
};
|
||||
tokio::select! {
|
||||
result = wait_for_sigint() => result,
|
||||
_ = parent_closed => Ok(()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Wait for terminal Ctrl-C.
|
||||
/// Wait for either Ctrl-C or process SIGTERM. Returning here means
|
||||
/// the user has asked us to stop.
|
||||
async fn wait_for_sigint() -> anyhow::Result<()> {
|
||||
tokio::signal::ctrl_c().await.context("listen for SIGINT")?;
|
||||
Ok(())
|
||||
|
||||
@@ -9,7 +9,8 @@ use anyhow::Context;
|
||||
use base64::Engine;
|
||||
use bsk_protocol::Method;
|
||||
use bsk_protocol::tools::{
|
||||
DownloadParams, DownloadResult, TransferChunkParams, TransferChunkResult,
|
||||
DownloadParams, DownloadResult, TransferChunkParams, TransferChunkResult, TransferIdParams,
|
||||
TransferReleaseResult,
|
||||
};
|
||||
use clap::Args;
|
||||
use uuid::Uuid;
|
||||
@@ -69,7 +70,13 @@ pub fn dispatch(args: DownloadArgs, format: Format) -> Result<(), CliError> {
|
||||
CliError::Local(anyhow::anyhow!("daemon returned no download transfer id"))
|
||||
})?;
|
||||
let write_result = write_transfer(&info.sock_path, &transfer_id, &args.out, args.overwrite);
|
||||
let _ = crate::cli::business_rpc::release_transfers(&info.sock_path, [transfer_id.as_str()]);
|
||||
let _: Result<TransferReleaseResult, CliError> = crate::cli::business_rpc::call(
|
||||
info.sock_path,
|
||||
"transfer-release",
|
||||
Method::TransferRelease,
|
||||
Some(TransferIdParams { transfer_id }),
|
||||
Duration::from_secs(5),
|
||||
);
|
||||
write_result?;
|
||||
match format {
|
||||
Format::Json => {
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user