mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
Compare commits
88
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
282e6f5623 | ||
|
|
bd68c293b6 | ||
|
|
35a20582ad | ||
|
|
4e3365450a | ||
|
|
df5ed1a495 | ||
|
|
eefbe03258 | ||
|
|
1d275f047c | ||
|
|
388b3ae2d5 | ||
|
|
36d0d91419 | ||
|
|
bcb9beb4f8 | ||
|
|
34ccb96fd6 | ||
|
|
8dfa097e55 | ||
|
|
383e3128eb | ||
|
|
2d2636030d | ||
|
|
a28b9d55e7 | ||
|
|
c39874591f | ||
|
|
1485addfd5 | ||
|
|
90b0ff301b | ||
|
|
e74f354e90 | ||
|
|
05c036518a | ||
|
|
020fb3212e | ||
|
|
0822c53c65 | ||
|
|
4450fbe28c | ||
|
|
a3bb81bfab | ||
|
|
caea98cd52 | ||
|
|
187b088013 | ||
|
|
5046b5ab44 | ||
|
|
e8137b42cc | ||
|
|
cb5dc148d5 | ||
|
|
3c5f838c56 | ||
|
|
9337bd557b | ||
|
|
162a31ae92 | ||
|
|
2f84234a31 | ||
|
|
72c10aab75 | ||
|
|
1eeb55af02 | ||
|
|
4992a54864 | ||
|
|
8b195eb0df | ||
|
|
eae4db1729 | ||
|
|
e961395be9 | ||
|
|
a6d1d02acc | ||
|
|
5ebe609c40 | ||
|
|
321494d93f | ||
|
|
f2184e74c5 | ||
|
|
eeb633d68e | ||
|
|
6d181105f2 | ||
|
|
ebad0ed1b5 | ||
|
|
498f8fb4a7 | ||
|
|
cef59137e4 | ||
|
|
f70978bfcf | ||
|
|
1498cd425c | ||
|
|
e831d65db3 | ||
|
|
47ac947f18 | ||
|
|
69d64c4eb1 | ||
|
|
99c997d6d5 | ||
|
|
93853ebc65 | ||
|
|
248bb90a2d | ||
|
|
6e6c178434 | ||
|
|
386ef91c24 | ||
|
|
0dc11b30ea | ||
|
|
759eebd4b6 | ||
|
|
fd3ce202a1 | ||
|
|
618bd6a0df | ||
|
|
c455a09fc3 | ||
|
|
b6be356e40 | ||
|
|
945bf1523d | ||
|
|
a928a9fb76 | ||
|
|
0b2d7e95cc | ||
|
|
556bea1645 | ||
|
|
0302d5592b | ||
|
|
eb34ec0dbf | ||
|
|
43067b6bfe | ||
|
|
6cc0d38489 | ||
|
|
0b71963344 | ||
|
|
59e2c3224e | ||
|
|
a77353bce8 | ||
|
|
91621b2545 | ||
|
|
a4fed136e3 | ||
|
|
3c77ef7b01 | ||
|
|
b1b1ae1d48 | ||
|
|
8822bba9a5 | ||
|
|
c74c119a82 | ||
|
|
b801bc1f62 | ||
|
|
bf59753aa3 | ||
|
|
a86cbf724f | ||
|
|
76d447caa0 | ||
|
|
14e5bbe4fc | ||
|
|
bef722107f | ||
|
|
6800f8fdfe |
@@ -37,6 +37,35 @@ jobs:
|
||||
- name: Run Rust tests
|
||||
run: cargo test --workspace --locked
|
||||
|
||||
windows-daemon:
|
||||
name: Windows daemon, installer and cancellation regression tests
|
||||
runs-on: windows-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Run Windows named-pipe tests
|
||||
run: cargo test -p bsk --lib --locked daemon::ipc::windows::tests
|
||||
|
||||
- name: Run Windows process liveness tests
|
||||
run: cargo test -p bsk --lib --locked daemon::lockfile::tests
|
||||
|
||||
- name: Run Windows update tests
|
||||
run: cargo test -p bsk --lib --locked cli::update::tests
|
||||
|
||||
- name: Run Windows update and cancellation process tests
|
||||
run: cargo test -p bsk --test windows_update --test windows_parent_cancel --locked
|
||||
|
||||
- name: Run installer tests (Windows PowerShell 5.1)
|
||||
shell: powershell
|
||||
run: ./scripts/install-windows.test.ps1
|
||||
|
||||
- name: Run installer tests (PowerShell 7)
|
||||
shell: pwsh
|
||||
run: ./scripts/install-windows.test.ps1
|
||||
|
||||
frontend:
|
||||
name: Frontend lint, typecheck, tests, build
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
+109
@@ -0,0 +1,109 @@
|
||||
# Changelog
|
||||
|
||||
All notable changes to BrowserSkill will be documented in this file.
|
||||
The format is based on [Keep a Changelog](https://keepachangelog.com/).
|
||||
|
||||
Starting from 0.2.0, CLI / Extension / DSH Plugin share the same version number.
|
||||
|
||||
## [0.2.1] - 2026-09-09
|
||||
|
||||
### Changed
|
||||
- DSH Plugin: clarified installation, configuration, profile restart, and upgrade instructions
|
||||
|
||||
### Fixed
|
||||
- CLI: reliable Windows self-updates, including executable replacement and daemon restart
|
||||
- CLI: Windows named-pipe connection continuity and stale daemon process detection
|
||||
- Windows installer: PowerShell 5.1 compatibility, path quoting, and staged executable replacement
|
||||
- CLI / DSH Plugin: cancellation propagation and upload/download cleanup when the parent process exits
|
||||
- Extension: tab ownership and session cleanup when agent tabs fail to open or borrowed tabs are returned or closed
|
||||
- Extension: navigation recovery when other extensions restrict Chrome DevTools Protocol access
|
||||
- Extension: background input filling, automatic append behavior, and verification of field values before reporting success
|
||||
- Extension: match observed form state to the correct element after page changes
|
||||
- Extension: UUID generation fallback for recording and overlays on insecure origins
|
||||
- Browser locale matching for regional language variants
|
||||
- DSH Plugin: observation state synchronization, thumbnail capture lifecycle, and stale preview cleanup
|
||||
- DSH Plugin: removed the obsolete client runtime host dependency
|
||||
|
||||
## [0.2.0] - 2026-09-02
|
||||
|
||||
### Added
|
||||
- File transfer: upload and download support across CLI, Extension, and DSH Plugin
|
||||
- File transfer: drag-and-drop upload (`drop-to-upload`)
|
||||
- VOM semantic graph, name enrichment, and hover perception modules
|
||||
- VOM hover probing (opt-in via `observe` parameter)
|
||||
- DSH Plugin: browser tool parity with CLI commands
|
||||
- Edge Add-ons automated publishing in CI
|
||||
- Protocol upgrade reminder when CLI / Extension protocol versions differ
|
||||
- Browser evaluation harness (`evals/browser/`)
|
||||
- Unified release script (`scripts/release.mjs`)
|
||||
|
||||
### Changed
|
||||
- **Version scheme**: all three components now share the same semver
|
||||
- VOM rendering algorithm optimizations
|
||||
- Leaner SKILL.md agent instructions
|
||||
- DSH Plugin: simplified browser commands
|
||||
- Borrow confirmation UX — proactive focus and longer timeout
|
||||
- PiP window now has a close button
|
||||
|
||||
### Fixed
|
||||
- Screenshot media type detection (was hard-coded to `image/png`)
|
||||
- DSH Plugin session lifecycle stability
|
||||
- DSH Plugin Cordis package ID mismatch
|
||||
- Observation thumbnail media type sniffing
|
||||
- Upload/download race conditions and layout bypass issues
|
||||
- VOM repeated name and safety policy issues
|
||||
|
||||
---
|
||||
|
||||
*Previous releases used independent version numbers per component.*
|
||||
|
||||
## CLI 0.1.11 / Extension 0.1.7 / DSH Plugin 0.1.2 — 2026-08-26 ~ 2026-08-29
|
||||
|
||||
### Added
|
||||
- DSH Plugin sidebar integration, session lifecycle, and archive cleanup
|
||||
- Recorder iframe and OOPIF support
|
||||
|
||||
### Changed
|
||||
- VOM functional refactor
|
||||
|
||||
### Fixed
|
||||
- Recorder safety policy and bug fixes
|
||||
|
||||
## CLI 0.1.10 / Extension 0.1.6 — 2026-08-08
|
||||
|
||||
### Added
|
||||
- VOM observation recording and settled-state detection
|
||||
- Record overlay timer
|
||||
|
||||
### Fixed
|
||||
- Browser keepalive disconnect handling
|
||||
|
||||
## CLI 0.1.9 / Extension 0.1.5 — 2026-07-29
|
||||
|
||||
### Added
|
||||
- CLI auto-update mechanism
|
||||
- Trace v3 protocol and recorder
|
||||
|
||||
### Fixed
|
||||
- MV3 keepalive disconnect
|
||||
|
||||
## CLI 0.1.8 / Extension 0.1.4 — 2026-07-22
|
||||
|
||||
### Added
|
||||
- More browser interaction actions
|
||||
|
||||
### Fixed
|
||||
- Windows named-pipe hash-only path issue
|
||||
|
||||
## CLI 0.1.7 / Extension 0.1.3 — 2026-07-07
|
||||
|
||||
Initial public release pair.
|
||||
|
||||
## CLI 0.1.6 — 2026-06-30
|
||||
|
||||
### Fixed
|
||||
- Minor CLI fixes
|
||||
|
||||
## CLI 0.1.5 / Extension 0.1.2 — 2026-06-22
|
||||
|
||||
First tagged releases.
|
||||
Generated
+2
-2
@@ -134,7 +134,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "bsk"
|
||||
version = "0.1.11"
|
||||
version = "0.2.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"base64",
|
||||
@@ -169,7 +169,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "bsk-protocol"
|
||||
version = "0.1.11"
|
||||
version = "0.2.1"
|
||||
dependencies = [
|
||||
"schemars",
|
||||
"semver",
|
||||
|
||||
+2
-2
@@ -7,7 +7,7 @@ members = [
|
||||
default-members = ["crates/bsk-cli"]
|
||||
|
||||
[workspace.package]
|
||||
version = "0.1.11"
|
||||
version = "0.2.1"
|
||||
edition = "2024"
|
||||
rust-version = "1.85"
|
||||
license = "MIT"
|
||||
@@ -43,7 +43,7 @@ rand = "0.8"
|
||||
uuid = { version = "1", features = ["v4", "serde"] }
|
||||
|
||||
# Workspace crates (version required for `cargo publish` of dependents)
|
||||
bsk-protocol = { version = "0.1.9", path = "crates/bsk-protocol" }
|
||||
bsk-protocol = { version = "0.2.1", path = "crates/bsk-protocol" }
|
||||
|
||||
[profile.release]
|
||||
strip = "symbols"
|
||||
|
||||
@@ -147,20 +147,26 @@ Start a new Agent session and write a prompt that needs the browser, for example
|
||||
Using [DeepSeek Harness](https://github.com/deepseek-ai/deepseek-harness) (`dsh`)?
|
||||
BrowserSkill ships a first-class dsh plugin on npm as
|
||||
[`@wxg-prc-cpg/browser-skill-dsh-plugin`](https://www.npmjs.com/package/@wxg-prc-cpg/browser-skill-dsh-plugin).
|
||||
It injects native `browser_*` tools (no shelling out to `bsk`) and a live Web UI
|
||||
overlay of each Agent Window.
|
||||
It gives the agent native `browser_*` tools and a live view of its browser sessions
|
||||
in the Web UI. The plugin runs `bsk` on the agent's behalf.
|
||||
|
||||
Add it to a dsh profile, then start that profile:
|
||||
Install the `bsk` CLI and connect the browser extension first. Then add the plugin
|
||||
to a dsh profile and start it (replace `web` with your profile name):
|
||||
|
||||
```sh
|
||||
dsh plugin --profile web add @wxg-prc-cpg/browser-skill-dsh-plugin
|
||||
dsh --profile web
|
||||
```
|
||||
|
||||
The plugin carries its own copy of the skill, so `bsk install-skill` is not needed
|
||||
for dsh — but the `bsk` CLI and the browser extension are still prerequisites. See
|
||||
the [plugin README](packages/dsh-plugin-browserskill/README.md) for the tool list,
|
||||
configuration, and the observation overlay.
|
||||
The plugin includes the `browser-skill` skill, so `bsk install-skill` is not needed
|
||||
for dsh. Installed plugins do not update automatically. To upgrade this plugin:
|
||||
|
||||
```sh
|
||||
dsh plugin --profile web update @wxg-prc-cpg/browser-skill-dsh-plugin --latest
|
||||
```
|
||||
|
||||
Restart the profile after upgrading. See the
|
||||
[plugin README](packages/dsh-plugin-browserskill/README.md) for usage and configuration.
|
||||
|
||||
## How It Works
|
||||
|
||||
|
||||
+9
-3
@@ -125,16 +125,22 @@ bsk install-skill
|
||||
|
||||
## DeepSeek Harness 插件
|
||||
|
||||
在用 [DeepSeek Harness](https://github.com/deepseek-ai/deepseek-harness)(`dsh`)?BrowserSkill 提供了官方 dsh 插件,已发布到 npm:[`@wxg-prc-cpg/browser-skill-dsh-plugin`](https://www.npmjs.com/package/@wxg-prc-cpg/browser-skill-dsh-plugin)。它会注入原生 `browser_*` 工具(无需再通过 Shell 调用 `bsk`),并在 Web UI 中实时观察每个 Agent Window。
|
||||
在用 [DeepSeek Harness](https://github.com/deepseek-ai/deepseek-harness)(`dsh`)?BrowserSkill 提供了官方 dsh 插件,已发布到 npm:[`@wxg-prc-cpg/browser-skill-dsh-plugin`](https://www.npmjs.com/package/@wxg-prc-cpg/browser-skill-dsh-plugin)。它为 Agent 提供原生 `browser_*` 工具,由插件代为调用 `bsk`,并在 Web UI 中实时展示浏览器会话。
|
||||
|
||||
把它装进某个 dsh profile,然后启动该 profile:
|
||||
先安装 `bsk` CLI 并连接浏览器扩展,再将插件装进 dsh profile 并启动(将 `web` 替换为你的 profile 名称):
|
||||
|
||||
```sh
|
||||
dsh plugin --profile web add @wxg-prc-cpg/browser-skill-dsh-plugin
|
||||
dsh --profile web
|
||||
```
|
||||
|
||||
插件自带 skill,所以在 dsh 下无需执行 `bsk install-skill`;但 `bsk` CLI 和浏览器扩展仍是前置条件。工具清单、配置项与观察浮层见[插件 README](packages/dsh-plugin-browserskill/README.md)。
|
||||
插件自带 `browser-skill` skill,所以在 dsh 下无需执行 `bsk install-skill`。已安装的插件不会自动更新;升级此插件请运行:
|
||||
|
||||
```sh
|
||||
dsh plugin --profile web update @wxg-prc-cpg/browser-skill-dsh-plugin --latest
|
||||
```
|
||||
|
||||
升级后重启该 profile。用法与配置见[插件 README](packages/dsh-plugin-browserskill/README.md)。
|
||||
|
||||
## 工作原理
|
||||
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<title>BrowserSkill file-transfer fixture</title>
|
||||
<style>
|
||||
body { font: 16px system-ui; max-width: 760px; margin: 48px auto; }
|
||||
.actions { display: flex; gap: 12px; align-items: center; margin: 24px 0; }
|
||||
button, a { padding: 10px 16px; border: 1px solid #777; border-radius: 8px; }
|
||||
a[aria-disabled="true"] { opacity: .45; pointer-events: none; }
|
||||
#drop { padding: 36px; border: 3px dashed #777; border-radius: 12px; text-align: center; }
|
||||
#drop.dragging { border-color: #2563eb; background: #eff6ff; }
|
||||
img { display: block; max-width: 480px; max-height: 360px; margin-top: 20px; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Upload then download the same image</h1>
|
||||
<p id="status">No image selected</p>
|
||||
<div class="actions">
|
||||
<button id="upload" type="button">Upload through file input</button>
|
||||
<input id="file" type="file" accept="image/png" hidden />
|
||||
<a id="download" aria-disabled="true">Download uploaded image</a>
|
||||
</div>
|
||||
<div id="drop" role="button" tabindex="0">Drop an image here</div>
|
||||
<img id="preview" alt="Uploaded image preview" hidden />
|
||||
<script>
|
||||
const input = document.querySelector('#file');
|
||||
const upload = document.querySelector('#upload');
|
||||
const drop = document.querySelector('#drop');
|
||||
const download = document.querySelector('#download');
|
||||
const preview = document.querySelector('#preview');
|
||||
const status = document.querySelector('#status');
|
||||
let objectUrl;
|
||||
|
||||
window.__bskTransferFixture = { events: [], result: null };
|
||||
|
||||
function accept(file, mechanism) {
|
||||
if (!file) return;
|
||||
if (objectUrl) URL.revokeObjectURL(objectUrl);
|
||||
objectUrl = URL.createObjectURL(file);
|
||||
preview.src = objectUrl;
|
||||
preview.hidden = false;
|
||||
download.href = objectUrl;
|
||||
download.download = file.name;
|
||||
download.removeAttribute('aria-disabled');
|
||||
status.textContent = `Ready via ${mechanism}: ${file.name} (${file.size} bytes)`;
|
||||
window.__bskTransferFixture.result = {
|
||||
mechanism,
|
||||
name: file.name,
|
||||
size: file.size,
|
||||
type: file.type,
|
||||
hasPathProperty: 'path' in file,
|
||||
};
|
||||
}
|
||||
|
||||
upload.addEventListener('click', () => input.click());
|
||||
input.addEventListener('change', () => accept(input.files[0], 'input'));
|
||||
|
||||
for (const type of ['dragenter', 'dragover']) {
|
||||
drop.addEventListener(type, event => {
|
||||
const types = [...event.dataTransfer.types];
|
||||
window.__bskTransferFixture.events.push({ type, types });
|
||||
if (!types.includes('Files')) return;
|
||||
event.preventDefault();
|
||||
drop.classList.add('dragging');
|
||||
});
|
||||
}
|
||||
drop.addEventListener('dragleave', () => drop.classList.remove('dragging'));
|
||||
drop.addEventListener('drop', event => {
|
||||
window.__bskTransferFixture.events.push({
|
||||
type: 'drop',
|
||||
types: [...event.dataTransfer.types],
|
||||
fileCount: event.dataTransfer.files.length,
|
||||
});
|
||||
event.preventDefault();
|
||||
drop.classList.remove('dragging');
|
||||
accept(event.dataTransfer.files[0], 'drop');
|
||||
});
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@browser-skill/extension",
|
||||
"version": "0.1.7",
|
||||
"version": "0.2.1",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
|
||||
@@ -254,6 +254,69 @@ describe("ChromiumCdp", () => {
|
||||
expect(cdp.isAttached(7)).toBe(false);
|
||||
});
|
||||
|
||||
it.each([
|
||||
false,
|
||||
true,
|
||||
])("waits for detach before reconnecting a returned tab (detach rejects: %s)", async (rejectDetach) => {
|
||||
const { api } = fakeApi();
|
||||
const attached = new Set<number>();
|
||||
let finishDetach!: () => void;
|
||||
vi.mocked(api.attach).mockImplementation(async ({ tabId }) => {
|
||||
if (attached.has(tabId!)) throw new Error("Another debugger is already attached");
|
||||
attached.add(tabId!);
|
||||
});
|
||||
vi.mocked(api.detach).mockImplementation(async ({ tabId }) => {
|
||||
attached.delete(tabId!);
|
||||
});
|
||||
vi.mocked(api.detach).mockImplementationOnce(
|
||||
({ tabId }) =>
|
||||
new Promise<void>((resolve, reject) => {
|
||||
finishDetach = () => {
|
||||
attached.delete(tabId!);
|
||||
if (rejectDetach) reject(new Error("tab already detached"));
|
||||
else resolve();
|
||||
};
|
||||
}),
|
||||
);
|
||||
const cdp = new ChromiumCdp(api);
|
||||
cdp.trackSessionTab("aa11", 7);
|
||||
await cdp.ensureAttached(7);
|
||||
const returning = cdp.releaseSessionTab("aa11", 7);
|
||||
await vi.waitFor(() => expect(api.detach).toHaveBeenCalledOnce());
|
||||
let duplicateDetachFinished = false;
|
||||
const duplicateDetach = cdp.detach(7).then(() => {
|
||||
duplicateDetachFinished = true;
|
||||
});
|
||||
|
||||
cdp.trackSessionTab("bb22", 7);
|
||||
// Register rejection handlers immediately; the pre-fix driver rejects
|
||||
// both commands while the browser still has the previous attachment.
|
||||
const pending = Promise.allSettled([
|
||||
cdp.send(7, "Runtime.evaluate", { expression: "document.title" }),
|
||||
cdp.send(7, "DOM.getDocument"),
|
||||
]);
|
||||
await cdp.send(8, "DOM.getDocument");
|
||||
const finishedEarly = duplicateDetachFinished;
|
||||
finishDetach();
|
||||
await Promise.all([returning, duplicateDetach]);
|
||||
|
||||
expect(await pending).toEqual([
|
||||
{ status: "fulfilled", value: { ok: true } },
|
||||
{ status: "fulfilled", value: { ok: true } },
|
||||
]);
|
||||
expect(finishedEarly).toBe(false);
|
||||
expect(vi.mocked(api.attach).mock.calls.filter(([target]) => target.tabId === 7)).toHaveLength(
|
||||
2,
|
||||
);
|
||||
expect(cdp.isAttached(7)).toBe(true);
|
||||
expect(cdp.isAttached(8)).toBe(true);
|
||||
await cdp.detachSession("bb22");
|
||||
expect(api.detach).toHaveBeenCalledTimes(2);
|
||||
expect(cdp.isAttached(7)).toBe(false);
|
||||
expect(attached.has(7)).toBe(false);
|
||||
expect(cdp.isAttached(8)).toBe(true);
|
||||
});
|
||||
|
||||
it("detachAll() iterates every cached tab", async () => {
|
||||
const { api } = fakeApi();
|
||||
const cdp = new ChromiumCdp(api);
|
||||
@@ -651,6 +714,155 @@ describe("ChromiumCdp", () => {
|
||||
expect(cdp.networkEntriesSince(33, 0, 50, 1000).entries[0].url).toBeUndefined();
|
||||
});
|
||||
|
||||
it("releases only the returning session's claim on a tab", async () => {
|
||||
const { api } = fakeApi();
|
||||
const cdp = new ChromiumCdp(api);
|
||||
await cdp.ensureAttached(1);
|
||||
await cdp.ensureAttached(2);
|
||||
cdp.trackSessionTab("aa11", 1);
|
||||
cdp.trackSessionTab("bb22", 1);
|
||||
cdp.trackSessionTab("aa11", 2);
|
||||
|
||||
await cdp.releaseSessionTab("unknown", 1);
|
||||
await cdp.releaseSessionTab("aa11", 1);
|
||||
expect(api.detach).not.toHaveBeenCalled();
|
||||
|
||||
await cdp.releaseSessionTab("bb22", 1);
|
||||
await cdp.releaseSessionTab("bb22", 1);
|
||||
expect(api.detach).toHaveBeenCalledExactlyOnceWith({ tabId: 1 });
|
||||
expect(cdp.isAttached(2)).toBe(true);
|
||||
|
||||
await cdp.detachSession("aa11");
|
||||
expect(api.detach).toHaveBeenCalledTimes(2);
|
||||
expect(api.detach).toHaveBeenLastCalledWith({ tabId: 2 });
|
||||
});
|
||||
|
||||
it.each([
|
||||
false,
|
||||
true,
|
||||
])("releases a tab with an attachment in flight (new owner: %s)", async (newOwner) => {
|
||||
const { api } = fakeApi();
|
||||
let finishAttach!: () => void;
|
||||
vi.mocked(api.attach).mockImplementationOnce(
|
||||
() =>
|
||||
new Promise<void>((resolve) => {
|
||||
finishAttach = resolve;
|
||||
}),
|
||||
);
|
||||
const cdp = new ChromiumCdp(api);
|
||||
cdp.trackSessionTab("aa11", 1);
|
||||
const attaching = cdp.ensureAttached(1);
|
||||
const releasing = cdp.releaseSessionTab("aa11", 1);
|
||||
if (newOwner) cdp.trackSessionTab("bb22", 1);
|
||||
finishAttach();
|
||||
await Promise.all([attaching, releasing]);
|
||||
|
||||
expect(cdp.isAttached(1)).toBe(newOwner);
|
||||
expect(api.detach).toHaveBeenCalledTimes(newOwner ? 0 : 1);
|
||||
});
|
||||
|
||||
it("stops accepting dialogs on a returned tab even when another session keeps CDP attached", async () => {
|
||||
const { api, onEvent } = fakeApi();
|
||||
let controlled = true;
|
||||
const cdp = new ChromiumCdp(api, { shouldAutoAcceptDialog: () => controlled });
|
||||
cdp.trackSessionTab("aa11", 1);
|
||||
cdp.trackSessionTab("bb22", 1);
|
||||
await cdp.ensureAttached(1);
|
||||
onEvent.fire({ tabId: 1 }, "Page.javascriptDialogOpening", {
|
||||
type: "confirm",
|
||||
message: "before return",
|
||||
});
|
||||
await vi.waitFor(() => expect(cdp.dialogsSince(1, 0)).toHaveLength(1));
|
||||
vi.mocked(api.sendCommand).mockClear();
|
||||
|
||||
controlled = false;
|
||||
await cdp.releaseSessionTab("aa11", 1);
|
||||
expect(cdp.isAttached(1)).toBe(true);
|
||||
onEvent.fire({ tabId: 1 }, "Page.javascriptDialogOpening", {
|
||||
type: "confirm",
|
||||
message: "after return",
|
||||
});
|
||||
await Promise.resolve();
|
||||
expect(api.sendCommand).not.toHaveBeenCalled();
|
||||
expect(cdp.dialogsSince(1, 0)).toHaveLength(1);
|
||||
|
||||
controlled = true;
|
||||
onEvent.fire({ tabId: 1 }, "Page.javascriptDialogOpening", {
|
||||
type: "alert",
|
||||
message: "borrowed again",
|
||||
});
|
||||
await vi.waitFor(() => expect(cdp.dialogsSince(1, 0)).toHaveLength(2));
|
||||
expect(api.sendCommand).toHaveBeenCalledExactlyOnceWith(
|
||||
{ tabId: 1 },
|
||||
"Page.handleJavaScriptDialog",
|
||||
{ accept: true },
|
||||
);
|
||||
});
|
||||
|
||||
it("ignores dialog events after detach, including a pending eligibility check", async () => {
|
||||
const { api, onEvent } = fakeApi();
|
||||
let resolveEligibility!: (value: boolean) => void;
|
||||
const shouldAutoAcceptDialog = vi.fn(
|
||||
() =>
|
||||
new Promise<boolean>((resolve) => {
|
||||
resolveEligibility = resolve;
|
||||
}),
|
||||
);
|
||||
const cdp = new ChromiumCdp(api, { shouldAutoAcceptDialog });
|
||||
await cdp.ensureAttached(1);
|
||||
onEvent.fire({ tabId: 1 }, "Page.javascriptDialogOpening", { type: "alert" });
|
||||
await cdp.detach(1);
|
||||
resolveEligibility(true);
|
||||
await Promise.resolve();
|
||||
onEvent.fire({ tabId: 1 }, "Page.javascriptDialogOpening", { type: "alert" });
|
||||
|
||||
expect(shouldAutoAcceptDialog).toHaveBeenCalledOnce();
|
||||
expect(api.sendCommand).not.toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
"Page.handleJavaScriptDialog",
|
||||
expect.anything(),
|
||||
);
|
||||
expect(cdp.dialogsSince(1, 0)).toEqual([]);
|
||||
});
|
||||
|
||||
it("does not accept dialogs when the tab's current scope cannot be determined", async () => {
|
||||
const { api, onEvent } = fakeApi();
|
||||
const shouldAutoAcceptDialog = vi.fn(async () => {
|
||||
throw new Error("tab is gone");
|
||||
});
|
||||
const cdp = new ChromiumCdp(api, { shouldAutoAcceptDialog });
|
||||
await cdp.ensureAttached(1);
|
||||
vi.mocked(api.sendCommand).mockClear();
|
||||
|
||||
onEvent.fire({ tabId: 1 }, "Page.javascriptDialogOpening", { type: "confirm" });
|
||||
await Promise.resolve();
|
||||
|
||||
expect(shouldAutoAcceptDialog).toHaveBeenCalledWith(1);
|
||||
expect(api.sendCommand).not.toHaveBeenCalled();
|
||||
expect(cdp.dialogsSince(1, 0)).toEqual([]);
|
||||
});
|
||||
|
||||
it("does not restore dialog state when an acceptance completes after detach", async () => {
|
||||
const { api, onEvent } = fakeApi();
|
||||
const cdp = new ChromiumCdp(api);
|
||||
await cdp.ensureAttached(1);
|
||||
let finishAccept!: () => void;
|
||||
vi.mocked(api.sendCommand).mockImplementationOnce(
|
||||
() =>
|
||||
new Promise<void>((resolve) => {
|
||||
finishAccept = resolve;
|
||||
}),
|
||||
);
|
||||
|
||||
onEvent.fire({ tabId: 1 }, "Page.javascriptDialogOpening", { type: "alert" });
|
||||
await cdp.detach(1);
|
||||
finishAccept();
|
||||
await Promise.resolve();
|
||||
|
||||
expect(cdp.dialogsSince(1, 0)).toEqual([]);
|
||||
expect(cdp.dialogCursor(1)).toBe(0);
|
||||
});
|
||||
|
||||
it("detachSession only detaches tabs no other session owns", async () => {
|
||||
const { api } = fakeApi();
|
||||
const cdp = new ChromiumCdp(api);
|
||||
|
||||
@@ -165,6 +165,7 @@ export class ChromiumCdp {
|
||||
private readonly api: CdpDebuggerApi;
|
||||
private readonly attachedTabs = new Set<number>();
|
||||
private readonly attachInFlight = new Map<number, Promise<void>>();
|
||||
private readonly detachInFlight = new Map<number, Promise<void>>();
|
||||
private readonly tabOwners = new Map<number, Set<string>>();
|
||||
private readonly dialogBuffers = new Map<number, JavaScriptDialogInfo[]>();
|
||||
private readonly dialogSequences = new Map<number, number>();
|
||||
@@ -182,7 +183,13 @@ export class ChromiumCdp {
|
||||
private networkSubscription: { dispose(): void } | null = null;
|
||||
private frameTargetSubscription: { dispose(): void } | null = null;
|
||||
|
||||
constructor(api: CdpDebuggerApi = chromeDebuggerApi) {
|
||||
constructor(
|
||||
api: CdpDebuggerApi = chromeDebuggerApi,
|
||||
private readonly options: {
|
||||
/** CDP observation alone does not authorize dismissing user dialogs. */
|
||||
shouldAutoAcceptDialog?: (tabId: number) => boolean | Promise<boolean>;
|
||||
} = {},
|
||||
) {
|
||||
this.api = api;
|
||||
this.bindAutoDetach();
|
||||
this.bindDialogHandler();
|
||||
@@ -193,6 +200,10 @@ export class ChromiumCdp {
|
||||
|
||||
/** Attach to `tabId` if we haven't already in this driver. */
|
||||
async ensureAttached(tabId: number): Promise<void> {
|
||||
// Returning a tab clears the cache before Chrome finishes detaching.
|
||||
// New observers must wait before opening the next connection to that tab.
|
||||
const detaching = this.detachInFlight.get(tabId);
|
||||
if (detaching) await detaching;
|
||||
if (this.attachedTabs.has(tabId)) return;
|
||||
const existing = this.attachInFlight.get(tabId);
|
||||
if (existing) {
|
||||
@@ -425,6 +436,11 @@ export class ChromiumCdp {
|
||||
|
||||
/** Detach if attached; never throws. */
|
||||
async detach(tabId: number): Promise<void> {
|
||||
const existing = this.detachInFlight.get(tabId);
|
||||
if (existing) {
|
||||
await existing;
|
||||
return;
|
||||
}
|
||||
this.attachInFlight.delete(tabId);
|
||||
if (!this.attachedTabs.has(tabId)) return;
|
||||
this.attachedTabs.delete(tabId);
|
||||
@@ -432,13 +448,19 @@ export class ChromiumCdp {
|
||||
this.clearConsoleState(tabId);
|
||||
this.clearNetworkState(tabId);
|
||||
this.clearFrameState(tabId);
|
||||
try {
|
||||
await this.api.detach({ tabId });
|
||||
} catch (err) {
|
||||
// Tab may already be gone — Chrome auto-detaches on close. Log
|
||||
// at debug so production builds aren't noisy.
|
||||
console.debug("[bsk cdp] detach failed (likely tab already closed)", err);
|
||||
}
|
||||
const detach = (async () => {
|
||||
try {
|
||||
await this.api.detach({ tabId });
|
||||
} catch (err) {
|
||||
// Tab may already be gone — Chrome auto-detaches on close. Log
|
||||
// at debug so production builds aren't noisy.
|
||||
console.debug("[bsk cdp] detach failed (likely tab already closed)", err);
|
||||
}
|
||||
})().finally(() => {
|
||||
this.detachInFlight.delete(tabId);
|
||||
});
|
||||
this.detachInFlight.set(tabId, detach);
|
||||
await detach;
|
||||
}
|
||||
|
||||
/** True iff `ensureAttached(tabId)` has succeeded since the last detach. */
|
||||
@@ -453,6 +475,17 @@ export class ChromiumCdp {
|
||||
this.tabOwners.set(tabId, owners);
|
||||
}
|
||||
|
||||
/** Release one session's claim, preserving attachments still used by another. */
|
||||
async releaseSessionTab(sessionId: string, tabId: number): Promise<void> {
|
||||
const owners = this.tabOwners.get(tabId);
|
||||
if (!owners?.delete(sessionId) || owners.size > 0) return;
|
||||
this.tabOwners.delete(tabId);
|
||||
// An observation may still be attaching when the tab is returned. Wait
|
||||
// for it so detach cannot miss the attachment or remove a new owner's claim.
|
||||
await this.attachInFlight.get(tabId)?.catch(() => {});
|
||||
if (!this.tabOwners.has(tabId)) await this.detach(tabId);
|
||||
}
|
||||
|
||||
/** Subscribe to all CDP events. Returned disposable removes the listener. */
|
||||
onEvent(handler: (source: CdpDebuggee, method: string, params: unknown) => void): {
|
||||
dispose(): void;
|
||||
@@ -639,13 +672,23 @@ export class ChromiumCdp {
|
||||
}
|
||||
|
||||
private async onJavaScriptDialogOpening(tabId: number, params: unknown): Promise<void> {
|
||||
if (!this.attachedTabs.has(tabId) && !this.attachInFlight.has(tabId)) return;
|
||||
const parsed = parseDialogOpeningParams(params);
|
||||
try {
|
||||
if (
|
||||
this.options.shouldAutoAcceptDialog &&
|
||||
!(await this.options.shouldAutoAcceptDialog(tabId))
|
||||
) {
|
||||
return;
|
||||
}
|
||||
// The tab may have been returned while its current scope was checked.
|
||||
if (!this.attachedTabs.has(tabId) && !this.attachInFlight.has(tabId)) return;
|
||||
const handleParams: { accept: boolean; promptText?: string } = { accept: true };
|
||||
if (parsed.type === "prompt") {
|
||||
handleParams.promptText = parsed.defaultPrompt ?? "";
|
||||
}
|
||||
await this.api.sendCommand({ tabId }, "Page.handleJavaScriptDialog", handleParams);
|
||||
if (!this.attachedTabs.has(tabId) && !this.attachInFlight.has(tabId)) return;
|
||||
const sequence = (this.dialogSequences.get(tabId) ?? 0) + 1;
|
||||
this.dialogSequences.set(tabId, sequence);
|
||||
this.appendDialog(tabId, {
|
||||
@@ -835,15 +878,9 @@ export class ChromiumCdp {
|
||||
|
||||
/** Detach tabs only when no other live session has claimed them. */
|
||||
async detachSession(sessionId: string): Promise<void> {
|
||||
const tabsToDetach: number[] = [];
|
||||
for (const [tabId, owners] of this.tabOwners) {
|
||||
owners.delete(sessionId);
|
||||
if (owners.size === 0) {
|
||||
this.tabOwners.delete(tabId);
|
||||
tabsToDetach.push(tabId);
|
||||
}
|
||||
}
|
||||
await Promise.all(tabsToDetach.map((tabId) => this.detach(tabId)));
|
||||
await Promise.all(
|
||||
Array.from(this.tabOwners.keys(), (tabId) => this.releaseSessionTab(sessionId, tabId)),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -45,10 +45,32 @@ function portHarness() {
|
||||
|
||||
describe("RecordFrameAgent", () => {
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals();
|
||||
document.documentElement.removeAttribute(RECORD_DOCUMENT_ATTRIBUTE);
|
||||
document.body.replaceChildren();
|
||||
});
|
||||
|
||||
it("starts when crypto.randomUUID is unavailable", async () => {
|
||||
const harness = portHarness();
|
||||
const getRandomValues = vi.fn((bytes: Uint8Array) => {
|
||||
bytes.fill(0x11);
|
||||
return bytes;
|
||||
});
|
||||
vi.stubGlobal("crypto", { getRandomValues });
|
||||
vi.stubGlobal("chrome", {
|
||||
runtime: {
|
||||
connect: vi.fn(() => harness.port),
|
||||
sendMessage: vi.fn(),
|
||||
},
|
||||
});
|
||||
const agent = new RecordFrameAgent();
|
||||
|
||||
await expect(
|
||||
agent.start({ type: RECORD_FRAME_START, requestId: "rec-http", startedAtMs: 10 }),
|
||||
).resolves.toEqual({ ok: true });
|
||||
expect(getRandomValues).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("keeps a failed stop retryable and flushes the final dirty fill", async () => {
|
||||
const harness = portHarness();
|
||||
const sendMessage = vi
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { createRandomUuid } from "@/lib/random-uuid";
|
||||
import {
|
||||
isAcceptedRecordStepAck,
|
||||
RECORD_STEP,
|
||||
@@ -18,7 +19,7 @@ export class RecordStepDelivery {
|
||||
constructor(
|
||||
requestId: string,
|
||||
send: SendRecordStepMessage = (message) => chrome.runtime.sendMessage(message),
|
||||
producerId: string = crypto.randomUUID(),
|
||||
producerId: string = createRandomUuid(),
|
||||
) {
|
||||
this.#requestId = requestId;
|
||||
this.#producerId = producerId;
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { createRandomUuid } from "@/lib/random-uuid";
|
||||
import {
|
||||
markRecordingDocument,
|
||||
type RecordingDocumentMarker,
|
||||
@@ -67,7 +68,7 @@ export class RecordFrameAgent {
|
||||
async #start(
|
||||
message: RecordFrameStartMessage,
|
||||
): Promise<{ ok: true } | { ok: false; error: string }> {
|
||||
const producerId = crypto.randomUUID();
|
||||
const producerId = createRandomUuid();
|
||||
const root = await waitForDocumentElement();
|
||||
if (this.#disposed) return { ok: false, error: "recording document is disposed" };
|
||||
const marker = markRecordingDocument(producerId, root);
|
||||
|
||||
@@ -25,7 +25,7 @@ import { recordFrameCoordinator } from "@/lib/recording/frame-coordinator";
|
||||
import { attachSessionsLiveFlag } from "@/lib/sessions-live-flag";
|
||||
import { createDisconnectCleanup } from "@/session-manager/disconnect-cleanup";
|
||||
import { attachSessionEventHandler } from "@/session-manager/event-handler";
|
||||
import { SessionManager } from "@/session-manager/manager";
|
||||
import { isAgentControlledTab, SessionManager } from "@/session-manager/manager";
|
||||
import {
|
||||
attachBorrowNotificationButtonHandler,
|
||||
attachBorrowNotificationClickHandler,
|
||||
@@ -41,6 +41,8 @@ import {
|
||||
attachRecordStepListener,
|
||||
type RecordRuntimeDeps,
|
||||
} from "@/tools/record";
|
||||
import { chromeTabsApi } from "@/tools/shared";
|
||||
import { chromeTabMutationApi } from "@/tools/tabs";
|
||||
import { detectBrowserMeta } from "@/transport/handshake";
|
||||
import type { Transport } from "@/transport/transport";
|
||||
import { WSTransport } from "@/transport/ws-transport";
|
||||
@@ -49,7 +51,12 @@ export default defineBackground(() => {
|
||||
const controller = new ConnectionController();
|
||||
const transport = new WSTransport({ url: __BSK_DAEMON_WS_URL__ });
|
||||
const sessions = new SessionManager();
|
||||
const cdp = new ChromiumCdp();
|
||||
const cdp = new ChromiumCdp(undefined, {
|
||||
shouldAutoAcceptDialog: async (tabId) => {
|
||||
const tab = await chrome.tabs.get(tabId);
|
||||
return sessions.findByWindowId(tab.windowId) !== null;
|
||||
},
|
||||
});
|
||||
const sessionsLive = attachSessionsLiveFlag({ manager: sessions });
|
||||
let overlayGeneration = 0;
|
||||
const controlModes = new Map<string, OverlayMode>();
|
||||
@@ -80,6 +87,24 @@ export default defineBackground(() => {
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Authoritative overlay state for a specific tab. Agent Window tabs are
|
||||
* free by default; only tabs explicitly claimed through session startup,
|
||||
* `tab_create`, or `tab_borrow` receive the control overlay.
|
||||
*/
|
||||
function overlayStateForTab(tabId?: number, windowId?: number): OverlayAgentStateMessage {
|
||||
if (typeof tabId === "number" && typeof windowId === "number") {
|
||||
const ctx = sessions.findByWindowId(windowId);
|
||||
if (ctx && isAgentControlledTab(ctx, tabId)) return overlayStateForWindow(windowId);
|
||||
}
|
||||
return {
|
||||
type: OVERLAY_AGENT_STATE,
|
||||
sessionId: null,
|
||||
mode: "hidden",
|
||||
generation: overlayGeneration,
|
||||
};
|
||||
}
|
||||
|
||||
async function pushOverlayStateToTab(
|
||||
tabId: number,
|
||||
state: OverlayAgentStateMessage,
|
||||
@@ -91,13 +116,18 @@ export default defineBackground(() => {
|
||||
}
|
||||
}
|
||||
|
||||
async function pushOverlayStateForTab(tabId: number, windowId?: number): Promise<void> {
|
||||
const state = overlayStateForTab(tabId, windowId);
|
||||
await pushOverlayStateToTab(tabId, state);
|
||||
}
|
||||
|
||||
async function pushOverlayStateForWindow(windowId: number): Promise<void> {
|
||||
const state = overlayStateForWindow(windowId);
|
||||
const tabs = await chrome.tabs.query({ windowId });
|
||||
await Promise.all(
|
||||
tabs.map((tab) =>
|
||||
typeof tab.id === "number" ? pushOverlayStateToTab(tab.id, state) : Promise.resolve(),
|
||||
),
|
||||
tabs.map((tab) => {
|
||||
if (typeof tab.id !== "number") return Promise.resolve();
|
||||
return pushOverlayStateForTab(tab.id, windowId);
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
@@ -136,13 +166,30 @@ export default defineBackground(() => {
|
||||
if (typeof tab.windowId !== "number") return;
|
||||
pushOverlayStateForAgentWindow(tab.windowId);
|
||||
});
|
||||
// Newly observed tabs are free unless the agent's own creation path has
|
||||
// already claimed their concrete id. This listener only renders that
|
||||
// state; it never infers or mutates ownership from event ordering.
|
||||
chrome.tabs.onCreated.addListener((tab) => {
|
||||
if (typeof tab.windowId !== "number" || typeof tab.id !== "number") return;
|
||||
if (!sessions.findByWindowId(tab.windowId)) return;
|
||||
void pushOverlayStateForTab(tab.id, tab.windowId);
|
||||
});
|
||||
chrome.tabs.onRemoved.addListener((tabId, removeInfo) => {
|
||||
sessions.forgetClosedTab(tabId, { isWindowClosing: removeInfo.isWindowClosing });
|
||||
});
|
||||
// Re-sync the storage.session flag on SW startup so a previous SW's
|
||||
// stale `true` does not keep waking us on every page load until the
|
||||
// first mutation (review M4/M5 round 3 m-R3-1).
|
||||
void sessionsLive.refresh();
|
||||
const cleanupAfterDisconnect = createDisconnectCleanup({
|
||||
manager: sessions,
|
||||
sessionStopDeps: { cdp },
|
||||
// Same contract as the dispatcher's `tool.session_stop`: without these
|
||||
// deps the agent-tab cleanup and the window-release path are dead code.
|
||||
sessionStopDeps: {
|
||||
cdp,
|
||||
tabManagement: { tabs: chromeTabMutationApi },
|
||||
tabsQuery: chromeTabsApi,
|
||||
},
|
||||
onSessionsChanged: () => {
|
||||
void sessionsLive.syncFromManager();
|
||||
},
|
||||
@@ -176,6 +223,9 @@ export default defineBackground(() => {
|
||||
recording: recordDeps,
|
||||
onSessionsChanged: onOverlaySessionStateChanged,
|
||||
onBrowserControlResumed,
|
||||
onAgentTabClaimed: (tabId, windowId) => {
|
||||
void pushOverlayStateForTab(tabId, windowId);
|
||||
},
|
||||
approveBorrow: (ctx) =>
|
||||
requestBorrowConfirmation(ctx.tabId, {
|
||||
...(ctx.signal !== undefined ? { signal: ctx.signal } : {}),
|
||||
@@ -309,7 +359,11 @@ export default defineBackground(() => {
|
||||
}
|
||||
|
||||
if (msg.kind === OVERLAY_MSG_READY) {
|
||||
sendResponse(overlayStateForWindow(sender.tab?.windowId));
|
||||
// Resolve from explicit tab ownership before sending so free tabs never
|
||||
// flash the control mask.
|
||||
if (typeof sender.tab?.id === "number") {
|
||||
void pushOverlayStateForTab(sender.tab.id, sender.tab.windowId);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
@@ -122,8 +122,8 @@ describe("App", () => {
|
||||
|
||||
expect(screen.queryByText(/^扩展 v/)).toBeNull();
|
||||
expect(screen.queryByText(/^daemon v/)).toBeNull();
|
||||
expect(screen.getByTitle("扩展版本").textContent).toBe(EXTENSION_VERSION);
|
||||
expect(screen.getByTitle("bsk 版本").textContent).toBe(mockDaemonVersion);
|
||||
expect(screen.getByTitle("扩展版本").textContent).toBe(`Ext ${EXTENSION_VERSION}`);
|
||||
expect(screen.getByTitle("bsk 版本").textContent).toBe(`CLI ${mockDaemonVersion}`);
|
||||
expect(screen.getByText("03c3e47f")).toBeTruthy();
|
||||
|
||||
const copyButton = screen.getByRole("button", { name: "复制实例 ID" });
|
||||
|
||||
@@ -267,9 +267,9 @@ export function App() {
|
||||
data-slot="popup-meta"
|
||||
>
|
||||
<div className="flex shrink-0 items-center gap-1">
|
||||
<span title={t("popup.extensionVersionHint")}>{extensionVersion}</span>
|
||||
<span title={t("popup.daemonVersionHint")}>CLI {daemonVersion}</span>
|
||||
<span aria-hidden>/</span>
|
||||
<span title={t("popup.daemonVersionHint")}>{daemonVersion}</span>
|
||||
<span title={t("popup.extensionVersionHint")}>Ext {extensionVersion}</span>
|
||||
</div>
|
||||
<div className="flex min-w-0 items-center justify-end gap-1">
|
||||
<span className="shrink-0">{t("popup.instanceTitle")}</span>
|
||||
|
||||
@@ -2,7 +2,7 @@ import { describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
CAPTURE_SUPPRESS,
|
||||
type CaptureSuppressMessage,
|
||||
withOverlaysHiddenForCapture,
|
||||
withExtensionOverlayHidden,
|
||||
} from "../capture-suppress-bridge";
|
||||
|
||||
function recordingSendToTab(events: string[]) {
|
||||
@@ -12,12 +12,12 @@ function recordingSendToTab(events: string[]) {
|
||||
});
|
||||
}
|
||||
|
||||
describe("withOverlaysHiddenForCapture", () => {
|
||||
describe("withExtensionOverlayHidden", () => {
|
||||
it("sends begin → fn → end and returns fn's result", async () => {
|
||||
const events: string[] = [];
|
||||
const sendToTab = recordingSendToTab(events);
|
||||
|
||||
const result = await withOverlaysHiddenForCapture(
|
||||
const result = await withExtensionOverlayHidden(
|
||||
7,
|
||||
async () => {
|
||||
events.push("capture");
|
||||
@@ -37,7 +37,7 @@ describe("withOverlaysHiddenForCapture", () => {
|
||||
const sendToTab = recordingSendToTab(events);
|
||||
|
||||
await expect(
|
||||
withOverlaysHiddenForCapture(
|
||||
withExtensionOverlayHidden(
|
||||
7,
|
||||
async () => {
|
||||
events.push("capture");
|
||||
@@ -55,7 +55,7 @@ describe("withOverlaysHiddenForCapture", () => {
|
||||
});
|
||||
const fn = vi.fn(async () => "shot");
|
||||
|
||||
const result = await withOverlaysHiddenForCapture(7, fn, sendToTab);
|
||||
const result = await withExtensionOverlayHidden(7, fn, sendToTab);
|
||||
|
||||
expect(result).toBe("shot");
|
||||
expect(fn).toHaveBeenCalledTimes(1);
|
||||
@@ -69,7 +69,7 @@ describe("withOverlaysHiddenForCapture", () => {
|
||||
return { type: CAPTURE_SUPPRESS, ok: true };
|
||||
});
|
||||
|
||||
const result = await withOverlaysHiddenForCapture(7, async () => "shot", sendToTab);
|
||||
const result = await withExtensionOverlayHidden(7, async () => "shot", sendToTab);
|
||||
|
||||
expect(result).toBe("shot");
|
||||
expect(sendToTab).toHaveBeenCalledTimes(2);
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { createRandomUuid } from "../random-uuid";
|
||||
|
||||
describe("createRandomUuid", () => {
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
it("uses crypto.randomUUID when it is available", () => {
|
||||
const randomUUID = vi.fn(() => "00000000-0000-4000-8000-000000000000");
|
||||
vi.stubGlobal("crypto", { randomUUID, getRandomValues: vi.fn() });
|
||||
|
||||
expect(createRandomUuid()).toBe("00000000-0000-4000-8000-000000000000");
|
||||
expect(randomUUID).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("falls back to getRandomValues and sets UUID v4 bits", () => {
|
||||
const bytes = new Uint8Array(16).fill(0xff);
|
||||
const getRandomValues = vi.fn((target: Uint8Array) => {
|
||||
target.set(bytes);
|
||||
return target;
|
||||
});
|
||||
vi.stubGlobal("crypto", { getRandomValues });
|
||||
|
||||
const uuid = createRandomUuid();
|
||||
|
||||
expect(uuid).toBe("ffffffff-ffff-4fff-bfff-ffffffffffff");
|
||||
expect(getRandomValues).toHaveBeenCalledOnce();
|
||||
});
|
||||
});
|
||||
@@ -1,9 +1,9 @@
|
||||
/**
|
||||
* Wire protocol and background-side helper for hiding the extension's
|
||||
* in-page overlay while a screenshot is captured, so captured frames only
|
||||
* contain the page itself.
|
||||
* Wire protocol and background-side helper for temporarily excluding the
|
||||
* extension's in-page overlay. Screenshots use it to avoid captured chrome;
|
||||
* coordinate-driven native input uses it so Chrome hit-tests the page.
|
||||
*
|
||||
* The background wraps every capture in `withOverlaysHiddenForCapture`,
|
||||
* The background wraps each bounded operation in `withExtensionOverlayHidden`,
|
||||
* which sends `begin` (the content script hides the overlay host and only
|
||||
* acks once the compositor has produced an overlay-free frame) and always
|
||||
* follows up with `end`, even when the capture throws. Tabs without the
|
||||
@@ -46,7 +46,7 @@ const defaultSendToTab: CaptureSuppressSendToTab = (tabId, message) =>
|
||||
* (or any other `begin` failure) simply skips suppression — there is no
|
||||
* overlay to hide in that tab.
|
||||
*/
|
||||
export async function withOverlaysHiddenForCapture<T>(
|
||||
export async function withExtensionOverlayHidden<T>(
|
||||
tabId: number,
|
||||
fn: () => Promise<T>,
|
||||
sendToTab: CaptureSuppressSendToTab = defaultSendToTab,
|
||||
|
||||
@@ -3,8 +3,10 @@
|
||||
* between the content-script control overlay and the background SW.
|
||||
*
|
||||
* Content script → background:
|
||||
* - `{ kind: "overlay.ready" }` → background replies with the authoritative
|
||||
* overlay state for the sender's window.
|
||||
* - `{ kind: "overlay.ready" }` → background decides the per-tab state first
|
||||
* (see overlayStateForTab) and pushes the authoritative overlay state for
|
||||
* that tab. It also proactively pushes on tab create and control-mode
|
||||
* change, so a tab usually receives state before it ever sends ready.
|
||||
* - `{ kind: "overlay.interrupt", sessionId }` → background asks the
|
||||
* daemon (via a `session.user_interrupt` WS event) to cancel
|
||||
* every inflight + queued tool call for that session with
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
/**
|
||||
* Generate a UUID v4 in browser contexts where randomUUID() is unavailable.
|
||||
*
|
||||
* Crypto.randomUUID() is restricted to secure contexts, while
|
||||
* crypto.getRandomValues() is available to content scripts on ordinary HTTP
|
||||
* origins. BrowserSkill records many intranet pages served over HTTP, so the
|
||||
* fallback is required for recording to work on those pages.
|
||||
*/
|
||||
export function createRandomUuid(): string {
|
||||
const cryptoApi = globalThis.crypto;
|
||||
if (typeof cryptoApi.randomUUID === "function") return cryptoApi.randomUUID();
|
||||
|
||||
const bytes = new Uint8Array(16);
|
||||
cryptoApi.getRandomValues(bytes);
|
||||
bytes[6] = (bytes[6]! & 0x0f) | 0x40;
|
||||
bytes[8] = (bytes[8]! & 0x3f) | 0x80;
|
||||
|
||||
const hex = Array.from(bytes, (byte) => byte.toString(16).padStart(2, "0")).join("");
|
||||
return [
|
||||
hex.slice(0, 8),
|
||||
hex.slice(8, 12),
|
||||
hex.slice(12, 16),
|
||||
hex.slice(16, 20),
|
||||
hex.slice(20),
|
||||
].join("-");
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
import { createRandomUuid } from "../random-uuid";
|
||||
import {
|
||||
isRecordFrameQueryMessage,
|
||||
RECORD_FRAME_PORT,
|
||||
@@ -188,7 +189,7 @@ export class RecordFrameCoordinator {
|
||||
const recording = this.#recordings.get(requestId);
|
||||
if (!recording) return true;
|
||||
recording.finishing = true;
|
||||
const commandId = crypto.randomUUID();
|
||||
const commandId = createRandomUuid();
|
||||
const results = await Promise.all(
|
||||
[...recording.agents.values()].map((agent) => this.#stopAgent(agent, requestId, commandId)),
|
||||
);
|
||||
|
||||
@@ -9,7 +9,7 @@ describe("disconnect session cleanup", () => {
|
||||
const manager = new SessionManager({
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => nextWindowId++),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
remove,
|
||||
},
|
||||
});
|
||||
@@ -41,7 +41,7 @@ describe("disconnect session cleanup", () => {
|
||||
const manager = new SessionManager({
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 100),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
remove: vi.fn(() => removeGate),
|
||||
},
|
||||
});
|
||||
|
||||
@@ -39,7 +39,7 @@ describe("attachSessionEventHandler", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 4242),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
await manager.start("aa11");
|
||||
@@ -73,7 +73,7 @@ describe("attachSessionEventHandler", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 4242),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
const ctx = await manager.start("aa11");
|
||||
@@ -87,6 +87,7 @@ describe("attachSessionEventHandler", () => {
|
||||
windowEvents: events.api,
|
||||
});
|
||||
|
||||
manager.forgetClosedTab(7, { isWindowClosing: true });
|
||||
events.emit(4242);
|
||||
for (let i = 0; i < 4; i += 1) await Promise.resolve();
|
||||
|
||||
@@ -113,7 +114,7 @@ describe("attachSessionEventHandler", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 1),
|
||||
remove: vi.fn(),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
const transport = fakeTransport();
|
||||
@@ -129,7 +130,7 @@ describe("attachSessionEventHandler", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 1),
|
||||
remove: vi.fn(),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
const transport = fakeTransport();
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import type { AgentWindowApi, AgentWindowCreateOptions } from "../agent-window";
|
||||
import { SessionManager } from "../manager";
|
||||
import { isAgentControlledTab, SessionManager } from "../manager";
|
||||
|
||||
function fakeAgentWindow(): AgentWindowApi & {
|
||||
createMock: ReturnType<typeof vi.fn>;
|
||||
@@ -13,7 +13,7 @@ function fakeAgentWindow(): AgentWindowApi & {
|
||||
return id;
|
||||
});
|
||||
const removeMock = vi.fn(async (_id: number) => {});
|
||||
const ensureActiveTabMock = vi.fn(async (_windowId: number, _url: string) => {});
|
||||
const ensureActiveTabMock = vi.fn(async (_windowId: number, _url: string) => 0);
|
||||
return {
|
||||
create: createMock,
|
||||
remove: removeMock,
|
||||
@@ -151,6 +151,76 @@ describe("SessionManager", () => {
|
||||
expect(sm.list()).toEqual([]);
|
||||
});
|
||||
|
||||
describe("explicit tab claims", () => {
|
||||
it("claims the exact home tab returned by AgentWindowApi", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow() });
|
||||
const ctx = await sm.start("aa11");
|
||||
expect(ctx.agentCreatedTabs).toEqual(new Set([0]));
|
||||
expect(isAgentControlledTab(ctx, 0)).toBe(true);
|
||||
});
|
||||
|
||||
it("keeps every other tab free without classification state", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow() });
|
||||
const ctx = await sm.start("aa11");
|
||||
expect(isAgentControlledTab(ctx, 99)).toBe(false);
|
||||
expect(ctx).not.toHaveProperty("userTabs");
|
||||
expect(ctx).not.toHaveProperty("pendingAgentTabCount");
|
||||
});
|
||||
|
||||
it("treats a committed borrow as an explicit claim", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow() });
|
||||
const ctx = await sm.start("aa11");
|
||||
const reservation = sm.tryReserveBorrow(42, ctx.sessionId);
|
||||
if ("borrowedBy" in reservation) throw new Error("unexpected borrow conflict");
|
||||
reservation.commit({ tabId: 42, originalWindowId: 7, originalIndex: 3 });
|
||||
expect(isAgentControlledTab(ctx, 42)).toBe(true);
|
||||
});
|
||||
|
||||
it("forgets an agent-created tab after Chrome removes it", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow() });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.agentCreatedTabs.add(42);
|
||||
sm.forgetClosedTab(42);
|
||||
expect(ctx.agentCreatedTabs.has(42)).toBe(false);
|
||||
expect(isAgentControlledTab(ctx, 0)).toBe(true);
|
||||
});
|
||||
|
||||
it("forgets closed borrows idempotently without affecting other tabs or sessions", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow() });
|
||||
const a = await sm.start("aa11");
|
||||
const b = await sm.start("bb22");
|
||||
a.borrowedTabs.set(42, { tabId: 42, originalWindowId: 7, originalIndex: 0 });
|
||||
a.borrowedTabs.set(43, { tabId: 43, originalWindowId: 7, originalIndex: 1 });
|
||||
b.borrowedTabs.set(44, { tabId: 44, originalWindowId: 8, originalIndex: 0 });
|
||||
|
||||
sm.forgetClosedTab(42);
|
||||
sm.forgetClosedTab(42);
|
||||
sm.forgetClosedTab(999);
|
||||
|
||||
expect(isAgentControlledTab(a, 42)).toBe(false);
|
||||
expect(sm.findBorrowingSession(42, null)).toBeNull();
|
||||
expect([...a.borrowedTabs.keys()]).toEqual([43]);
|
||||
expect([...b.borrowedTabs.keys()]).toEqual([44]);
|
||||
expect(sm.list()).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("prevents an in-flight borrow from reclaiming a closed tab", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow() });
|
||||
const ctx = await sm.start("aa11");
|
||||
const reservation = sm.tryReserveBorrow(42, ctx.sessionId);
|
||||
if ("borrowedBy" in reservation) throw new Error("unexpected borrow conflict");
|
||||
|
||||
sm.forgetClosedTab(42);
|
||||
|
||||
expect(() =>
|
||||
reservation.commit({ tabId: 42, originalWindowId: 7, originalIndex: 0 }),
|
||||
).toThrow(/reservation disappeared/);
|
||||
reservation.release();
|
||||
expect(sm.findBorrowingSession(42, null)).toBeNull();
|
||||
expect(ctx.borrowedTabs.has(42)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("findBorrowingSession", () => {
|
||||
it("returns null when no session has borrowed the tab", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow() });
|
||||
|
||||
@@ -14,8 +14,11 @@ export interface AgentWindowApi {
|
||||
* Guarantee the Agent Window has an active, CDP-navigable tab.
|
||||
* `chrome://` pages (including the New Tab page) reject `Page.navigate`,
|
||||
* so sessions bootstrap with `about:blank` instead.
|
||||
*
|
||||
* Resolves with the id of the activated (or newly created) tab, so callers
|
||||
* can track the session's home tab without re-querying Chrome.
|
||||
*/
|
||||
ensureActiveTab(windowId: number, url: string): Promise<void>;
|
||||
ensureActiveTab(windowId: number, url: string): Promise<number>;
|
||||
}
|
||||
|
||||
/** Creation hints for a new Agent Window. */
|
||||
@@ -49,15 +52,19 @@ export const chromeAgentWindowApi: AgentWindowApi = {
|
||||
// cancellation success while the Agent Window remains open.
|
||||
await chrome.windows.remove(windowId);
|
||||
},
|
||||
async ensureActiveTab(windowId: number, url: string): Promise<void> {
|
||||
async ensureActiveTab(windowId: number, url: string): Promise<number> {
|
||||
const tabs = await chrome.tabs.query({ windowId });
|
||||
const first = tabs.find((t) => typeof t.id === "number");
|
||||
if (first?.id) {
|
||||
if (first?.id !== undefined) {
|
||||
if (!first.active) {
|
||||
await chrome.tabs.update(first.id, { active: true });
|
||||
}
|
||||
return;
|
||||
return first.id;
|
||||
}
|
||||
await chrome.tabs.create({ windowId, url, active: true });
|
||||
const created = await chrome.tabs.create({ windowId, url, active: true });
|
||||
if (typeof created?.id !== "number") {
|
||||
throw new Error("[bh] chrome.tabs.create returned no tab id");
|
||||
}
|
||||
return created.id;
|
||||
},
|
||||
};
|
||||
|
||||
@@ -6,9 +6,20 @@ export interface SessionContext {
|
||||
agentWindowId: number;
|
||||
refStore: RefStore;
|
||||
borrowedTabs: Map<number, BorrowedTab>;
|
||||
/**
|
||||
* Tabs explicitly claimed by the agent because it created them. This
|
||||
* includes the Agent Window's home tab and tabs created by `tool.tab_create`.
|
||||
* Tabs opened through Chrome UI never enter this set.
|
||||
*/
|
||||
agentCreatedTabs: Set<number>;
|
||||
createdAtMs: number;
|
||||
}
|
||||
|
||||
/** Whether this session has explicitly claimed control of `tabId`. */
|
||||
export function isAgentControlledTab(ctx: SessionContext, tabId: number): boolean {
|
||||
return ctx.agentCreatedTabs.has(tabId) || ctx.borrowedTabs.has(tabId);
|
||||
}
|
||||
|
||||
export interface BorrowedTab {
|
||||
tabId: number;
|
||||
originalWindowId: number;
|
||||
@@ -106,6 +117,19 @@ export class SessionManager {
|
||||
return Array.from(this.sessions.values());
|
||||
}
|
||||
|
||||
/**
|
||||
* Forget a tab Chrome has removed, including any uncommitted borrow.
|
||||
* Whole-window closures keep committed borrows until the window-removed
|
||||
* handler reports which user tabs could not be returned.
|
||||
*/
|
||||
forgetClosedTab(tabId: number, { isWindowClosing = false } = {}): void {
|
||||
this.borrowReservations.delete(tabId);
|
||||
for (const ctx of this.sessions.values()) {
|
||||
ctx.agentCreatedTabs.delete(tabId);
|
||||
if (!isWindowClosing) ctx.borrowedTabs.delete(tabId);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Look up whether `tabId` is currently borrowed by some *other*
|
||||
* session than the one calling. Used by M8 `tab_borrow` to refuse
|
||||
@@ -179,7 +203,7 @@ export class SessionManager {
|
||||
const { signal: _signal, ...createOptions } = opts;
|
||||
windowId = await this.agentWindow.create(AGENT_WINDOW_HOME, createOptions);
|
||||
throwIfSessionStartAborted(opts.signal);
|
||||
await this.agentWindow.ensureActiveTab(windowId, AGENT_WINDOW_HOME);
|
||||
const homeTabId = await this.agentWindow.ensureActiveTab(windowId, AGENT_WINDOW_HOME);
|
||||
throwIfSessionStartAborted(opts.signal);
|
||||
|
||||
const ctx: SessionContext = {
|
||||
@@ -187,6 +211,10 @@ export class SessionManager {
|
||||
agentWindowId: windowId,
|
||||
refStore: new RefStore(),
|
||||
borrowedTabs: new Map(),
|
||||
// The home tab is the session's first explicit claim. Every other
|
||||
// tab remains free until `tab_create` or `tab_borrow` identifies it
|
||||
// by its concrete Chrome tab id.
|
||||
agentCreatedTabs: new Set([homeTabId]),
|
||||
createdAtMs: this.now(),
|
||||
};
|
||||
this.sessions.set(sessionId, ctx);
|
||||
|
||||
@@ -13,7 +13,7 @@ function fakeAgentWindow(ids: number[]) {
|
||||
return id;
|
||||
}),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -49,13 +49,54 @@ describe("ToolDispatcher", () => {
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
it.each([
|
||||
"tool.snapshot",
|
||||
"tool.evaluate",
|
||||
"tool.get_html",
|
||||
"tool.console",
|
||||
])("annotates Chrome extension-access failures from %s", async (method) => {
|
||||
const tab = { id: 7, windowId: 4242, active: true, url: "https://example.test" };
|
||||
vi.stubGlobal("chrome", {
|
||||
tabs: {
|
||||
get: vi.fn(async () => tab),
|
||||
query: vi.fn(async () => [tab]),
|
||||
},
|
||||
});
|
||||
const { transport, sent, deliver } = fakeTransport();
|
||||
const sessions = new SessionManager({
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 4242),
|
||||
remove: vi.fn(),
|
||||
ensureActiveTab: vi.fn(),
|
||||
},
|
||||
});
|
||||
await sessions.start("test");
|
||||
const error = new Error("Cannot access a chrome-extension:// URL of different extension");
|
||||
const cdp = {
|
||||
send: vi.fn().mockRejectedValue(error),
|
||||
ensureConsoleCapture: vi.fn().mockRejectedValue(error),
|
||||
consoleEntriesSince: vi.fn(),
|
||||
} as unknown as TestDispatcherCdp;
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions, cdp });
|
||||
dispatcher.start();
|
||||
deliver(makeRequest(method, { session_id: "test", expression: "document.title" }));
|
||||
await vi.waitFor(() => expect(sent).toHaveLength(1));
|
||||
expect(sent[0]).toMatchObject({
|
||||
error: {
|
||||
code: "cdp_failed",
|
||||
data: { reason: "cdp_extension_access_denied" },
|
||||
},
|
||||
});
|
||||
dispatcher.stop();
|
||||
});
|
||||
|
||||
it("uses the configured recording runtime for start and stop", async () => {
|
||||
const { transport, sent, deliver } = fakeTransport();
|
||||
const sessions = new SessionManager({
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 4242),
|
||||
remove: vi.fn(),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
await sessions.start("aa11");
|
||||
@@ -117,7 +158,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 4242),
|
||||
remove: vi.fn(),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions });
|
||||
@@ -140,7 +181,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create,
|
||||
remove: vi.fn(),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions });
|
||||
@@ -158,7 +199,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 4242),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
await sessions.start("aa11");
|
||||
@@ -172,6 +213,47 @@ describe("ToolDispatcher", () => {
|
||||
expect(sent[0]).toEqual({ id: "r-1", result: {} });
|
||||
});
|
||||
|
||||
it("wires the production tab APIs into tool.session_stop so a surviving user tab releases the window", async () => {
|
||||
// Regression guard for the deps that session_stop reads directly: when
|
||||
// `tabManagement.tabs` / `tabsQuery` are not injected by the dispatcher,
|
||||
// the agent-tab cleanup and the release decision silently no-op and the
|
||||
// Agent Window is closed even though a user tab is still open (issue #57).
|
||||
const removeTab = vi.fn(async () => {});
|
||||
const closeWindow = vi.fn(async () => {});
|
||||
// After cleanup the window still holds one tab (id 99) that is neither
|
||||
// the home tab nor an agent-created tab — i.e. a genuine user tab.
|
||||
vi.stubGlobal("chrome", {
|
||||
tabs: {
|
||||
remove: removeTab,
|
||||
query: vi.fn(async () => [{ id: 99, windowId: 4242, active: true }]),
|
||||
},
|
||||
});
|
||||
const { transport, sent, deliver } = fakeTransport();
|
||||
const sessions = new SessionManager({
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 4242),
|
||||
remove: closeWindow,
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
await sessions.start("aa11");
|
||||
// Register the exact tab id returned by the agent's creation path.
|
||||
sessions.get("aa11")?.agentCreatedTabs.add(7);
|
||||
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions });
|
||||
dispatcher.start();
|
||||
|
||||
deliver(makeRequest("tool.session_stop", { session_id: "aa11" }));
|
||||
await flushMicrotasks();
|
||||
|
||||
// The agent tab is closed through the real chrome.tabs surface...
|
||||
expect(removeTab).toHaveBeenCalledWith(7);
|
||||
// ...and the window is released rather than closed, so the user's tab survives.
|
||||
expect(closeWindow).not.toHaveBeenCalled();
|
||||
expect(sessions.has("aa11")).toBe(false);
|
||||
expect(sent[0]).toEqual({ id: "r-1", result: { window_released: true } });
|
||||
});
|
||||
|
||||
it("routes tool.console through the CDP console buffer", async () => {
|
||||
vi.stubGlobal("chrome", {
|
||||
tabs: {
|
||||
@@ -183,7 +265,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 4242),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
await sessions.start("aa11");
|
||||
@@ -246,7 +328,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 4242),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
const ctx = await sessions.start("aa11");
|
||||
@@ -339,7 +421,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 4242),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
await sessions.start("aa11");
|
||||
@@ -375,7 +457,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 4242),
|
||||
remove,
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
await sessions.start("aa11");
|
||||
@@ -423,7 +505,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 1),
|
||||
remove: vi.fn(),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions });
|
||||
@@ -443,7 +525,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 1),
|
||||
remove: vi.fn(),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions });
|
||||
@@ -467,7 +549,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 1),
|
||||
remove: vi.fn(),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions });
|
||||
@@ -484,7 +566,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 1),
|
||||
remove: vi.fn(),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions });
|
||||
@@ -501,7 +583,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 1),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
const onSessionsChanged = vi.fn();
|
||||
@@ -518,16 +600,32 @@ describe("ToolDispatcher", () => {
|
||||
});
|
||||
|
||||
it("invokes onBrowserControlResumed for browser-control tools but not passive reads", async () => {
|
||||
vi.stubGlobal("chrome", {
|
||||
tabs: {
|
||||
create: vi.fn(async (props: chrome.tabs.CreateProperties) => ({
|
||||
id: 7,
|
||||
windowId: props.windowId,
|
||||
url: props.url,
|
||||
active: props.active,
|
||||
})),
|
||||
},
|
||||
});
|
||||
const { transport, sent, deliver } = fakeTransport();
|
||||
const sessions = new SessionManager({
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 1),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
const onBrowserControlResumed = vi.fn();
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions, onBrowserControlResumed });
|
||||
const onAgentTabClaimed = vi.fn();
|
||||
const dispatcher = new ToolDispatcher({
|
||||
transport,
|
||||
sessions,
|
||||
onBrowserControlResumed,
|
||||
onAgentTabClaimed,
|
||||
});
|
||||
dispatcher.start();
|
||||
|
||||
deliver(makeRequest("tool.session_start", { session_id: "aa11" }));
|
||||
@@ -544,6 +642,7 @@ describe("ToolDispatcher", () => {
|
||||
});
|
||||
await flushMicrotasks();
|
||||
expect(onBrowserControlResumed).toHaveBeenCalledWith("aa11");
|
||||
expect(onAgentTabClaimed).toHaveBeenCalledWith(7, 1);
|
||||
});
|
||||
|
||||
it("reasserts remembered hover before follow-up work and releases only after actions", async () => {
|
||||
@@ -557,7 +656,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 1),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
const order: string[] = [];
|
||||
@@ -639,7 +738,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 1),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
const cdp = {
|
||||
@@ -689,6 +788,135 @@ describe("ToolDispatcher", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ tab_id: 7, code: "not_found" },
|
||||
{ tab_id: undefined, code: "invalid_params" },
|
||||
])("keeps hover when tab_return is rejected with $code", async ({ tab_id, code }) => {
|
||||
const sendMessage = vi.fn(async () => undefined);
|
||||
vi.stubGlobal("chrome", {
|
||||
tabs: {
|
||||
sendMessage,
|
||||
get: vi.fn(async () => ({ id: 7, windowId: 100, active: true })),
|
||||
query: vi.fn(async () => [{ id: 7, windowId: 100, active: true }]),
|
||||
},
|
||||
});
|
||||
const { transport, sent, deliver } = fakeTransport();
|
||||
const sessions = new SessionManager({
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 100),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
await sessions.start("aa11");
|
||||
const cdp = {
|
||||
send: vi.fn(async <T>() => ({}) as T),
|
||||
detachSession: vi.fn(async () => {}),
|
||||
releaseSessionTab: vi.fn(async () => {}),
|
||||
};
|
||||
const dispatcher = new ToolDispatcher({
|
||||
transport,
|
||||
sessions,
|
||||
cdp: cdp as unknown as TestDispatcherCdp,
|
||||
});
|
||||
const helpers = dispatcher as unknown as {
|
||||
rememberHover: (sessionId: string, result: unknown) => unknown;
|
||||
setHoverBypass: (sessionId: string, tabId: number, enabled: boolean) => Promise<void>;
|
||||
withHoverReassert: <T>(
|
||||
params: { session_id: string; tab_id?: number },
|
||||
work: () => Promise<T>,
|
||||
) => Promise<T>;
|
||||
};
|
||||
helpers.rememberHover("aa11", { tab_id: 7, x: 10, y: 20 });
|
||||
await helpers.setHoverBypass("aa11", 7, true);
|
||||
dispatcher.start();
|
||||
deliver(makeRequest("tool.tab_return", { session_id: "aa11", tab_id }));
|
||||
await vi.waitFor(() => expect(sent).toHaveLength(1));
|
||||
|
||||
expect(sent[0]).toMatchObject({ error: { code } });
|
||||
expect(cdp.releaseSessionTab).not.toHaveBeenCalled();
|
||||
expect(sendMessage).not.toHaveBeenCalledWith(7, expect.objectContaining({ enabled: false }));
|
||||
await helpers.withHoverReassert({ session_id: "aa11", tab_id: 7 }, async () => ({}));
|
||||
expect(cdp.send).toHaveBeenCalledExactlyOnceWith(7, "Input.dispatchMouseEvent", {
|
||||
type: "mouseMoved",
|
||||
x: 10,
|
||||
y: 20,
|
||||
});
|
||||
dispatcher.stop();
|
||||
});
|
||||
|
||||
it("releases the returned tab's hover before moving it and leaves other tabs alone", async () => {
|
||||
const sendMessage = vi.fn(async () => undefined);
|
||||
const move = vi.fn(async () => {
|
||||
expect(sendMessage).toHaveBeenCalledWith(7, expect.objectContaining({ enabled: false }));
|
||||
return { id: 7, windowId: 200, index: 4 };
|
||||
});
|
||||
vi.stubGlobal("chrome", {
|
||||
tabs: { sendMessage, move },
|
||||
windows: { get: vi.fn(async () => ({ id: 200 })) },
|
||||
});
|
||||
const { transport, sent, deliver } = fakeTransport();
|
||||
const sessions = new SessionManager({
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 100),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
const ctx = await sessions.start("aa11");
|
||||
ctx.borrowedTabs.set(7, { tabId: 7, originalWindowId: 200, originalIndex: 4 });
|
||||
const cdp = {
|
||||
send: vi.fn(async <T>() => ({}) as T),
|
||||
detachSession: vi.fn(async () => {}),
|
||||
releaseSessionTab: vi.fn(async () => {}),
|
||||
};
|
||||
const dispatcher = new ToolDispatcher({
|
||||
transport,
|
||||
sessions,
|
||||
cdp: cdp as unknown as TestDispatcherCdp,
|
||||
});
|
||||
const helpers = dispatcher as unknown as {
|
||||
rememberHover: (sessionId: string, result: unknown) => unknown;
|
||||
setHoverBypass: (sessionId: string, tabId: number, enabled: boolean) => Promise<void>;
|
||||
withHoverReassert: <T>(
|
||||
params: { session_id: string; tab_id?: number },
|
||||
work: () => Promise<T>,
|
||||
) => Promise<T>;
|
||||
};
|
||||
for (const [sessionId, tabId] of [
|
||||
["aa11", 7],
|
||||
["aa11", 8],
|
||||
["bb22", 9],
|
||||
] as const) {
|
||||
helpers.rememberHover(sessionId, { tab_id: tabId, x: 10, y: 20 });
|
||||
await helpers.setHoverBypass(sessionId, tabId, true);
|
||||
}
|
||||
dispatcher.start();
|
||||
deliver(makeRequest("tool.tab_return", { session_id: "aa11", tab_id: 7 }));
|
||||
await vi.waitFor(() => expect(sent).toHaveLength(1));
|
||||
|
||||
expect(sent[0]).toMatchObject({ result: { tab_id: 7, returned_to_window_id: 200 } });
|
||||
expect(cdp.releaseSessionTab).toHaveBeenCalledExactlyOnceWith("aa11", 7);
|
||||
await helpers.withHoverReassert({ session_id: "aa11", tab_id: 7 }, async () => ({}));
|
||||
expect(cdp.send).not.toHaveBeenCalled();
|
||||
for (const [sessionId, tabId] of [
|
||||
["aa11", 8],
|
||||
["bb22", 9],
|
||||
] as const) {
|
||||
expect(sendMessage).not.toHaveBeenCalledWith(
|
||||
tabId,
|
||||
expect.objectContaining({ enabled: false }),
|
||||
);
|
||||
await helpers.withHoverReassert({ session_id: sessionId, tab_id: tabId }, async () => ({}));
|
||||
expect(cdp.send).toHaveBeenLastCalledWith(tabId, "Input.dispatchMouseEvent", {
|
||||
type: "mouseMoved",
|
||||
x: 10,
|
||||
y: 20,
|
||||
});
|
||||
}
|
||||
dispatcher.stop();
|
||||
});
|
||||
|
||||
it("transfers hover overlay bypass ownership between sessions", async () => {
|
||||
vi.stubGlobal("chrome", {
|
||||
tabs: {
|
||||
@@ -700,7 +928,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 1),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions });
|
||||
@@ -737,7 +965,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 4242),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
await sessions.start("aa11");
|
||||
@@ -789,7 +1017,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 1),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions });
|
||||
@@ -823,7 +1051,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 4242),
|
||||
remove: vi.fn(),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
transport.send = () => {
|
||||
@@ -845,7 +1073,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 5555),
|
||||
remove,
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
transport.send = () => {
|
||||
@@ -873,7 +1101,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(() => createPromise),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions });
|
||||
@@ -924,7 +1152,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 1),
|
||||
remove: vi.fn(),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions });
|
||||
@@ -945,7 +1173,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(() => createPromise),
|
||||
remove: vi.fn(),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions });
|
||||
|
||||
@@ -20,7 +20,7 @@ function fakeAgentWindow(ids: number[]) {
|
||||
return id;
|
||||
}),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { RpcError } from "@/transport/types";
|
||||
import { cdpError, classifyCdpError } from "../errors";
|
||||
|
||||
const denied = "Cannot access a chrome-extension:// URL of different extension";
|
||||
|
||||
describe("CDP extension access errors", () => {
|
||||
it("classifies both Chrome Error instances and string failures", () => {
|
||||
for (const error of [new Error(denied), denied]) {
|
||||
expect(cdpError(error)).toEqual({
|
||||
code: "cdp_failed",
|
||||
message: denied,
|
||||
data: { reason: "cdp_extension_access_denied" },
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
it("preserves transfer outcome and cleanup instructions when CDP was denied", () => {
|
||||
const error: RpcError = {
|
||||
code: "cdp_failed",
|
||||
message: denied,
|
||||
data: {
|
||||
reason: "transfer_outcome_unknown",
|
||||
effect_state: "unknown",
|
||||
cleanup_state: "failed",
|
||||
},
|
||||
};
|
||||
expect(classifyCdpError(error)).toBe(error);
|
||||
});
|
||||
|
||||
it("leaves other error codes and unrelated debugger failures unchanged", () => {
|
||||
for (const error of [
|
||||
{ code: "permission_denied", message: denied },
|
||||
{ code: "cdp_failed", message: "Another debugger is already attached" },
|
||||
] satisfies RpcError[]) {
|
||||
expect(classifyCdpError(error)).toBe(error);
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -12,7 +12,7 @@ function fakeAgentWindow(ids: number[]) {
|
||||
return id;
|
||||
}),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@ import { describe, expect, it, vi } from "vitest";
|
||||
import { SessionManager } from "@/session-manager/manager";
|
||||
import { type DownloadsApi, handleDownload } from "../download";
|
||||
import { captureBrowserDownload } from "../download-capture";
|
||||
import { uploadThroughFileDrop } from "../file-drop-transaction";
|
||||
import { uploadThroughActivatedFileInput } from "../file-input-transaction";
|
||||
import type { ResolvedActionTarget } from "../interaction";
|
||||
import type { CdpRunner } from "../shared";
|
||||
@@ -12,7 +13,7 @@ function sessions() {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 100),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -37,16 +38,20 @@ function fakeEvent<T extends (...args: never[]) => unknown>() {
|
||||
};
|
||||
}
|
||||
|
||||
function actionTarget(frameId?: string): ResolvedActionTarget {
|
||||
function actionTarget(frameId?: string, sessionId?: string): ResolvedActionTarget {
|
||||
return {
|
||||
tab: { tabId: 4, windowId: 100, active: true },
|
||||
backendNodeId: 123,
|
||||
cdpTarget: { tabId: 4 },
|
||||
cdpTarget: { tabId: 4, ...(sessionId ? { sessionId } : {}) },
|
||||
...(frameId ? { frameId } : {}),
|
||||
usedRef: "e3",
|
||||
};
|
||||
}
|
||||
|
||||
function withoutOverlay<T>(operation: () => Promise<T>): Promise<T> {
|
||||
return operation();
|
||||
}
|
||||
|
||||
function uploadCdp(
|
||||
options: {
|
||||
inputCount?: number;
|
||||
@@ -59,6 +64,7 @@ function uploadCdp(
|
||||
let cdpEvent: Parameters<NonNullable<CdpRunner["onEvent"]>>[0] | undefined;
|
||||
const send = vi.fn(async (_tabId: number, method: string, params?: object) => {
|
||||
calls.push({ method, params });
|
||||
if (method === "DOM.scrollIntoViewIfNeeded") return {};
|
||||
if (method === "Page.setInterceptFileChooserDialog") return {};
|
||||
if (method === "Page.getLayoutMetrics")
|
||||
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
|
||||
@@ -110,6 +116,50 @@ function uploadCdp(
|
||||
};
|
||||
}
|
||||
|
||||
function dropCdp(
|
||||
options: {
|
||||
targetContainsPoint?: boolean;
|
||||
dispatchError?: (type: string) => Error | Promise<never> | undefined;
|
||||
onCall?: (method: string) => void;
|
||||
} = {},
|
||||
) {
|
||||
const calls: Array<{ method: string; params?: object }> = [];
|
||||
const detach = vi.fn(async () => {});
|
||||
const send = vi.fn(async (_tabId: number, method: string, params?: object) => {
|
||||
calls.push({ method, params });
|
||||
options.onCall?.(method);
|
||||
if (method === "DOM.scrollIntoViewIfNeeded") return {};
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
|
||||
}
|
||||
if (method === "DOM.getContentQuads") {
|
||||
return { quads: [[10, 20, 210, 20, 210, 120, 10, 120]] };
|
||||
}
|
||||
if (method === "DOM.getNodeForLocation") return { backendNodeId: 124 };
|
||||
if (method === "DOM.resolveNode") {
|
||||
const id = (params as { backendNodeId?: number }).backendNodeId;
|
||||
return { object: { objectId: id === 123 ? "drop-target" : "hit-target" } };
|
||||
}
|
||||
if (method === "Runtime.callFunctionOn") {
|
||||
return { result: { value: options.targetContainsPoint ?? true } };
|
||||
}
|
||||
if (method === "Input.dispatchDragEvent") {
|
||||
const type = (params as { type?: string }).type ?? "";
|
||||
const error = options.dispatchError?.(type);
|
||||
if (error instanceof Promise) return error;
|
||||
if (error) throw error;
|
||||
return {};
|
||||
}
|
||||
if (method === "Runtime.releaseObjectGroup") return {};
|
||||
throw new Error(`unexpected drop CDP command ${method}`);
|
||||
});
|
||||
const cdp: CdpRunner = {
|
||||
send: send as unknown as CdpRunner["send"],
|
||||
detach,
|
||||
};
|
||||
return { calls, cdp, detach };
|
||||
}
|
||||
|
||||
describe("file transfer tools", () => {
|
||||
it("captures the file input activated by the requested click and injects only staged paths", async () => {
|
||||
const manager = sessions();
|
||||
@@ -246,104 +296,6 @@ describe("file transfer tools", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("does not dispatch browser work when upload is already cancelled", async () => {
|
||||
const controller = new AbortController();
|
||||
controller.abort();
|
||||
const { cdp, calls } = uploadCdp();
|
||||
|
||||
const result = await uploadThroughActivatedFileInput({
|
||||
cdp,
|
||||
actionTarget: actionTarget(),
|
||||
files: ["/private/stage/one"],
|
||||
timeoutMs: 100,
|
||||
signal: controller.signal,
|
||||
trigger: vi.fn(),
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({
|
||||
code: "cancelled",
|
||||
data: { effect_state: "none", phase: "arm_interception" },
|
||||
});
|
||||
expect(calls).not.toContainEqual({
|
||||
method: "Page.setInterceptFileChooserDialog",
|
||||
params: { enabled: true },
|
||||
});
|
||||
expect(calls.some((call) => call.method === "DOM.setFileInputFiles")).toBe(false);
|
||||
});
|
||||
|
||||
it("stops before file assignment when cancellation interrupts input resolution", async () => {
|
||||
const controller = new AbortController();
|
||||
const fixture = uploadCdp();
|
||||
const originalSend = fixture.cdp.send;
|
||||
let markProbeStarted!: () => void;
|
||||
const probeStarted = new Promise<void>((resolve) => {
|
||||
markProbeStarted = resolve;
|
||||
});
|
||||
fixture.cdp.send = vi.fn((tabId: number, method: string, params?: object) => {
|
||||
const declaration = (params as { functionDeclaration?: string } | undefined)
|
||||
?.functionDeclaration;
|
||||
if (
|
||||
method === "Runtime.callFunctionOn" &&
|
||||
declaration?.includes("count: state.inputs.length")
|
||||
) {
|
||||
markProbeStarted();
|
||||
return new Promise<never>(() => {});
|
||||
}
|
||||
return originalSend(tabId, method, params);
|
||||
}) as CdpRunner["send"];
|
||||
|
||||
const transaction = uploadThroughActivatedFileInput({
|
||||
cdp: fixture.cdp,
|
||||
actionTarget: actionTarget(),
|
||||
files: ["/private/stage/one"],
|
||||
timeoutMs: 1_000,
|
||||
signal: controller.signal,
|
||||
trigger: async () => ({ tab_id: 4, x: 10, y: 10 }),
|
||||
});
|
||||
await probeStarted;
|
||||
controller.abort();
|
||||
const result = await transaction;
|
||||
|
||||
expect(result).toMatchObject({
|
||||
code: "cancelled",
|
||||
data: { effect_state: "none", phase: "resolve_input" },
|
||||
});
|
||||
expect(fixture.calls.some((call) => call.method === "DOM.setFileInputFiles")).toBe(false);
|
||||
});
|
||||
|
||||
it("disarms chooser interception when its enable response times out", async () => {
|
||||
const calls: Array<{ method: string; params?: object }> = [];
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async (_tabId: number, method: string, params?: object) => {
|
||||
calls.push({ method, params });
|
||||
if (
|
||||
method === "Page.setInterceptFileChooserDialog" &&
|
||||
(params as { enabled?: boolean }).enabled
|
||||
) {
|
||||
return new Promise<never>(() => {});
|
||||
}
|
||||
return {};
|
||||
}) as CdpRunner["send"],
|
||||
};
|
||||
|
||||
const result = await uploadThroughActivatedFileInput({
|
||||
cdp,
|
||||
actionTarget: actionTarget(),
|
||||
files: ["/private/stage/one"],
|
||||
timeoutMs: 5,
|
||||
trigger: vi.fn(),
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({
|
||||
code: "timeout",
|
||||
data: { effect_state: "none", phase: "arm_interception" },
|
||||
});
|
||||
expect(calls).toContainEqual({
|
||||
method: "Page.setInterceptFileChooserDialog",
|
||||
params: { enabled: false, cancel: true },
|
||||
});
|
||||
});
|
||||
|
||||
it("marks a timed-out file assignment unknown and detaches browser state", async () => {
|
||||
const fixture = uploadCdp();
|
||||
const originalSend = fixture.cdp.send;
|
||||
@@ -369,6 +321,236 @@ describe("file transfer tools", () => {
|
||||
expect(detach).toHaveBeenCalledWith(4);
|
||||
});
|
||||
|
||||
it("delivers staged files through one explicit native drop without clicking", async () => {
|
||||
const manager = sessions();
|
||||
const ctx = await manager.start("s1");
|
||||
ctx.refStore.set("e3", 123, { tabId: 4 });
|
||||
const lifecycle: string[] = [];
|
||||
const { cdp, calls } = dropCdp({ onCall: (method) => lifecycle.push(method) });
|
||||
const overlayPhases: string[] = [];
|
||||
|
||||
const result = await handleUpload(
|
||||
manager,
|
||||
{
|
||||
session_id: "s1",
|
||||
ref: "@e3",
|
||||
mode: "drop",
|
||||
files: [{ transfer_id: "tr_1", name: "one.png", staged_path: "/stage/one.png" }],
|
||||
},
|
||||
{
|
||||
cdp,
|
||||
tabsApi: tabsApi(),
|
||||
sendToTab: vi.fn(async (_tabId, message) => {
|
||||
overlayPhases.push(message.phase);
|
||||
lifecycle.push(`overlay:${message.phase}`);
|
||||
}),
|
||||
},
|
||||
);
|
||||
|
||||
expect(result).toMatchObject({ tab_id: 4, used_ref: "e3", file_names: ["one.png"] });
|
||||
expect(calls.filter((call) => call.method === "Input.dispatchDragEvent")).toEqual([
|
||||
{
|
||||
method: "Input.dispatchDragEvent",
|
||||
params: {
|
||||
type: "dragEnter",
|
||||
x: 110,
|
||||
y: 70,
|
||||
data: { items: [], files: ["/stage/one.png"], dragOperationsMask: 1 },
|
||||
},
|
||||
},
|
||||
{
|
||||
method: "Input.dispatchDragEvent",
|
||||
params: {
|
||||
type: "dragOver",
|
||||
x: 110,
|
||||
y: 70,
|
||||
data: { items: [], files: ["/stage/one.png"], dragOperationsMask: 1 },
|
||||
},
|
||||
},
|
||||
{
|
||||
method: "Input.dispatchDragEvent",
|
||||
params: {
|
||||
type: "drop",
|
||||
x: 110,
|
||||
y: 70,
|
||||
data: { items: [], files: ["/stage/one.png"], dragOperationsMask: 1 },
|
||||
},
|
||||
},
|
||||
]);
|
||||
expect(calls.some((call) => call.method === "Input.dispatchMouseEvent")).toBe(false);
|
||||
expect(overlayPhases).toEqual(["begin", "end"]);
|
||||
expect(lifecycle.indexOf("DOM.getContentQuads")).toBeLessThan(
|
||||
lifecycle.indexOf("overlay:begin"),
|
||||
);
|
||||
expect(lifecycle.indexOf("overlay:begin")).toBeLessThan(
|
||||
lifecycle.indexOf("DOM.getNodeForLocation"),
|
||||
);
|
||||
expect(lifecycle.lastIndexOf("Input.dispatchDragEvent")).toBeLessThan(
|
||||
lifecycle.indexOf("overlay:end"),
|
||||
);
|
||||
expect(lifecycle.indexOf("overlay:end")).toBeLessThan(
|
||||
lifecycle.indexOf("Runtime.releaseObjectGroup"),
|
||||
);
|
||||
});
|
||||
|
||||
it("fails closed before dragging when another element owns the action point", async () => {
|
||||
const { cdp, calls } = dropCdp({ targetContainsPoint: false });
|
||||
const result = await uploadThroughFileDrop({
|
||||
cdp,
|
||||
actionTarget: actionTarget(),
|
||||
files: ["/stage/one.png"],
|
||||
timeoutMs: 100,
|
||||
withOverlayHidden: withoutOverlay,
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({
|
||||
code: "permission_denied",
|
||||
data: {
|
||||
reason: "file_drop_target_unavailable",
|
||||
mechanism: "drop",
|
||||
effect_state: "none",
|
||||
phase: "resolve_target",
|
||||
},
|
||||
});
|
||||
expect(calls.some((call) => call.method === "Input.dispatchDragEvent")).toBe(false);
|
||||
});
|
||||
|
||||
it("dispatches an OOPIF file drop in the child target's local coordinates", async () => {
|
||||
const rootCalls: Array<{ method: string; params?: object }> = [];
|
||||
const childCalls: Array<{ method: string; params?: object }> = [];
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async (_tabId, method, params) => {
|
||||
rootCalls.push({ method, params });
|
||||
if (method === "DOM.scrollIntoViewIfNeeded") return {};
|
||||
if (method === "DOM.getBoxModel") {
|
||||
return { model: { content: [100, 100, 300, 100, 300, 200, 100, 200] } };
|
||||
}
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 800, clientHeight: 600 } };
|
||||
}
|
||||
throw new Error(`unexpected root command ${method}`);
|
||||
}) as CdpRunner["send"],
|
||||
sendToTarget: vi.fn(async (_target, method, params) => {
|
||||
childCalls.push({ method, params });
|
||||
if (method === "DOM.scrollIntoViewIfNeeded") return {};
|
||||
if (method === "DOM.getContentQuads") {
|
||||
return { quads: [[10, 20, 210, 20, 210, 120, 10, 120]] };
|
||||
}
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 400, clientHeight: 300 } };
|
||||
}
|
||||
if (method === "DOM.getNodeForLocation") {
|
||||
return { backendNodeId: 124, frameId: "child" };
|
||||
}
|
||||
if (method === "DOM.resolveNode") {
|
||||
const id = (params as { backendNodeId?: number }).backendNodeId;
|
||||
return { object: { objectId: id === 123 ? "drop-target" : "hit-target" } };
|
||||
}
|
||||
if (method === "Runtime.callFunctionOn") return { result: { value: true } };
|
||||
if (method === "Input.dispatchDragEvent" || method === "Runtime.releaseObjectGroup") {
|
||||
return {};
|
||||
}
|
||||
throw new Error(`unexpected child command ${method}`);
|
||||
}) as CdpRunner["sendToTarget"],
|
||||
getFrameGraph: vi.fn(async () => ({
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "child",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 10,
|
||||
target: { tabId: 4, sessionId: "child-session" },
|
||||
},
|
||||
],
|
||||
})),
|
||||
};
|
||||
|
||||
const result = await uploadThroughFileDrop({
|
||||
cdp,
|
||||
actionTarget: actionTarget("child", "child-session"),
|
||||
files: ["/stage/one.png"],
|
||||
timeoutMs: 100,
|
||||
withOverlayHidden: withoutOverlay,
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({ x: 110, y: 70 });
|
||||
expect(
|
||||
childCalls
|
||||
.filter((call) => call.method === "Input.dispatchDragEvent")
|
||||
.map((call) => call.params),
|
||||
).toEqual([
|
||||
expect.objectContaining({ type: "dragEnter", x: 110, y: 70 }),
|
||||
expect.objectContaining({ type: "dragOver", x: 110, y: 70 }),
|
||||
expect.objectContaining({ type: "drop", x: 110, y: 70 }),
|
||||
]);
|
||||
expect(rootCalls.some((call) => call.method === "Input.dispatchDragEvent")).toBe(false);
|
||||
});
|
||||
|
||||
it("reports a missing native drag command as unsupported before commit", async () => {
|
||||
const { cdp } = dropCdp({
|
||||
dispatchError: (type) =>
|
||||
type === "dragEnter" ? new Error("'Input.dispatchDragEvent' wasn't found") : undefined,
|
||||
});
|
||||
const result = await uploadThroughFileDrop({
|
||||
cdp,
|
||||
actionTarget: actionTarget(),
|
||||
files: ["/stage/one.png"],
|
||||
timeoutMs: 100,
|
||||
withOverlayHidden: withoutOverlay,
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({
|
||||
code: "unsupported",
|
||||
data: {
|
||||
reason: "upload_mechanism_unsupported",
|
||||
effect_state: "none",
|
||||
phase: "drag_enter",
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("cancels an unresolved drag before returning a pre-commit timeout", async () => {
|
||||
const { cdp, calls } = dropCdp({
|
||||
dispatchError: (type) => (type === "dragEnter" ? new Promise<never>(() => {}) : undefined),
|
||||
});
|
||||
const result = await uploadThroughFileDrop({
|
||||
cdp,
|
||||
actionTarget: actionTarget(),
|
||||
files: ["/stage/one.png"],
|
||||
timeoutMs: 10,
|
||||
withOverlayHidden: withoutOverlay,
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({
|
||||
code: "timeout",
|
||||
data: { reason: "file_drop_failed", effect_state: "none", phase: "drag_enter" },
|
||||
});
|
||||
expect(calls).toContainEqual({
|
||||
method: "Input.dispatchDragEvent",
|
||||
params: expect.objectContaining({ type: "dragCancel" }),
|
||||
});
|
||||
});
|
||||
|
||||
it("marks an unresolved drop unknown and detaches browser state", async () => {
|
||||
const { cdp, detach } = dropCdp({
|
||||
dispatchError: (type) => (type === "drop" ? new Promise<never>(() => {}) : undefined),
|
||||
});
|
||||
const result = await uploadThroughFileDrop({
|
||||
cdp,
|
||||
actionTarget: actionTarget(),
|
||||
files: ["/stage/one.png"],
|
||||
timeoutMs: 10,
|
||||
withOverlayHidden: withoutOverlay,
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({
|
||||
code: "timeout",
|
||||
data: { reason: "file_drop_failed", effect_state: "unknown", phase: "drop" },
|
||||
});
|
||||
expect(detach).toHaveBeenCalledWith(4);
|
||||
});
|
||||
|
||||
it("routes one exact-target download through a browser-relative capability", async () => {
|
||||
const manager = sessions();
|
||||
const ctx = await manager.start("s1");
|
||||
@@ -665,10 +847,7 @@ describe("file transfer tools", () => {
|
||||
expect(downloads.cancel).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each([
|
||||
["cancel resolves", false],
|
||||
["cancel reports that completion won", true],
|
||||
])("reconciles a download that completes while %s", async (_label, cancelThrows) => {
|
||||
it("reconciles a download that completes while cancellation is being requested", async () => {
|
||||
const onCreated = fakeEvent<(item: chrome.downloads.DownloadItem) => void>();
|
||||
const onChanged = fakeEvent<(delta: chrome.downloads.DownloadDelta) => void>();
|
||||
const onDeterminingFilename =
|
||||
@@ -702,7 +881,7 @@ describe("file transfer tools", () => {
|
||||
onDeterminingFilename,
|
||||
search: vi.fn().mockResolvedValueOnce([initial]).mockResolvedValueOnce([complete]),
|
||||
cancel: vi.fn(async () => {
|
||||
if (cancelThrows) throw new Error("download already complete");
|
||||
throw new Error("download already complete");
|
||||
}),
|
||||
removeFile: vi.fn(async () => {}),
|
||||
};
|
||||
|
||||
@@ -0,0 +1,573 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { SessionManager } from "@/session-manager/manager";
|
||||
import type { CdpRunner } from "@/tools/shared";
|
||||
import { handleFill } from "../interaction";
|
||||
|
||||
interface ScriptParams {
|
||||
functionDeclaration: string;
|
||||
arguments?: Array<{ value: unknown }>;
|
||||
}
|
||||
|
||||
async function setup(markup = '<input value="old">') {
|
||||
document.body.innerHTML = markup;
|
||||
const element = document.body.firstElementChild as HTMLInputElement;
|
||||
const manager = new SessionManager({
|
||||
agentWindow: {
|
||||
create: async () => 100,
|
||||
remove: async () => {},
|
||||
ensureActiveTab: async () => 4,
|
||||
},
|
||||
});
|
||||
const ctx = await manager.start("aa11");
|
||||
ctx.refStore.set("e1", 12, { tabId: 4 });
|
||||
const script = vi.fn(async (params: ScriptParams): Promise<unknown> => {
|
||||
try {
|
||||
const fn = new Function(`return (${params.functionDeclaration})`)();
|
||||
return {
|
||||
result: {
|
||||
value: await fn.apply(
|
||||
element,
|
||||
(params.arguments ?? []).map((arg) => arg.value),
|
||||
),
|
||||
},
|
||||
};
|
||||
} catch {
|
||||
return { exceptionDetails: { text: "test page script failed" } };
|
||||
}
|
||||
});
|
||||
// Number/email controls have a browser caret even though their selection
|
||||
// getters return null. Model that caret instead of assuming it is at the end.
|
||||
const hiddenCaret = { start: 0, end: 0 };
|
||||
const moveToEnd = vi.fn(async () => {
|
||||
if (typeof element.selectionStart === "number") {
|
||||
element.setSelectionRange(element.value.length, element.value.length);
|
||||
} else {
|
||||
hiddenCaret.start = hiddenCaret.end = element.value.length;
|
||||
}
|
||||
});
|
||||
const insert = vi.fn(async (text: string) => {
|
||||
const focused = document.activeElement as HTMLInputElement;
|
||||
if (focused.readOnly || focused.disabled) return;
|
||||
const start = focused.selectionStart ?? hiddenCaret.start;
|
||||
const end = focused.selectionEnd ?? hiddenCaret.end;
|
||||
const value = focused.value.slice(0, start) + text + focused.value.slice(end);
|
||||
focused.value = focused.maxLength >= 0 ? value.slice(0, focused.maxLength) : value;
|
||||
hiddenCaret.start = hiddenCaret.end = Math.min(start + text.length, focused.value.length);
|
||||
focused.dispatchEvent(new Event("input", { bubbles: true }));
|
||||
});
|
||||
const release = vi.fn(async () => ({}));
|
||||
const send = vi.fn(async (_tabId: number, method: string, params?: object) => {
|
||||
switch (method) {
|
||||
case "DOM.describeNode":
|
||||
return {
|
||||
node: {
|
||||
nodeName: element.tagName,
|
||||
attributes: Array.from(element.attributes).flatMap((attr) => [attr.name, attr.value]),
|
||||
},
|
||||
};
|
||||
case "DOM.scrollIntoViewIfNeeded":
|
||||
return {};
|
||||
case "DOM.focus":
|
||||
element.focus();
|
||||
return {};
|
||||
case "DOM.resolveNode":
|
||||
return { object: { objectId: "fill-target" } };
|
||||
case "Runtime.callFunctionOn":
|
||||
return script(params as ScriptParams);
|
||||
case "Input.insertText":
|
||||
await insert((params as { text: string }).text);
|
||||
return {};
|
||||
case "Input.dispatchKeyEvent":
|
||||
if ((params as { commands?: string[] }).commands?.includes("moveToEndOfDocument")) {
|
||||
await moveToEnd();
|
||||
}
|
||||
return {};
|
||||
case "Runtime.releaseObject":
|
||||
return release();
|
||||
default:
|
||||
throw new Error(`unexpected ${method}`);
|
||||
}
|
||||
});
|
||||
return {
|
||||
element,
|
||||
hiddenCaret,
|
||||
moveToEnd,
|
||||
script,
|
||||
insert,
|
||||
release,
|
||||
send,
|
||||
fill: (value = "hello", clearBefore = true, signal?: AbortSignal) =>
|
||||
handleFill(
|
||||
manager,
|
||||
{ session_id: "aa11", ref: "e1", value, clear_before: clearBefore },
|
||||
{
|
||||
cdp: { send: send as CdpRunner["send"] },
|
||||
tabsApi: {
|
||||
get: async (id) => ({ id, windowId: 100, active: true }) as chrome.tabs.Tab,
|
||||
query: async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab],
|
||||
},
|
||||
signal,
|
||||
},
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
document.body.innerHTML = "";
|
||||
});
|
||||
|
||||
describe("fill result verification", () => {
|
||||
it.each([
|
||||
"readonly",
|
||||
"disabled",
|
||||
])("preserves a %s field before rejecting it", async (attribute) => {
|
||||
const h = await setup(`<input ${attribute} value="old">`);
|
||||
expect(await h.fill()).toMatchObject({
|
||||
code: "invalid_params",
|
||||
data: { reason: "target_not_fillable" },
|
||||
});
|
||||
expect(h.element.value).toBe("old");
|
||||
expect(h.insert).not.toHaveBeenCalled();
|
||||
expect(h.release).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it.each([
|
||||
"checkbox",
|
||||
"radio",
|
||||
"range",
|
||||
"date",
|
||||
"file",
|
||||
])("rejects an unsupported %s input before clearing", async (type) => {
|
||||
const h = await setup(`<input type="${type}">`);
|
||||
const before = h.element.value;
|
||||
expect(await h.fill()).toMatchObject({
|
||||
code: "invalid_params",
|
||||
data: { reason: "target_not_fillable" },
|
||||
});
|
||||
expect(h.element.value).toBe(before);
|
||||
expect(h.insert).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("rejects an overlong value before clearing", async () => {
|
||||
const h = await setup('<input maxlength="3" value="old">');
|
||||
expect(await h.fill()).toMatchObject({
|
||||
code: "invalid_params",
|
||||
message: /maxlength/,
|
||||
data: { reason: "fill_value_invalid" },
|
||||
});
|
||||
expect(h.element.value).toBe("old");
|
||||
expect(h.insert).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each([
|
||||
true,
|
||||
false,
|
||||
])("classifies a DOM.focus rejection on a disabled=%s target", async (disabled) => {
|
||||
const h = await setup(`<input ${disabled ? "disabled" : ""} value="old">`);
|
||||
const send = h.send.getMockImplementation()!;
|
||||
h.send.mockImplementation(async (tabId, method, params) => {
|
||||
if (method === "DOM.focus") throw new Error("Element is not focusable");
|
||||
return send(tabId, method, params);
|
||||
});
|
||||
expect(await h.fill()).toMatchObject({
|
||||
code: disabled ? "invalid_params" : "cdp_failed",
|
||||
data: { reason: disabled ? "target_not_fillable" : "fill_failed" },
|
||||
});
|
||||
expect(h.element.value).toBe("old");
|
||||
expect(h.insert).not.toHaveBeenCalled();
|
||||
expect(h.release).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("fills a background document without requiring foreground browser focus", async () => {
|
||||
const h = await setup();
|
||||
vi.spyOn(document, "hasFocus").mockReturnValue(false);
|
||||
expect(await h.fill()).toMatchObject({ value_length: 5 });
|
||||
expect(h.element.value).toBe("hello");
|
||||
});
|
||||
|
||||
it("restores focus once after clearing without typing into another field", async () => {
|
||||
const h = await setup();
|
||||
const other = document.createElement("input");
|
||||
document.body.append(other);
|
||||
h.element.addEventListener("input", () => other.focus());
|
||||
expect(await h.fill()).toMatchObject({ value_length: 5 });
|
||||
expect(h.element.value).toBe("hello");
|
||||
expect(other.value).toBe("");
|
||||
expect(h.send.mock.calls.filter(([, method]) => method === "DOM.focus")).toHaveLength(2);
|
||||
});
|
||||
|
||||
it.each([
|
||||
"focus",
|
||||
"value",
|
||||
])("rechecks %s after restoring focus, without retrying again", async (action) => {
|
||||
const h = await setup();
|
||||
const other = document.createElement("input");
|
||||
document.body.append(other);
|
||||
h.element.addEventListener(
|
||||
"input",
|
||||
() => {
|
||||
other.focus();
|
||||
h.element.addEventListener("focus", () => {
|
||||
if (action === "focus") queueMicrotask(() => other.focus());
|
||||
else h.element.value = "changed";
|
||||
});
|
||||
},
|
||||
{ once: true },
|
||||
);
|
||||
expect(await h.fill()).toMatchObject({
|
||||
code: "cdp_failed",
|
||||
data: { reason: action === "focus" ? "fill_focus_lost" : "fill_target_changed" },
|
||||
});
|
||||
expect(other.value).toBe("");
|
||||
expect(h.insert).not.toHaveBeenCalled();
|
||||
expect(h.send.mock.calls.filter(([, method]) => method === "DOM.focus")).toHaveLength(2);
|
||||
});
|
||||
|
||||
it.each([
|
||||
"remove",
|
||||
"readonly",
|
||||
"restore",
|
||||
"microtask-restore",
|
||||
])("stops when clearing causes the page to %s the target", async (action) => {
|
||||
const h = await setup();
|
||||
h.element.addEventListener("input", () => {
|
||||
if (action === "remove") h.element.remove();
|
||||
if (action === "readonly") h.element.readOnly = true;
|
||||
if (action === "restore") h.element.value = "restored";
|
||||
if (action === "microtask-restore")
|
||||
queueMicrotask(() => {
|
||||
h.element.value = "restored";
|
||||
});
|
||||
});
|
||||
expect(await h.fill()).toMatchObject({
|
||||
code: "cdp_failed",
|
||||
data: { reason: "fill_target_changed" },
|
||||
});
|
||||
expect(h.insert).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("fails when insertText is acknowledged without changing the value", async () => {
|
||||
const h = await setup();
|
||||
h.insert.mockImplementation(async () => {});
|
||||
expect(await h.fill()).toMatchObject({
|
||||
code: "cdp_failed",
|
||||
data: { reason: "fill_value_mismatch" },
|
||||
});
|
||||
expect(h.release).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("detects truncation when maxlength changes after preparation", async () => {
|
||||
const h = await setup();
|
||||
h.element.addEventListener(
|
||||
"input",
|
||||
() => {
|
||||
h.element.maxLength = 3;
|
||||
},
|
||||
{ once: true },
|
||||
);
|
||||
expect(await h.fill()).toMatchObject({
|
||||
code: "cdp_failed",
|
||||
data: { reason: "fill_value_mismatch" },
|
||||
});
|
||||
expect(h.element.value).toBe("hel");
|
||||
});
|
||||
|
||||
it.each([
|
||||
"input",
|
||||
"change",
|
||||
"microtask-change",
|
||||
])("detects a value reverted by the %s handler", async (event) => {
|
||||
const h = await setup();
|
||||
h.element.addEventListener(event === "microtask-change" ? "change" : event, () => {
|
||||
if (event === "microtask-change")
|
||||
queueMicrotask(() => {
|
||||
h.element.value = "";
|
||||
});
|
||||
else h.element.value = "";
|
||||
});
|
||||
expect(await h.fill()).toMatchObject({ code: "cdp_failed", message: /expected value/ });
|
||||
});
|
||||
|
||||
it("compares content rather than length without including field values in the error", async () => {
|
||||
const h = await setup('<input type="password">');
|
||||
h.element.addEventListener("change", () => {
|
||||
h.element.value = "other-secret";
|
||||
});
|
||||
const result = await h.fill("given-secret");
|
||||
expect(result).toMatchObject({ code: "cdp_failed", message: /expected value/ });
|
||||
expect(JSON.stringify(result)).not.toMatch(/given-secret|other-secret/);
|
||||
});
|
||||
|
||||
it("does not report success for a target replaced by a change handler", async () => {
|
||||
const h = await setup();
|
||||
h.element.addEventListener("change", () => h.element.replaceWith(h.element.cloneNode()));
|
||||
expect(await h.fill()).toMatchObject({
|
||||
code: "cdp_failed",
|
||||
data: { reason: "fill_target_changed" },
|
||||
});
|
||||
});
|
||||
|
||||
it.each([
|
||||
1, 2, 3, 4, 5,
|
||||
])("rejects script exceptions at phase %s and releases the target", async (phase) => {
|
||||
const h = await setup();
|
||||
vi.spyOn(document, "hasFocus").mockReturnValue(false);
|
||||
const run = h.script.getMockImplementation()!;
|
||||
let count = 0;
|
||||
h.script.mockImplementation(async (params) =>
|
||||
++count === phase ? { exceptionDetails: { text: "page secret" } } : run(params),
|
||||
);
|
||||
const result = await h.fill();
|
||||
expect(result).toMatchObject({ code: "cdp_failed", data: { reason: "fill_failed" } });
|
||||
expect(JSON.stringify(result)).not.toContain("page secret");
|
||||
if (phase <= 3) expect(h.insert).not.toHaveBeenCalled();
|
||||
expect(h.release).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it.each([1, 2, 3, 5])("rejects missing results at phase %s", async (phase) => {
|
||||
const h = await setup();
|
||||
vi.spyOn(document, "hasFocus").mockReturnValue(false);
|
||||
const run = h.script.getMockImplementation()!;
|
||||
let count = 0;
|
||||
h.script.mockImplementation(async (params) =>
|
||||
++count === phase ? { result: { type: "undefined" } } : run(params),
|
||||
);
|
||||
expect(await h.fill()).toMatchObject({ code: "cdp_failed" });
|
||||
if (phase <= 3) expect(h.insert).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each(["input", "change"])("checks state after nested microtasks from %s", async (event) => {
|
||||
const h = await setup();
|
||||
h.element.addEventListener(event, () => {
|
||||
queueMicrotask(() =>
|
||||
queueMicrotask(() => {
|
||||
h.element.value = "reverted";
|
||||
}),
|
||||
);
|
||||
});
|
||||
expect(await h.fill()).toMatchObject({ code: "cdp_failed" });
|
||||
if (event === "input") expect(h.insert).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each([
|
||||
"<input>",
|
||||
'<input type="password">',
|
||||
"<textarea></textarea>",
|
||||
])("verifies successful text in %s", async (markup) => {
|
||||
const h = await setup(markup);
|
||||
expect(await h.fill("你好🙂")).toMatchObject({ value_length: 4 });
|
||||
expect(h.element.value).toBe("你好🙂");
|
||||
expect(h.release).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("verifies append against the old value plus the requested text", async () => {
|
||||
const h = await setup();
|
||||
h.element.focus();
|
||||
h.element.setSelectionRange(3, 3);
|
||||
expect(await h.fill("!", false)).toMatchObject({ value_length: 4 });
|
||||
expect(h.element.value).toBe("old!");
|
||||
});
|
||||
|
||||
it.each([
|
||||
["<textarea></textarea>", "a\r\nb", "a\nb"],
|
||||
["<input>", "a\nb", "a b"],
|
||||
["<input>", "a\r\nb", "a b"],
|
||||
[
|
||||
'<input type="email" multiple>',
|
||||
" a@example.com , b@example.com ",
|
||||
"a@example.com,b@example.com",
|
||||
],
|
||||
])("accepts native normalization for %s", async (markup, requested, normalized) => {
|
||||
if (markup.startsWith("<textarea")) {
|
||||
// Happy DOM does not implement Chrome's textarea line-ending normalization.
|
||||
const setter = Object.getOwnPropertyDescriptor(HTMLTextAreaElement.prototype, "value")!.set!;
|
||||
vi.spyOn(HTMLTextAreaElement.prototype, "value", "set").mockImplementation(function (
|
||||
this: HTMLTextAreaElement,
|
||||
value,
|
||||
) {
|
||||
setter.call(this, value.replace(/\r\n?/g, "\n"));
|
||||
});
|
||||
}
|
||||
const h = await setup(markup);
|
||||
h.insert.mockImplementation(async () => {
|
||||
h.element.value = normalized;
|
||||
});
|
||||
expect(await h.fill(requested)).toMatchObject({ value_length: normalized.length });
|
||||
});
|
||||
|
||||
it("does not count an editable padding break as an extra typed character", async () => {
|
||||
const h = await setup('<div contenteditable="true" tabindex="0"></div>');
|
||||
// These two layout/editability properties are missing in Happy DOM.
|
||||
let innerText = "";
|
||||
Object.defineProperty(h.element, "isContentEditable", { get: () => true });
|
||||
Object.defineProperty(h.element, "innerText", { get: () => innerText });
|
||||
h.insert.mockImplementation(async () => {
|
||||
h.element.innerHTML = "one<div><br></div>";
|
||||
innerText = "one\n\n";
|
||||
});
|
||||
expect(await h.fill("one\n")).toMatchObject({ value_length: 4 });
|
||||
});
|
||||
|
||||
it.each([
|
||||
[0, 0],
|
||||
[1, 1],
|
||||
[0, 3],
|
||||
])("appends after moving selection %s:%s to the end", async (start, end) => {
|
||||
const h = await setup();
|
||||
h.element.focus();
|
||||
h.element.setSelectionRange(start, end);
|
||||
expect(await h.fill("🙂", false)).toMatchObject({ value_length: 5 });
|
||||
expect(h.element.value).toBe("old🙂");
|
||||
});
|
||||
|
||||
it.each(
|
||||
["number", "email"].flatMap((type) =>
|
||||
[
|
||||
[0, 0],
|
||||
[1, 1],
|
||||
[0, 2],
|
||||
].map(([start, end]) => ({ type, start, end })),
|
||||
),
|
||||
)("appends to $type from selection $start:$end without using its unsupported API", async ({
|
||||
type,
|
||||
start,
|
||||
end,
|
||||
}) => {
|
||||
const h = await setup(`<input type="${type}" value="12">`);
|
||||
Object.assign(h.hiddenCaret, { start, end });
|
||||
vi.spyOn(h.element, "selectionStart", "get").mockReturnValue(null);
|
||||
vi.spyOn(h.element, "selectionEnd", "get").mockReturnValue(null);
|
||||
const selection = vi.spyOn(h.element, "setSelectionRange").mockImplementation(() => {
|
||||
throw new DOMException("unsupported", "InvalidStateError");
|
||||
});
|
||||
expect(await h.fill("3", false)).toMatchObject({ value_length: 3 });
|
||||
expect(h.element.value).toBe("123");
|
||||
expect(selection).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each([
|
||||
"number",
|
||||
"email",
|
||||
])("does not report successful %s append when the caret command is ignored", async (type) => {
|
||||
const h = await setup(`<input type="${type}" value="12">`);
|
||||
vi.spyOn(h.element, "selectionStart", "get").mockReturnValue(null);
|
||||
vi.spyOn(h.element, "selectionEnd", "get").mockReturnValue(null);
|
||||
h.moveToEnd.mockResolvedValueOnce(undefined);
|
||||
expect(await h.fill("3", false)).toMatchObject({
|
||||
code: "cdp_failed",
|
||||
data: { reason: "fill_value_mismatch" },
|
||||
});
|
||||
expect(h.element.value).toBe("312");
|
||||
expect(h.insert).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("rejects invalid number text before clearing", async () => {
|
||||
const h = await setup('<input type="number" value="12">');
|
||||
expect(await h.fill("abc")).toMatchObject({
|
||||
code: "invalid_params",
|
||||
data: { reason: "fill_value_invalid" },
|
||||
});
|
||||
expect(h.element.value).toBe("12");
|
||||
expect(h.insert).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("reports page formatting as unconfirmed without retrying the write", async () => {
|
||||
const h = await setup();
|
||||
h.element.addEventListener("change", () => {
|
||||
h.element.value = h.element.value.toUpperCase();
|
||||
});
|
||||
expect(await h.fill()).toMatchObject({
|
||||
code: "cdp_failed",
|
||||
data: { reason: "fill_value_mismatch" },
|
||||
});
|
||||
expect(h.element.value).toBe("HELLO");
|
||||
expect(h.insert).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it.each([
|
||||
["focus", false],
|
||||
["focus", true],
|
||||
["value", false],
|
||||
["value", true],
|
||||
["cancel", false],
|
||||
["cancel", true],
|
||||
["throw", false],
|
||||
["throw", true],
|
||||
] as const)("stops when positioning triggers %s (clear=%s) and releases the key", async (action, clearBefore) => {
|
||||
const h = await setup();
|
||||
vi.spyOn(document, "hasFocus").mockReturnValue(false);
|
||||
const controller = new AbortController();
|
||||
const other = document.createElement("input");
|
||||
document.body.append(other);
|
||||
const send = h.send.getMockImplementation()!;
|
||||
h.send.mockImplementation(async (tabId, method, params) => {
|
||||
const reply = await send(tabId, method, params);
|
||||
if (
|
||||
method === "Input.dispatchKeyEvent" &&
|
||||
(params as { type: string }).type === "rawKeyDown"
|
||||
) {
|
||||
if (action === "focus") other.focus();
|
||||
if (action === "value") h.element.value = "changed";
|
||||
if (action === "cancel") controller.abort();
|
||||
if (action === "throw") throw new Error("dispatch failed");
|
||||
}
|
||||
return reply;
|
||||
});
|
||||
const result = await h.fill("!", clearBefore, controller.signal);
|
||||
expect(result).toMatchObject({ code: action === "cancel" ? "cancelled" : "cdp_failed" });
|
||||
expect(h.insert).not.toHaveBeenCalled();
|
||||
expect(other.value).toBe("");
|
||||
expect(h.send).toHaveBeenCalledWith(
|
||||
4,
|
||||
"Input.dispatchKeyEvent",
|
||||
expect.objectContaining({ type: "keyUp" }),
|
||||
);
|
||||
expect(h.release).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it.each([true, false])("verifies an empty request with clear_before=%s", async (clearBefore) => {
|
||||
const h = await setup();
|
||||
expect(await h.fill("", clearBefore)).toMatchObject({ value_length: clearBefore ? 0 : 3 });
|
||||
expect(h.element.value).toBe(clearBefore ? "" : "old");
|
||||
expect(h.insert).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("cancels after insertion without sending change events", async () => {
|
||||
const h = await setup();
|
||||
const controller = new AbortController();
|
||||
h.insert.mockImplementation(async () => controller.abort());
|
||||
expect(await h.fill("hello", true, controller.signal)).toMatchObject({ code: "cancelled" });
|
||||
expect(h.script).toHaveBeenCalledTimes(2);
|
||||
expect(h.release).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("cancels after resolving the remote object without clearing the field", async () => {
|
||||
const h = await setup();
|
||||
const controller = new AbortController();
|
||||
const send = h.send.getMockImplementation()!;
|
||||
h.send.mockImplementation(async (tabId, method, params) => {
|
||||
const result = await send(tabId, method, params);
|
||||
if (method === "DOM.resolveNode") controller.abort();
|
||||
return result;
|
||||
});
|
||||
expect(await h.fill("hello", true, controller.signal)).toMatchObject({ code: "cancelled" });
|
||||
expect(h.element.value).toBe("old");
|
||||
expect(h.script).not.toHaveBeenCalled();
|
||||
expect(h.release).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("honors cancellation during verification", async () => {
|
||||
const h = await setup();
|
||||
const controller = new AbortController();
|
||||
h.element.addEventListener("change", () => controller.abort());
|
||||
expect(await h.fill("hello", true, controller.signal)).toMatchObject({ code: "cancelled" });
|
||||
expect(h.release).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("keeps the verified outcome when releasing the remote object fails", async () => {
|
||||
const h = await setup();
|
||||
h.release.mockRejectedValueOnce(new Error("target closed"));
|
||||
expect(await h.fill()).toMatchObject({ value_length: 5 });
|
||||
});
|
||||
});
|
||||
@@ -1,5 +1,5 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { resolveFrameProjection, resolveNodeGeometry } from "../frame-geometry";
|
||||
import { resolveNodeGeometry } from "../frame-geometry";
|
||||
import {
|
||||
clipPolygon,
|
||||
polygonArea,
|
||||
@@ -10,6 +10,7 @@ import {
|
||||
rectPolygon,
|
||||
regionBounds,
|
||||
} from "../geometry";
|
||||
import { GeometryContext } from "../geometry/frame-context";
|
||||
import type { CdpRunner } from "../shared";
|
||||
|
||||
describe("frame geometry projection", () => {
|
||||
@@ -82,28 +83,24 @@ describe("frame geometry projection", () => {
|
||||
throw new Error(`unexpected child command ${method}`);
|
||||
}) as CdpRunner["sendToTarget"],
|
||||
};
|
||||
const projection = await resolveFrameProjection(
|
||||
cdp,
|
||||
{
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "same-process-parent",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 10,
|
||||
target: { tabId: 4 },
|
||||
},
|
||||
{
|
||||
frameId: "oopif",
|
||||
parentFrameId: "same-process-parent",
|
||||
ownerBackendNodeId: 20,
|
||||
target: { tabId: 4, sessionId: "oopif-session" },
|
||||
},
|
||||
],
|
||||
},
|
||||
"oopif",
|
||||
);
|
||||
const projection = await new GeometryContext(cdp, 4, {
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "same-process-parent",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 10,
|
||||
target: { tabId: 4 },
|
||||
},
|
||||
{
|
||||
frameId: "oopif",
|
||||
parentFrameId: "same-process-parent",
|
||||
ownerBackendNodeId: 20,
|
||||
target: { tabId: 4, sessionId: "oopif-session" },
|
||||
},
|
||||
],
|
||||
}).targetProjection("oopif");
|
||||
|
||||
expect(projection?.edges).toEqual([
|
||||
{
|
||||
@@ -169,6 +166,7 @@ describe("frame geometry projection", () => {
|
||||
expect(geometry).toMatchObject({
|
||||
topBounds: { x: 100, y: 100, width: 200, height: 100 },
|
||||
actionPoint: { x: 200, y: 150 },
|
||||
targetActionPoint: { x: 100, y: 50 },
|
||||
});
|
||||
});
|
||||
|
||||
@@ -261,6 +259,7 @@ describe("frame geometry projection", () => {
|
||||
expect(geometry).toMatchObject({
|
||||
topBounds: { x: 100, y: 100, width: 200, height: 100 },
|
||||
actionPoint: { x: 200, y: 150 },
|
||||
targetActionPoint: { x: 200, y: 150 },
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -0,0 +1,464 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import type { CdpFrameGraph } from "@/browser-driver/frame-graph";
|
||||
import { resolveNodeGeometry } from "../frame-geometry";
|
||||
import { rectPolygon } from "../geometry";
|
||||
import {
|
||||
projectSnapshotRect,
|
||||
screenshotPageRect,
|
||||
snapshotCoordinates,
|
||||
snapshotViewportRect,
|
||||
} from "../geometry/coordinate-types";
|
||||
import { GeometryContext, snapshotLayoutScale } from "../geometry/frame-context";
|
||||
import type { CdpRunner } from "../shared";
|
||||
|
||||
const target = { tabId: 4 };
|
||||
const graph: CdpFrameGraph = {
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target },
|
||||
{ frameId: "child", parentFrameId: "main", ownerBackendNodeId: 10, target },
|
||||
],
|
||||
};
|
||||
|
||||
function driver() {
|
||||
const send = vi.fn(async (_tab: number, method: string): Promise<object> => {
|
||||
if (method === "Page.getLayoutMetrics")
|
||||
return {
|
||||
visualViewport: { clientWidth: 1000 },
|
||||
cssVisualViewport: { clientWidth: 1000 },
|
||||
cssLayoutViewport: { clientWidth: 1000, clientHeight: 1000 },
|
||||
};
|
||||
if (method === "DOM.getBoxModel")
|
||||
return { model: { content: [52.5, 612.5, 427.5, 612.5, 427.5, 862.5, 52.5, 862.5] } };
|
||||
if (method === "DOM.resolveNode") return { object: { objectId: "owner" } };
|
||||
if (method === "Runtime.callFunctionOn")
|
||||
return { result: { value: { width: 300, height: 200 } } };
|
||||
if (method === "Runtime.releaseObject") return {};
|
||||
throw new Error(method);
|
||||
});
|
||||
return { send: send as CdpRunner["send"], getFrameGraph: vi.fn(async () => graph), calls: send };
|
||||
}
|
||||
|
||||
describe("measurement geometry context", () => {
|
||||
it.each([0.8, 1, 1.25, 2])("normalizes snapshot layout units at scale %s", (scale) => {
|
||||
const measured = snapshotLayoutScale({
|
||||
visualViewport: { clientWidth: 997.25, zoom: 1.1 },
|
||||
cssVisualViewport: { clientWidth: 997.25 / scale, zoom: 1.1 },
|
||||
// Rounded layout dimensions must not supply the scale.
|
||||
layoutViewport: { clientWidth: 997 },
|
||||
cssLayoutViewport: { clientWidth: Math.floor(997.25 / scale) },
|
||||
});
|
||||
expect(measured).toBeCloseTo(scale, 12);
|
||||
const coordinates = snapshotCoordinates(
|
||||
{ scrollOffsetX: 80 * scale, scrollOffsetY: 200 * scale },
|
||||
measured,
|
||||
{ x: 999, y: 999 },
|
||||
);
|
||||
expect(
|
||||
snapshotViewportRect(
|
||||
[160 * scale, 320 * scale, 120 * scale, 40 * scale],
|
||||
{ target },
|
||||
coordinates,
|
||||
)?.rect,
|
||||
).toEqual({ x: 80, y: 120, width: 120, height: 40 });
|
||||
});
|
||||
|
||||
it("uses CSS root scroll only for missing snapshot offsets, without scaling it twice", () => {
|
||||
const coordinates = snapshotCoordinates({ scrollOffsetX: 80 }, 2, { x: 999, y: 100 });
|
||||
expect(snapshotViewportRect([200, 800, 240, 80], { target }, coordinates)?.rect).toEqual({
|
||||
x: 60,
|
||||
y: 300,
|
||||
width: 120,
|
||||
height: 40,
|
||||
});
|
||||
expect(snapshotCoordinates({ scrollOffsetX: 0 }, 2)).toBeNull();
|
||||
expect(
|
||||
snapshotCoordinates({ scrollOffsetX: NaN, scrollOffsetY: 0 }, 2, { x: 0, y: 0 }),
|
||||
).toBeNull();
|
||||
});
|
||||
|
||||
it("does not infer snapshot units from missing, invalid, or CSS-only metrics", () => {
|
||||
expect(snapshotLayoutScale({ cssLayoutViewport: { clientWidth: 800 } })).toBeNull();
|
||||
for (const raw of [0, -1, NaN, Infinity]) {
|
||||
expect(
|
||||
snapshotLayoutScale({
|
||||
visualViewport: { clientWidth: raw },
|
||||
cssVisualViewport: { clientWidth: 800 },
|
||||
}),
|
||||
).toBeNull();
|
||||
}
|
||||
expect(
|
||||
snapshotLayoutScale({
|
||||
visualViewport: { clientWidth: 0, clientHeight: 600 },
|
||||
cssVisualViewport: { clientWidth: 0, clientHeight: 300 },
|
||||
}),
|
||||
).toBe(2);
|
||||
expect(snapshotCoordinates({ scrollOffsetX: 0, scrollOffsetY: 0 }, null)).toBeNull();
|
||||
expect(snapshotViewportRect([10, 20, 120, 40], { target }, null)).toBeNull();
|
||||
});
|
||||
|
||||
it("shares in-flight frame measurements, but never shares them with the next context", async () => {
|
||||
const cdp = driver();
|
||||
const context = new GeometryContext(cdp, 4);
|
||||
const results = await Promise.all(
|
||||
Array.from({ length: 20 }, () => context.targetProjection("child")),
|
||||
);
|
||||
expect(results.every((value) => value === results[0])).toBe(true);
|
||||
expect(cdp.getFrameGraph).toHaveBeenCalledTimes(1);
|
||||
expect(cdp.calls.mock.calls.map(([, method]) => method)).toEqual([
|
||||
"Page.getLayoutMetrics",
|
||||
"DOM.getBoxModel",
|
||||
]);
|
||||
await new GeometryContext(cdp, 4).targetProjection("child");
|
||||
expect(cdp.getFrameGraph).toHaveBeenCalledTimes(2);
|
||||
expect(cdp.calls.mock.calls).toHaveLength(4);
|
||||
});
|
||||
|
||||
it("projects frame-local snapshot CSS bounds through the scaled owner content quad", async () => {
|
||||
const cdp = driver();
|
||||
const context = new GeometryContext(cdp, 4);
|
||||
const source = { target, frameId: "child" };
|
||||
const projection = await context.snapshotProjection(source, 10, [], {
|
||||
width: 1000,
|
||||
height: 1000,
|
||||
});
|
||||
if (projection.status !== "available") throw new Error("expected available projection");
|
||||
const input = snapshotViewportRect([17, 23, 120, 40], source, {
|
||||
layoutUnitsPerCssPixel: 1,
|
||||
scrollCss: { x: 0, y: 0 },
|
||||
});
|
||||
expect(projectSnapshotRect(input!, projection.projection)).toEqual({
|
||||
x: 73.75,
|
||||
y: 641.25,
|
||||
width: 150,
|
||||
height: 50,
|
||||
});
|
||||
const scrolled = snapshotViewportRect([17, 23, 120, 40], source, {
|
||||
layoutUnitsPerCssPixel: 1,
|
||||
scrollCss: { x: 0, y: 10 },
|
||||
});
|
||||
expect(projectSnapshotRect(scrolled!, projection.projection)?.y).toBe(628.75);
|
||||
await context.snapshotProjection(source, 10, [], { width: 1000, height: 1000 });
|
||||
expect(cdp.calls.mock.calls.filter(([, method]) => method === "DOM.resolveNode")).toHaveLength(
|
||||
1,
|
||||
);
|
||||
expect(
|
||||
cdp.calls.mock.calls.filter(([, method]) => method === "Runtime.releaseObject"),
|
||||
).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("falls back only for missing or invalid batch entries and does not reuse sizes across contexts", async () => {
|
||||
const cdp = driver();
|
||||
const original = cdp.send;
|
||||
const calls: string[] = [];
|
||||
cdp.send = async (tabId, method, params) => {
|
||||
calls.push(method);
|
||||
if (method === "Runtime.evaluate")
|
||||
return {
|
||||
result: {
|
||||
deepSerializedValue: {
|
||||
type: "array",
|
||||
value: [
|
||||
{
|
||||
type: "array",
|
||||
value: [
|
||||
{ type: "node", value: { backendNodeId: 10 } },
|
||||
{ type: "string", value: '{"width":300,"height":200}' },
|
||||
],
|
||||
},
|
||||
{
|
||||
type: "array",
|
||||
value: [
|
||||
{ type: "node", value: { backendNodeId: 11 } },
|
||||
{ type: "string", value: '{"width":0,"height":200}' },
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
} as never;
|
||||
if (method === "Runtime.releaseObjectGroup") return {} as never;
|
||||
return original(tabId, method, params);
|
||||
};
|
||||
const context = new GeometryContext(cdp, 4);
|
||||
context.registerSnapshotOwners(target, new Set([10, 11]));
|
||||
const values = await Promise.all(
|
||||
[10, 11].map((id) =>
|
||||
context.snapshotProjection({ target, frameId: String(id) }, id, [], {
|
||||
width: 1000,
|
||||
height: 1000,
|
||||
}),
|
||||
),
|
||||
);
|
||||
expect(values.every((value) => value.status === "available")).toBe(true);
|
||||
expect(calls.filter((method) => method === "DOM.resolveNode")).toHaveLength(1);
|
||||
expect(calls.filter((method) => method === "Runtime.evaluate")).toHaveLength(1);
|
||||
await new GeometryContext(cdp, 4).snapshotProjection({ target, frameId: "child" }, 10, [], {
|
||||
width: 1000,
|
||||
height: 1000,
|
||||
});
|
||||
expect(calls.filter((method) => method === "DOM.resolveNode")).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("waits for batch object cleanup and propagates cancellation", async () => {
|
||||
const cdp = driver();
|
||||
const original = cdp.send;
|
||||
const controller = new AbortController();
|
||||
let release!: () => void;
|
||||
let cleanup!: () => void;
|
||||
const started = new Promise<void>((resolve) => {
|
||||
cleanup = resolve;
|
||||
});
|
||||
cdp.send = async (tabId, method, params) => {
|
||||
if (method === "Runtime.evaluate") return {} as never;
|
||||
if (method === "Runtime.releaseObjectGroup") {
|
||||
cleanup();
|
||||
await new Promise<void>((resolve) => {
|
||||
release = resolve;
|
||||
});
|
||||
return {} as never;
|
||||
}
|
||||
return original(tabId, method, params);
|
||||
};
|
||||
const context = new GeometryContext(cdp, 4, undefined, controller.signal);
|
||||
context.registerSnapshotOwners(target, new Set([10, 11]));
|
||||
let settled = false;
|
||||
const operation = context
|
||||
.snapshotProjection({ target, frameId: "child" }, 10, [], { width: 1000, height: 1000 })
|
||||
.finally(() => {
|
||||
settled = true;
|
||||
});
|
||||
const rejected = expect(operation).rejects.toMatchObject({ name: "AbortError" });
|
||||
await started;
|
||||
controller.abort();
|
||||
expect(settled).toBe(false);
|
||||
release();
|
||||
await rejected;
|
||||
expect(cdp.calls.mock.calls.some(([, method]) => method === "DOM.resolveNode")).toBe(false);
|
||||
});
|
||||
|
||||
it("rejects owner mismatches and invalid numeric bounds", () => {
|
||||
const input = snapshotViewportRect(
|
||||
[30, 50, 120, 40],
|
||||
{ target, frameId: "main" },
|
||||
{ layoutUnitsPerCssPixel: 1, scrollCss: { x: 10, y: 20 } },
|
||||
)!;
|
||||
const projection = {
|
||||
source: { target, frameId: "main" },
|
||||
geometry: { sourceClips: [], edges: [], topViewport: { width: 800, height: 600 } },
|
||||
};
|
||||
expect(projectSnapshotRect(input, projection)).toEqual({
|
||||
x: 20,
|
||||
y: 30,
|
||||
width: 120,
|
||||
height: 40,
|
||||
});
|
||||
expect(
|
||||
projectSnapshotRect(input, { ...projection, source: { target, frameId: "other" } }),
|
||||
).toBeNull();
|
||||
expect(
|
||||
projectSnapshotRect(input, {
|
||||
...projection,
|
||||
source: { target: { tabId: 5 }, frameId: "main" },
|
||||
}),
|
||||
).toBeNull();
|
||||
expect(
|
||||
snapshotViewportRect(
|
||||
[0, 0, NaN, 40],
|
||||
{ target },
|
||||
{ layoutUnitsPerCssPixel: 1, scrollCss: { x: 0, y: 0 } },
|
||||
),
|
||||
).toBeNull();
|
||||
expect(
|
||||
snapshotViewportRect(
|
||||
[0, 0, 10, 40],
|
||||
{ target },
|
||||
{ layoutUnitsPerCssPixel: 1, scrollCss: { x: Infinity, y: 0 } },
|
||||
),
|
||||
).toBeNull();
|
||||
});
|
||||
|
||||
it("does not turn a border quad or a failed owner read into a content projection", async () => {
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async () => ({
|
||||
model: { border: [0, 0, 100, 0, 100, 100, 0, 100] },
|
||||
})) as CdpRunner["send"],
|
||||
};
|
||||
expect(
|
||||
await new GeometryContext(cdp, 4).snapshotProjection({ target, frameId: "child" }, 10, [], {
|
||||
width: 100,
|
||||
height: 100,
|
||||
}),
|
||||
).toEqual({
|
||||
status: "unavailable",
|
||||
source: { target, frameId: "child" },
|
||||
ownerBackendNodeId: 10,
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps ancestor clips in target coordinates instead of applying nested offsets twice", async () => {
|
||||
const cdp = driver();
|
||||
const projection = await new GeometryContext(cdp, 4).snapshotProjection(
|
||||
{ target, frameId: "nested" },
|
||||
10,
|
||||
[rectPolygon({ x: 60, y: 620, w: 100, h: 100 })],
|
||||
{ width: 1000, height: 1000 },
|
||||
);
|
||||
if (projection.status !== "available") throw new Error("expected available projection");
|
||||
const input = snapshotViewportRect(
|
||||
[0, 0, 300, 200],
|
||||
{ target, frameId: "nested" },
|
||||
{ layoutUnitsPerCssPixel: 1, scrollCss: { x: 0, y: 0 } },
|
||||
);
|
||||
expect(projectSnapshotRect(input!, projection.projection)).toEqual({
|
||||
x: 60,
|
||||
y: 620,
|
||||
width: 100,
|
||||
height: 100,
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects malformed frame ancestry without looping", async () => {
|
||||
const cdp = driver();
|
||||
const cycle: CdpFrameGraph = {
|
||||
rootFrameId: "a",
|
||||
frames: [
|
||||
{ frameId: "a", parentFrameId: "b", target },
|
||||
{ frameId: "b", parentFrameId: "a", target },
|
||||
],
|
||||
};
|
||||
expect(await new GeometryContext(cdp, 4, cycle).targetProjection("a")).toBeNull();
|
||||
expect(cdp.calls).not.toHaveBeenCalled();
|
||||
const orphan: CdpFrameGraph = {
|
||||
rootFrameId: "a",
|
||||
frames: [{ frameId: "a", parentFrameId: "missing", target }],
|
||||
};
|
||||
expect(await new GeometryContext(cdp, 4, orphan).targetProjection("a")).toBeNull();
|
||||
});
|
||||
|
||||
it("rejects a live node whose frame belongs to another target before any DOM input", async () => {
|
||||
const cdp = driver();
|
||||
expect(
|
||||
await resolveNodeGeometry(
|
||||
cdp,
|
||||
4,
|
||||
{ target: { tabId: 4, sessionId: "wrong" }, frameId: "child", backendNodeId: 20 },
|
||||
{ scrollIntoView: true },
|
||||
),
|
||||
).toMatchObject({ code: "cdp_failed" });
|
||||
expect(cdp.calls).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("bounds concurrent measurements and does not dispatch queued reads after cancellation", async () => {
|
||||
let active = 0;
|
||||
let peak = 0;
|
||||
const release: Array<() => void> = [];
|
||||
const send = vi.fn(async () => {
|
||||
active++;
|
||||
peak = Math.max(peak, active);
|
||||
await new Promise<void>((resolve) => release.push(resolve));
|
||||
active--;
|
||||
return {
|
||||
visualViewport: { clientWidth: 1000 },
|
||||
cssVisualViewport: { clientWidth: 1000 },
|
||||
cssLayoutViewport: { clientWidth: 100, clientHeight: 100 },
|
||||
};
|
||||
});
|
||||
const controller = new AbortController();
|
||||
const context = new GeometryContext(
|
||||
{ send: send as CdpRunner["send"] },
|
||||
4,
|
||||
undefined,
|
||||
controller.signal,
|
||||
);
|
||||
const tasks = Array.from({ length: 8 }, (_, index) =>
|
||||
context.viewport({ tabId: 4, sessionId: String(index) }),
|
||||
);
|
||||
const settled = Promise.allSettled(tasks);
|
||||
expect(send).toHaveBeenCalledTimes(4);
|
||||
controller.abort();
|
||||
for (const done of release) done();
|
||||
const results = await settled;
|
||||
expect(peak).toBe(4);
|
||||
expect(send).toHaveBeenCalledTimes(4);
|
||||
expect(results.filter((item) => item.status === "rejected")).toHaveLength(4);
|
||||
});
|
||||
|
||||
it("releases an owner object even when cancellation arrives before its viewport read", async () => {
|
||||
const controller = new AbortController();
|
||||
const cdp = driver();
|
||||
const original = cdp.calls.getMockImplementation()!;
|
||||
cdp.calls.mockImplementation(async (tab, method) => {
|
||||
const result = await original(tab, method);
|
||||
if (method === "DOM.resolveNode") controller.abort();
|
||||
return result;
|
||||
});
|
||||
const context = new GeometryContext(cdp, 4, undefined, controller.signal);
|
||||
await expect(
|
||||
context.snapshotProjection({ target, frameId: "child" }, 10, [], {
|
||||
width: 1000,
|
||||
height: 1000,
|
||||
}),
|
||||
).rejects.toMatchObject({ name: "AbortError" });
|
||||
expect(cdp.calls.mock.calls.map(([, method]) => method)).toEqual([
|
||||
"DOM.getBoxModel",
|
||||
"DOM.resolveNode",
|
||||
"Runtime.releaseObject",
|
||||
]);
|
||||
});
|
||||
|
||||
it("reads live owner geometry only after scrolling", async () => {
|
||||
const cdp = driver();
|
||||
const original = cdp.calls.getMockImplementation()!;
|
||||
let scrolled = false;
|
||||
cdp.calls.mockImplementation(async (tab, method) => {
|
||||
if (method === "DOM.scrollIntoViewIfNeeded") {
|
||||
scrolled = true;
|
||||
return {};
|
||||
}
|
||||
if (method === "DOM.getContentQuads") {
|
||||
expect(scrolled).toBe(true);
|
||||
return { quads: [[60, 620, 100, 620, 100, 640, 60, 640]] };
|
||||
}
|
||||
if (method === "DOM.getBoxModel") expect(scrolled).toBe(true);
|
||||
return original(tab, method);
|
||||
});
|
||||
expect(
|
||||
await resolveNodeGeometry(
|
||||
cdp,
|
||||
4,
|
||||
{ target, frameId: "child", backendNodeId: 20 },
|
||||
{ scrollIntoView: true },
|
||||
),
|
||||
).toMatchObject({ topBounds: { x: 60, y: 620, width: 40, height: 20 } });
|
||||
expect(cdp.getFrameGraph).toHaveBeenCalledTimes(1);
|
||||
expect(cdp.calls.mock.calls.filter(([, method]) => method === "DOM.getBoxModel")).toHaveLength(
|
||||
1,
|
||||
);
|
||||
});
|
||||
|
||||
it("adapts viewport CSS bounds to page DIP with scroll and browser zoom, never raster DPR", () => {
|
||||
const viewport = { width: 1600, height: 1000, scrollX: 10, scrollY: 100, cssToDip: 0.9 };
|
||||
const clip = screenshotPageRect({ x: 20, y: 30, width: 120, height: 40 }, viewport);
|
||||
expect(clip).toEqual({ space: "page-dip", rect: { x: 27, y: 117, width: 108, height: 36 } });
|
||||
expect(
|
||||
screenshotPageRect({ x: 20, y: 30, width: 120, height: 40 }, { ...viewport, cssToDip: NaN }),
|
||||
).toBeNull();
|
||||
expect(
|
||||
screenshotPageRect({ x: 20, y: 30, width: 120, height: 40 }, { ...viewport, cssToDip: 0 }),
|
||||
).toBeNull();
|
||||
});
|
||||
|
||||
it("retries a rejected measurement only in a new operation", async () => {
|
||||
const cdp = driver();
|
||||
cdp.calls.mockRejectedValueOnce(new Error("unavailable"));
|
||||
const context = new GeometryContext(cdp, 4);
|
||||
await expect(context.viewport(target)).rejects.toThrow("unavailable");
|
||||
await expect(context.viewport(target)).rejects.toThrow("unavailable");
|
||||
expect(cdp.calls).toHaveBeenCalledTimes(1);
|
||||
expect(await new GeometryContext(cdp, 4).viewport(target)).toEqual({
|
||||
width: 1000,
|
||||
height: 1000,
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -21,7 +21,7 @@ function fakeAgentWindow(ids: number[]) {
|
||||
return id;
|
||||
}),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -540,6 +540,21 @@ describe("handleHover", () => {
|
||||
});
|
||||
});
|
||||
|
||||
function successfulFillScript(params: unknown) {
|
||||
const script = params as { arguments?: Array<{ value: unknown }>; functionDeclaration: string };
|
||||
const args = script.arguments ?? [];
|
||||
return {
|
||||
result: {
|
||||
value:
|
||||
args.length === 2
|
||||
? { before: "", expected: args[0].value }
|
||||
: script.functionDeclaration.startsWith("function(expected)")
|
||||
? { connected: true, matches: true, valueLength: String(args[0].value).length }
|
||||
: "ready",
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe("handleFill", () => {
|
||||
it("returns not_found for unknown ref", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
@@ -586,7 +601,8 @@ describe("handleFill", () => {
|
||||
"DOM.scrollIntoViewIfNeeded": () => ({}),
|
||||
"DOM.focus": () => ({}),
|
||||
"DOM.resolveNode": () => ({ object: { objectId: "obj-1" } }),
|
||||
"Runtime.callFunctionOn": () => ({ result: { type: "undefined" } }),
|
||||
"Runtime.callFunctionOn": successfulFillScript,
|
||||
"Runtime.releaseObject": () => ({}),
|
||||
"Input.insertText": () => ({}),
|
||||
});
|
||||
const res = await handleFill(
|
||||
@@ -600,13 +616,12 @@ describe("handleFill", () => {
|
||||
expect(res.used_ref).toBe("e1");
|
||||
const insert = fake.sent.find((c) => c.method === "Input.insertText");
|
||||
expect(insert?.params).toEqual({ text: "hello" });
|
||||
// clear_before defaults to true → callFunctionOn invoked once to
|
||||
// clear, once to fire input/change after typing.
|
||||
// Foreground replacement needs no extra caret-positioning round trip.
|
||||
const callFns = fake.sent.filter((c) => c.method === "Runtime.callFunctionOn");
|
||||
expect(callFns.length).toBeGreaterThanOrEqual(2);
|
||||
expect(callFns).toHaveLength(4);
|
||||
});
|
||||
|
||||
it("clear_before=false skips the wipe call", async () => {
|
||||
it("passes clear_before=false to preparation and verifies the result", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.refStore.set("e1", 1, { tabId: 4 });
|
||||
@@ -618,12 +633,12 @@ describe("handleFill", () => {
|
||||
"DOM.focus": () => ({}),
|
||||
"DOM.resolveNode": () => ({ object: { objectId: "obj-2" } }),
|
||||
"Runtime.callFunctionOn": (p) => {
|
||||
const fn = (p as { functionDeclaration?: string }).functionDeclaration ?? "";
|
||||
// No "this.value = ''" clearing on a no-clear path; only the
|
||||
// post-input dispatchEvent.
|
||||
expect(fn).not.toMatch(/this\.value\s*=\s*''/);
|
||||
return { result: { type: "undefined" } };
|
||||
const args = (p as { arguments?: Array<{ value: unknown }> }).arguments ?? [];
|
||||
if (args.length === 2) expect(args[1].value).toBe(false);
|
||||
return successfulFillScript(p);
|
||||
},
|
||||
"Input.dispatchKeyEvent": () => ({}),
|
||||
"Runtime.releaseObject": () => ({}),
|
||||
"Input.insertText": () => ({}),
|
||||
});
|
||||
const res = await handleFill(
|
||||
@@ -650,7 +665,8 @@ describe("handleFill", () => {
|
||||
"DOM.scrollIntoViewIfNeeded": () => ({}),
|
||||
"DOM.focus": () => ({}),
|
||||
"DOM.resolveNode": () => ({ object: { objectId: "obj-3" } }),
|
||||
"Runtime.callFunctionOn": () => ({ result: { type: "undefined" } }),
|
||||
"Runtime.callFunctionOn": successfulFillScript,
|
||||
"Runtime.releaseObject": () => ({}),
|
||||
"Input.insertText": () => ({}),
|
||||
});
|
||||
const res = await handleFill(
|
||||
|
||||
@@ -0,0 +1,266 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { SessionManager } from "@/session-manager/manager";
|
||||
import type { BrowserNavigationApi } from "../browser-navigation";
|
||||
import { handleNavigate, handleReload } from "../navigation";
|
||||
import type { CdpRunner } from "../shared";
|
||||
|
||||
const denied = "Cannot access a chrome-extension:// URL of different extension";
|
||||
const url = "https://example.test/recovered";
|
||||
|
||||
function event<T extends (...args: never[]) => void>() {
|
||||
const listeners = new Set<T>();
|
||||
return {
|
||||
listeners,
|
||||
addListener: (listener: T) => listeners.add(listener),
|
||||
removeListener: (listener: T) => listeners.delete(listener),
|
||||
fire: (...args: Parameters<T>) => {
|
||||
for (const listener of [...listeners]) listener(...args);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function fixture() {
|
||||
const manager = new SessionManager({
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 100),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 4),
|
||||
},
|
||||
});
|
||||
await manager.start("test");
|
||||
type Listener = Parameters<BrowserNavigationApi["onCommitted"]["addListener"]>[0];
|
||||
const events = {
|
||||
onCommitted: event<Listener>(),
|
||||
onDOMContentLoaded: event<Listener>(),
|
||||
onCompleted: event<Listener>(),
|
||||
onErrorOccurred: event<Listener>(),
|
||||
};
|
||||
const cdpEvents = event<Parameters<NonNullable<CdpRunner["onEvent"]>>[0]>();
|
||||
const state = { blocked: true, destinationBlocked: false, emitIdle: true };
|
||||
const send = vi.fn(async (_tabId: number, method: string) => {
|
||||
if (state.blocked) throw new Error(denied);
|
||||
if (method === "Page.getFrameTree") {
|
||||
return { frameTree: { frame: { id: "main", loaderId: "new-loader" } } };
|
||||
}
|
||||
if (method === "Runtime.evaluate") return { result: { value: "complete" } };
|
||||
if (method === "Page.setLifecycleEventsEnabled" && state.emitIdle) {
|
||||
cdpEvents.fire({ tabId: 4 }, "Page.lifecycleEvent", {
|
||||
frameId: "main",
|
||||
loaderId: "new-loader",
|
||||
name: "networkIdle",
|
||||
});
|
||||
}
|
||||
if (method === "Page.navigate") return { frameId: "main", loaderId: "new-loader" };
|
||||
return {};
|
||||
});
|
||||
const cdp: CdpRunner = {
|
||||
send: send as CdpRunner["send"],
|
||||
onEvent: (listener) => {
|
||||
cdpEvents.addListener(listener);
|
||||
return { dispose: () => cdpEvents.removeListener(listener) };
|
||||
},
|
||||
};
|
||||
const main = { tabId: 4, frameId: 0, documentId: "new-document" };
|
||||
const navigate = async () => {
|
||||
state.blocked = state.destinationBlocked;
|
||||
events.onCommitted.fire(main);
|
||||
events.onDOMContentLoaded.fire(main);
|
||||
events.onCompleted.fire(main);
|
||||
};
|
||||
const browserNavigation = { ...events, update: vi.fn(navigate), reload: vi.fn(navigate) };
|
||||
const tab = { id: 4, windowId: 100, active: true, url } as chrome.tabs.Tab;
|
||||
const tabsApi = { get: vi.fn(async () => tab), query: vi.fn(async () => [tab]) };
|
||||
const deps = { cdp, tabsApi, browserNavigation, defaultTimeoutMs: 1000 };
|
||||
const expectCleanedUp = () => {
|
||||
for (const entry of Object.values(events)) expect(entry.listeners.size).toBe(0);
|
||||
expect(cdpEvents.listeners.size).toBe(0);
|
||||
};
|
||||
return { manager, deps, state, send, events, main, cdpEvents, expectCleanedUp };
|
||||
}
|
||||
|
||||
describe("navigation after Chrome denies extension-frame access", () => {
|
||||
it.each([
|
||||
"load",
|
||||
"commit",
|
||||
"domcontentloaded",
|
||||
"networkidle",
|
||||
] as const)("recovers the same tab with the requested %s checkpoint", async (waitUntil) => {
|
||||
const f = await fixture();
|
||||
const result = await handleNavigate(
|
||||
f.manager,
|
||||
{
|
||||
session_id: "test",
|
||||
url,
|
||||
wait_until: waitUntil,
|
||||
},
|
||||
f.deps,
|
||||
);
|
||||
|
||||
expect(result).toMatchObject({ tab_id: 4, final_url: url, reached: waitUntil });
|
||||
expect(f.deps.browserNavigation.update).toHaveBeenCalledExactlyOnceWith(4, { url });
|
||||
expect(f.send.mock.calls.some(([, method]) => method === "Page.navigate")).toBe(false);
|
||||
expect(f.send).toHaveBeenCalledWith(4, "Page.enable", {});
|
||||
f.expectCleanedUp();
|
||||
});
|
||||
|
||||
it("recovers reload through tabs.reload and preserves hard reload", async () => {
|
||||
const f = await fixture();
|
||||
const result = await handleReload(f.manager, { session_id: "test", hard: true }, f.deps);
|
||||
expect(result).toMatchObject({ tab_id: 4, reached: "load" });
|
||||
expect(f.deps.browserNavigation.reload).toHaveBeenCalledExactlyOnceWith(4, {
|
||||
bypassCache: true,
|
||||
});
|
||||
expect(f.deps.browserNavigation.update).not.toHaveBeenCalled();
|
||||
f.expectCleanedUp();
|
||||
});
|
||||
|
||||
it("reports the restriction when reload leaves the conflicting frame present", async () => {
|
||||
const f = await fixture();
|
||||
f.state.destinationBlocked = true;
|
||||
const result = await handleReload(f.manager, { session_id: "test" }, f.deps);
|
||||
expect(result).toMatchObject({
|
||||
code: "cdp_failed",
|
||||
data: { reason: "cdp_extension_access_denied" },
|
||||
});
|
||||
expect(f.deps.browserNavigation.reload).toHaveBeenCalledOnce();
|
||||
f.expectCleanedUp();
|
||||
});
|
||||
|
||||
it("does not fall back for unrelated debugger failures", async () => {
|
||||
const f = await fixture();
|
||||
f.send.mockRejectedValue(new Error("Another debugger is already attached"));
|
||||
const result = await handleNavigate(f.manager, { session_id: "test", url }, f.deps);
|
||||
expect(result).toMatchObject({
|
||||
code: "cdp_failed",
|
||||
message: "Another debugger is already attached",
|
||||
});
|
||||
expect(f.deps.browserNavigation.update).not.toHaveBeenCalled();
|
||||
f.expectCleanedUp();
|
||||
});
|
||||
|
||||
it("does not replay navigation when a dispatched CDP command fails", async () => {
|
||||
const f = await fixture();
|
||||
f.state.blocked = false;
|
||||
const send = f.send.getMockImplementation()!;
|
||||
f.send.mockImplementation(async (tabId, method) => {
|
||||
if (method === "Page.navigate") throw new Error(denied);
|
||||
return send(tabId, method);
|
||||
});
|
||||
const result = await handleNavigate(f.manager, { session_id: "test", url }, f.deps);
|
||||
expect(result).toMatchObject({
|
||||
code: "cdp_failed",
|
||||
data: { reason: "cdp_extension_access_denied" },
|
||||
});
|
||||
expect(f.deps.browserNavigation.update).not.toHaveBeenCalled();
|
||||
f.expectCleanedUp();
|
||||
});
|
||||
|
||||
it("keeps the Agent Window guard in front of the recovery", async () => {
|
||||
const f = await fixture();
|
||||
f.deps.tabsApi.get.mockResolvedValue({
|
||||
id: 4,
|
||||
windowId: 200,
|
||||
active: true,
|
||||
url,
|
||||
} as chrome.tabs.Tab);
|
||||
const result = await handleNavigate(f.manager, { session_id: "test", tab_id: 4, url }, f.deps);
|
||||
expect(result).toMatchObject({ code: "permission_denied" });
|
||||
expect(f.send).not.toHaveBeenCalled();
|
||||
expect(f.deps.browserNavigation.update).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("ignores old-document, subframe, and other-tab completion events", async () => {
|
||||
const f = await fixture();
|
||||
f.deps.browserNavigation.update.mockImplementation(async () => {});
|
||||
let settled = false;
|
||||
const work = handleNavigate(f.manager, { session_id: "test", url }, f.deps).then((result) => {
|
||||
settled = true;
|
||||
return result;
|
||||
});
|
||||
await vi.waitFor(() => expect(f.deps.browserNavigation.update).toHaveBeenCalledOnce());
|
||||
f.events.onCompleted.fire(f.main);
|
||||
f.events.onCommitted.fire({ ...f.main, tabId: 9 });
|
||||
f.events.onCommitted.fire({ ...f.main, frameId: 1 });
|
||||
f.events.onCompleted.fire(f.main);
|
||||
await Promise.resolve();
|
||||
expect(settled).toBe(false);
|
||||
f.events.onCommitted.fire(f.main);
|
||||
f.events.onCompleted.fire({ ...f.main, documentId: "old-document" });
|
||||
await Promise.resolve();
|
||||
expect(settled).toBe(false);
|
||||
f.state.blocked = false;
|
||||
f.events.onCompleted.fire(f.main);
|
||||
expect(await work).toMatchObject({ reached: "load" });
|
||||
f.expectCleanedUp();
|
||||
});
|
||||
|
||||
it("does not infer network idle from a browser load event", async () => {
|
||||
const f = await fixture();
|
||||
f.state.emitIdle = false;
|
||||
const result = await handleNavigate(
|
||||
f.manager,
|
||||
{
|
||||
session_id: "test",
|
||||
url,
|
||||
wait_until: "networkidle",
|
||||
timeout_ms: 10,
|
||||
},
|
||||
f.deps,
|
||||
);
|
||||
expect(result).toMatchObject({ reached: "timeout" });
|
||||
f.expectCleanedUp();
|
||||
});
|
||||
|
||||
it("cleans up browser listeners on timeout", async () => {
|
||||
const f = await fixture();
|
||||
f.deps.browserNavigation.update.mockImplementation(async () => {});
|
||||
const result = await handleNavigate(
|
||||
f.manager,
|
||||
{ session_id: "test", url, timeout_ms: 10 },
|
||||
f.deps,
|
||||
);
|
||||
expect(result).toMatchObject({ reached: "timeout" });
|
||||
f.expectCleanedUp();
|
||||
});
|
||||
|
||||
it("does not navigate an already-cancelled request", async () => {
|
||||
const f = await fixture();
|
||||
const result = await handleNavigate(
|
||||
f.manager,
|
||||
{ session_id: "test", url },
|
||||
{
|
||||
...f.deps,
|
||||
signal: AbortSignal.abort(),
|
||||
},
|
||||
);
|
||||
expect(result).toMatchObject({ code: "cancelled" });
|
||||
expect(f.deps.browserNavigation.update).not.toHaveBeenCalled();
|
||||
f.expectCleanedUp();
|
||||
});
|
||||
|
||||
it("cleans up browser listeners when cancelled after dispatch", async () => {
|
||||
const f = await fixture();
|
||||
const ac = new AbortController();
|
||||
f.deps.browserNavigation.update.mockImplementation(async () => {
|
||||
ac.abort();
|
||||
});
|
||||
const result = await handleNavigate(
|
||||
f.manager,
|
||||
{ session_id: "test", url },
|
||||
{ ...f.deps, signal: ac.signal },
|
||||
);
|
||||
expect(result).toMatchObject({ code: "cancelled" });
|
||||
expect(f.deps.browserNavigation.update).toHaveBeenCalledOnce();
|
||||
f.expectCleanedUp();
|
||||
});
|
||||
|
||||
it("reports browser navigation errors and releases listeners", async () => {
|
||||
const f = await fixture();
|
||||
f.deps.browserNavigation.update.mockImplementation(async () => {
|
||||
f.events.onErrorOccurred.fire({ ...f.main, error: "net::ERR_NAME_NOT_RESOLVED" });
|
||||
});
|
||||
const result = await handleNavigate(f.manager, { session_id: "test", url }, f.deps);
|
||||
expect(result).toMatchObject({ code: "cdp_failed", message: "net::ERR_NAME_NOT_RESOLVED" });
|
||||
f.expectCleanedUp();
|
||||
});
|
||||
});
|
||||
@@ -21,7 +21,7 @@ function fakeAgentWindow(ids: number[]) {
|
||||
return id;
|
||||
}),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -12,7 +12,7 @@ function fakeAgentWindow(ids: number[]) {
|
||||
return id;
|
||||
}),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,8 +1,26 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { SessionManager } from "@/session-manager/manager";
|
||||
import { handleSessionStop } from "../session";
|
||||
import type { ChromeTabsApi } from "../shared";
|
||||
import { type AgentOverlayResetApi, type ChromeWindowsApi, type TabMutationApi } from "../tabs";
|
||||
|
||||
/** Build a read-only query api over a FakeState. */
|
||||
function makeQuery(state: FakeState): ChromeTabsApi {
|
||||
return {
|
||||
get: vi.fn(async (id) => {
|
||||
const t = state.tabs.get(id);
|
||||
if (!t) throw new Error(`tab ${id} not found`);
|
||||
return t;
|
||||
}),
|
||||
query: vi.fn(async (q: chrome.tabs.QueryInfo) => {
|
||||
const w = q.windowId;
|
||||
return Array.from(state.tabs.values()).filter(
|
||||
(t) => typeof w !== "number" || t.windowId === w,
|
||||
);
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
function fakeAgentWindow(ids: number[]) {
|
||||
let i = 0;
|
||||
const create = vi.fn(async () => {
|
||||
@@ -11,7 +29,7 @@ function fakeAgentWindow(ids: number[]) {
|
||||
return id;
|
||||
});
|
||||
const remove = vi.fn(async () => {});
|
||||
const ensureActiveTab = vi.fn(async () => {});
|
||||
const ensureActiveTab = vi.fn(async () => 0);
|
||||
return { create, remove, ensureActiveTab };
|
||||
}
|
||||
|
||||
@@ -30,7 +48,9 @@ function makeApis(
|
||||
} {
|
||||
const tabs: TabMutationApi = {
|
||||
create: vi.fn(),
|
||||
remove: vi.fn(async () => {}),
|
||||
remove: vi.fn(async (id: number) => {
|
||||
state.tabs.delete(id);
|
||||
}),
|
||||
update: vi.fn(async (_id, _p) => undefined),
|
||||
get: vi.fn(async (id) => {
|
||||
const t = state.tabs.get(id);
|
||||
@@ -237,11 +257,15 @@ describe("handleSessionStop with auto-return", () => {
|
||||
moves: [],
|
||||
};
|
||||
const { tabs, windows } = makeApis(state, { moveThrowsFor: new Set([1]) });
|
||||
const cdp = {
|
||||
detachSession: vi.fn(async () => {}),
|
||||
releaseSessionTab: vi.fn(async () => {}),
|
||||
};
|
||||
|
||||
const res = await handleSessionStop(
|
||||
sm,
|
||||
{ session_id: "aa11" },
|
||||
{ tabManagement: { tabs, windows } },
|
||||
{ cdp, tabManagement: { tabs, windows } },
|
||||
);
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
expect(res.return_failures?.map((f) => f.tab_id)).toEqual([1]);
|
||||
@@ -249,6 +273,8 @@ describe("handleSessionStop with auto-return", () => {
|
||||
expect(sm.has("aa11")).toBe(true);
|
||||
expect(ctx.borrowedTabs.has(1)).toBe(true);
|
||||
expect(ctx.borrowedTabs.has(2)).toBe(false);
|
||||
expect(cdp.releaseSessionTab).toHaveBeenCalledExactlyOnceWith("aa11", 2);
|
||||
expect(cdp.detachSession).not.toHaveBeenCalled();
|
||||
expect(aw.remove).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
@@ -267,4 +293,275 @@ describe("handleSessionStop with auto-return", () => {
|
||||
await handleSessionStop(sm, { session_id: "aa11" }, { tabManagement: { tabs, windows } });
|
||||
expect(ctx.refStore.isEmpty()).toBe(true);
|
||||
});
|
||||
|
||||
it.each([
|
||||
"before teardown",
|
||||
"during its return",
|
||||
"after its return failed",
|
||||
"before its turn in the return loop",
|
||||
])("finishes stopping when a borrowed tab closes %s", async (timing) => {
|
||||
const aw = fakeAgentWindow([100]);
|
||||
const sm = new SessionManager({ agentWindow: aw });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.borrowedTabs.set(1, { tabId: 1, originalWindowId: 200, originalIndex: 0 });
|
||||
ctx.borrowedTabs.set(2, { tabId: 2, originalWindowId: 200, originalIndex: 1 });
|
||||
const state: FakeState = {
|
||||
tabs: new Map([
|
||||
[1, { id: 1, windowId: 100 } as chrome.tabs.Tab],
|
||||
[2, { id: 2, windowId: 100 } as chrome.tabs.Tab],
|
||||
]),
|
||||
windowsClosed: new Set(),
|
||||
moves: [],
|
||||
};
|
||||
const { tabs, windows } = makeApis(state);
|
||||
const cdp = { detachSession: vi.fn(async () => {}) };
|
||||
const closeTab = (tabId: number) => {
|
||||
state.tabs.delete(tabId);
|
||||
sm.forgetClosedTab(tabId);
|
||||
};
|
||||
const baseMove = tabs.move;
|
||||
tabs.move = vi.fn(async (tabId, props) => {
|
||||
if (tabId === 1 && timing === "during its return") closeTab(1);
|
||||
if (tabId === 1 && timing === "after its return failed") {
|
||||
throw new Error("simulated move failure");
|
||||
}
|
||||
if (tabId === 2 && timing === "after its return failed") closeTab(1);
|
||||
if (tabId === 1 && timing === "before its turn in the return loop") closeTab(2);
|
||||
if (!state.tabs.has(tabId)) throw new Error(`tab ${tabId} not found`);
|
||||
return baseMove(tabId, props);
|
||||
});
|
||||
if (timing === "before teardown") closeTab(1);
|
||||
|
||||
const res = await handleSessionStop(
|
||||
sm,
|
||||
{ session_id: "aa11" },
|
||||
{
|
||||
cdp,
|
||||
tabManagement: {
|
||||
tabs,
|
||||
windows,
|
||||
agentOverlayReset: { resetAgentOverlays: vi.fn(async () => {}) },
|
||||
},
|
||||
tabsQuery: makeQuery(state),
|
||||
},
|
||||
);
|
||||
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
const closedTabId = timing === "before its turn in the return loop" ? 2 : 1;
|
||||
expect(res.returned_tab_ids).toEqual([closedTabId === 1 ? 2 : 1]);
|
||||
expect(res.return_failures).toBeUndefined();
|
||||
expect(ctx.borrowedTabs.size).toBe(0);
|
||||
expect(sm.has("aa11")).toBe(false);
|
||||
expect(cdp.detachSession).toHaveBeenCalledWith("aa11");
|
||||
expect(aw.remove).toHaveBeenCalledWith(100);
|
||||
if (timing === "before teardown" || timing === "before its turn in the return loop") {
|
||||
expect(tabs.move).not.toHaveBeenCalledWith(closedTabId, expect.anything());
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("handleSessionStop window release (issue #57)", () => {
|
||||
const agentWindowId = 100;
|
||||
|
||||
it("releases the window when user-created tabs remain", async () => {
|
||||
const aw = fakeAgentWindow([agentWindowId]);
|
||||
// ensureActiveTab returns the home tab id (10). We override the fake to
|
||||
// return a known id.
|
||||
aw.ensureActiveTab = vi.fn(async () => 10);
|
||||
const sm = new SessionManager({ agentWindow: aw });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.agentCreatedTabs.add(11);
|
||||
ctx.agentCreatedTabs.add(12);
|
||||
// user tab 99 (created via Chrome UI, NOT in agentCreatedTabs)
|
||||
const state: FakeState = {
|
||||
tabs: new Map([
|
||||
[10, { id: 10, windowId: agentWindowId } as chrome.tabs.Tab],
|
||||
[11, { id: 11, windowId: agentWindowId } as chrome.tabs.Tab],
|
||||
[12, { id: 12, windowId: agentWindowId } as chrome.tabs.Tab],
|
||||
[99, { id: 99, windowId: agentWindowId } as chrome.tabs.Tab],
|
||||
]),
|
||||
windowsClosed: new Set(),
|
||||
moves: [],
|
||||
};
|
||||
const { tabs, windows } = makeApis(state);
|
||||
const query = makeQuery(state);
|
||||
|
||||
const res = await handleSessionStop(
|
||||
sm,
|
||||
{ session_id: "aa11" },
|
||||
{ tabManagement: { tabs, windows }, tabsQuery: query },
|
||||
);
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
expect(res.window_released).toBe(true);
|
||||
// agent tabs + home removed, user tab 99 kept
|
||||
expect((tabs.remove as ReturnType<typeof vi.fn>).mock.calls.map((c) => c[0]).sort()).toEqual([
|
||||
10, 11, 12,
|
||||
]);
|
||||
expect(state.tabs.has(99)).toBe(true);
|
||||
// window released (dropOnly), not closed
|
||||
expect(aw.remove).not.toHaveBeenCalled();
|
||||
expect(sm.has("aa11")).toBe(false);
|
||||
});
|
||||
|
||||
it("closes the window (not release) when no user tabs remain", async () => {
|
||||
const aw = fakeAgentWindow([agentWindowId]);
|
||||
aw.ensureActiveTab = vi.fn(async () => 10);
|
||||
const sm = new SessionManager({ agentWindow: aw });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.agentCreatedTabs.add(11);
|
||||
const state: FakeState = {
|
||||
tabs: new Map([
|
||||
[10, { id: 10, windowId: agentWindowId } as chrome.tabs.Tab],
|
||||
[11, { id: 11, windowId: agentWindowId } as chrome.tabs.Tab],
|
||||
]),
|
||||
windowsClosed: new Set(),
|
||||
moves: [],
|
||||
};
|
||||
const { tabs, windows } = makeApis(state);
|
||||
const query = makeQuery(state);
|
||||
|
||||
const res = await handleSessionStop(
|
||||
sm,
|
||||
{ session_id: "aa11" },
|
||||
{ tabManagement: { tabs, windows }, tabsQuery: query },
|
||||
);
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
expect(res.window_released).toBeFalsy();
|
||||
expect(aw.remove).toHaveBeenCalledWith(agentWindowId);
|
||||
expect(sm.has("aa11")).toBe(false);
|
||||
});
|
||||
|
||||
it("is non-fatal when an agent tab is already gone", async () => {
|
||||
const aw = fakeAgentWindow([agentWindowId]);
|
||||
aw.ensureActiveTab = vi.fn(async () => 10);
|
||||
const sm = new SessionManager({ agentWindow: aw });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.agentCreatedTabs.add(11); // no longer in state → remove throws
|
||||
const state: FakeState = {
|
||||
tabs: new Map([[10, { id: 10, windowId: agentWindowId } as chrome.tabs.Tab]]),
|
||||
windowsClosed: new Set(),
|
||||
moves: [],
|
||||
};
|
||||
const { tabs, windows } = makeApis(state);
|
||||
(tabs.remove as ReturnType<typeof vi.fn>).mockImplementation(async (id: number) => {
|
||||
if (id === 11) throw new Error("tab 11 already closed");
|
||||
state.tabs.delete(id); // home (10) removed normally
|
||||
});
|
||||
const query = makeQuery(state);
|
||||
|
||||
const res = await handleSessionStop(
|
||||
sm,
|
||||
{ session_id: "aa11" },
|
||||
{ tabManagement: { tabs, windows }, tabsQuery: query },
|
||||
);
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
// user tab 10? no — 10 is home, removed. window would be empty → close.
|
||||
expect(res.window_released).toBeFalsy();
|
||||
expect(aw.remove).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("degrades to close-window when tabsQuery is missing", async () => {
|
||||
const aw = fakeAgentWindow([agentWindowId]);
|
||||
aw.ensureActiveTab = vi.fn(async () => 10);
|
||||
const sm = new SessionManager({ agentWindow: aw });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.agentCreatedTabs.add(11);
|
||||
const state: FakeState = {
|
||||
tabs: new Map([
|
||||
[10, { id: 10, windowId: agentWindowId } as chrome.tabs.Tab],
|
||||
[11, { id: 11, windowId: agentWindowId } as chrome.tabs.Tab],
|
||||
]),
|
||||
windowsClosed: new Set(),
|
||||
moves: [],
|
||||
};
|
||||
const { tabs, windows } = makeApis(state);
|
||||
|
||||
const res = await handleSessionStop(
|
||||
sm,
|
||||
{ session_id: "aa11" },
|
||||
{ tabManagement: { tabs, windows } }, // no tabsQuery
|
||||
);
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
// No query available → conservatively close the window.
|
||||
expect(res.window_released).toBeFalsy();
|
||||
expect(aw.remove).toHaveBeenCalledWith(agentWindowId);
|
||||
});
|
||||
|
||||
it("degrades to close-window when tabsQuery.query throws", async () => {
|
||||
const aw = fakeAgentWindow([agentWindowId]);
|
||||
aw.ensureActiveTab = vi.fn(async () => 10);
|
||||
const sm = new SessionManager({ agentWindow: aw });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.agentCreatedTabs.add(11);
|
||||
const state: FakeState = {
|
||||
tabs: new Map([[11, { id: 11, windowId: agentWindowId } as chrome.tabs.Tab]]),
|
||||
windowsClosed: new Set(),
|
||||
moves: [],
|
||||
};
|
||||
const { tabs, windows } = makeApis(state);
|
||||
const query = makeQuery(state);
|
||||
(query.query as ReturnType<typeof vi.fn>).mockImplementation(async () => {
|
||||
throw new Error("query failed");
|
||||
});
|
||||
|
||||
const res = await handleSessionStop(
|
||||
sm,
|
||||
{ session_id: "aa11" },
|
||||
{ tabManagement: { tabs, windows }, tabsQuery: query },
|
||||
);
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
expect(res.window_released).toBeFalsy();
|
||||
expect(aw.remove).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("tracks agentCreatedTabs across tab_create / tab_close", async () => {
|
||||
const aw = fakeAgentWindow([agentWindowId]);
|
||||
aw.ensureActiveTab = vi.fn(async () => 10);
|
||||
const sm = new SessionManager({ agentWindow: aw });
|
||||
const ctx = await sm.start("aa11");
|
||||
// Simulate handleTabCreate adding, then handleTabClose removing.
|
||||
ctx.agentCreatedTabs.add(11);
|
||||
expect(ctx.agentCreatedTabs.has(11)).toBe(true);
|
||||
ctx.agentCreatedTabs.delete(11); // as handleTabClose does on success
|
||||
expect(ctx.agentCreatedTabs.has(11)).toBe(false);
|
||||
// Unknown id delete is a safe no-op (user-closed tab).
|
||||
expect(() => ctx.agentCreatedTabs.delete(999)).not.toThrow();
|
||||
});
|
||||
|
||||
it("closes the window (not release) when an agent tab fails to close", async () => {
|
||||
// Regression: a leaked agent tab (still in agentCreatedTabs, but
|
||||
// still present because Step 4 remove() threw) must NOT be mistaken
|
||||
// for a user tab. Otherwise the window would be released (dropOnly)
|
||||
// and the agent tab would leak (issue #57 regression).
|
||||
const aw = fakeAgentWindow([agentWindowId]);
|
||||
aw.ensureActiveTab = vi.fn(async () => 10);
|
||||
const sm = new SessionManager({ agentWindow: aw });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.agentCreatedTabs.add(11); // Step 4 remove() will throw for this tab
|
||||
const state: FakeState = {
|
||||
tabs: new Map([
|
||||
[10, { id: 10, windowId: agentWindowId } as chrome.tabs.Tab],
|
||||
[11, { id: 11, windowId: agentWindowId } as chrome.tabs.Tab],
|
||||
]),
|
||||
windowsClosed: new Set(),
|
||||
moves: [],
|
||||
};
|
||||
const { tabs, windows } = makeApis(state);
|
||||
(tabs.remove as ReturnType<typeof vi.fn>).mockImplementation(async (id: number) => {
|
||||
if (id === 11) throw new Error("tab 11 failed to close");
|
||||
state.tabs.delete(id); // home (10) removed normally; 11 lingers
|
||||
});
|
||||
const query = makeQuery(state);
|
||||
|
||||
const res = await handleSessionStop(
|
||||
sm,
|
||||
{ session_id: "aa11" },
|
||||
{ tabManagement: { tabs, windows }, tabsQuery: query },
|
||||
);
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
// The leaked agent tab must NOT keep the window open.
|
||||
expect(res.window_released).toBeFalsy();
|
||||
expect(aw.remove).toHaveBeenCalledWith(agentWindowId);
|
||||
expect(sm.has("aa11")).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -12,7 +12,7 @@ function fakeAgentWindow() {
|
||||
return {
|
||||
create: vi.fn(async () => 100),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,233 @@
|
||||
// @vitest-environment node
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { CdpFrame, CdpFrameGraph, CdpTarget } from "@/browser-driver/frame-graph";
|
||||
import type { CdpRunner } from "../shared";
|
||||
import { captureObservationFacts } from "../vom/capture-coordinator";
|
||||
|
||||
type Send = <T = Record<string, unknown>>(
|
||||
method: string,
|
||||
params?: object,
|
||||
sessionId?: string,
|
||||
) => Promise<T>;
|
||||
type Tree = { frame: { id: string; parentId?: string; name?: string }; childFrames?: Tree[] };
|
||||
type Rect = { x: number; y: number; w: number; h: number };
|
||||
type Oracle = { probe: Rect; owners: Record<string, { x: number; y: number; scale: number }> };
|
||||
|
||||
// The independent oracle uses DOM border boxes and this fixture's axis-aligned
|
||||
// iframe transforms. No production snapshot conversion/projection builds expectations.
|
||||
const oracleExpression = `(() => {
|
||||
const box = document.querySelector('#probe').getBoundingClientRect();
|
||||
const owners = {};
|
||||
for (const frame of document.querySelectorAll('iframe')) {
|
||||
const rect = frame.getBoundingClientRect();
|
||||
const style = getComputedStyle(frame);
|
||||
const scale = rect.width / frame.offsetWidth;
|
||||
owners[frame.name] = {
|
||||
x: rect.x + (frame.clientLeft + parseFloat(style.paddingLeft)) * scale,
|
||||
y: rect.y + (frame.clientTop + parseFloat(style.paddingTop)) * scale,
|
||||
scale,
|
||||
};
|
||||
}
|
||||
return { probe: { x: box.x, y: box.y, w: box.width, h: box.height }, owners };
|
||||
})()`;
|
||||
|
||||
describe.skipIf(!process.env.BSK_GEOMETRY_CHROME)("real DOMSnapshot coordinate contract", () => {
|
||||
it.each([
|
||||
{ deviceScale: 1, zoom: 1 },
|
||||
{ deviceScale: 0.8, zoom: 1 },
|
||||
{ deviceScale: 1, zoom: 1.25 },
|
||||
{ deviceScale: 2, zoom: 1 },
|
||||
{ deviceScale: 2, zoom: 0.8 },
|
||||
])("matches DOM geometry at device scale $deviceScale and zoom $zoom", async (configuration) => {
|
||||
const evalRoot = new URL("../../../../../evals/browser/", import.meta.url);
|
||||
const { createEvalServer } = await import(new URL("lib/server.mjs", evalRoot).href);
|
||||
const { withChrome } = await import(
|
||||
new URL("cases/regression/snapshot-coordinates/chrome.mjs", evalRoot).href
|
||||
);
|
||||
const server = createEvalServer();
|
||||
const { baseUrl } = await server.start();
|
||||
try {
|
||||
await withChrome(
|
||||
{ executable: process.env.BSK_GEOMETRY_CHROME, ...configuration },
|
||||
async (send: Send) => {
|
||||
const { targetId } = await send<{ targetId: string }>("Target.createTarget", {
|
||||
url: "about:blank",
|
||||
});
|
||||
const { sessionId: rootSession } = await send<{ sessionId: string }>(
|
||||
"Target.attachToTarget",
|
||||
{ targetId, flatten: true },
|
||||
);
|
||||
await send(
|
||||
"Page.navigate",
|
||||
{ url: `${baseUrl}/snapshot-coordinates?run=coordinates` },
|
||||
rootSession,
|
||||
);
|
||||
await expect
|
||||
.poll(
|
||||
() =>
|
||||
server
|
||||
.snapshot("coordinates")
|
||||
.events.some(
|
||||
(event: { type: string; data: { root?: boolean } }) =>
|
||||
event.type === "geometry.ready" && event.data.root,
|
||||
),
|
||||
{ timeout: 10_000 },
|
||||
)
|
||||
.toBe(true);
|
||||
|
||||
const targets = await send<{ targetInfos: { targetId: string; type: string }[] }>(
|
||||
"Target.getTargets",
|
||||
);
|
||||
const sessions = [rootSession];
|
||||
for (const target of targets.targetInfos.filter((target) => target.type === "iframe")) {
|
||||
const { sessionId } = await send<{ sessionId: string }>("Target.attachToTarget", {
|
||||
targetId: target.targetId,
|
||||
flatten: true,
|
||||
});
|
||||
sessions.push(sessionId);
|
||||
}
|
||||
expect(sessions.length).toBe(2); // The cross-site fixture must actually be an OOPIF.
|
||||
const frames: CdpFrame[] = [];
|
||||
const names = new Map<string, string>();
|
||||
const sessionFor = (target: CdpTarget) => target.sessionId ?? rootSession;
|
||||
for (const sessionId of sessions) {
|
||||
const { frameTree } = await send<{ frameTree: Tree }>(
|
||||
"Page.getFrameTree",
|
||||
{},
|
||||
sessionId,
|
||||
);
|
||||
const target: CdpTarget = {
|
||||
tabId: 1,
|
||||
...(sessionId === rootSession ? {} : { sessionId }),
|
||||
};
|
||||
const visit = (tree: Tree, parentFrameId = tree.frame.parentId) => {
|
||||
frames.push({ frameId: tree.frame.id, parentFrameId, target });
|
||||
names.set(tree.frame.id, tree.frame.name ?? "");
|
||||
for (const child of tree.childFrames ?? []) visit(child, tree.frame.id);
|
||||
};
|
||||
visit(frameTree);
|
||||
}
|
||||
expect(frames).toHaveLength(5);
|
||||
for (const frame of frames) {
|
||||
if (!frame.parentFrameId) continue;
|
||||
const parent = frames.find((item) => item.frameId === frame.parentFrameId)!;
|
||||
const owner = await send<{ backendNodeId: number }>(
|
||||
"DOM.getFrameOwner",
|
||||
{ frameId: frame.frameId },
|
||||
sessionFor(parent.target),
|
||||
);
|
||||
frame.ownerBackendNodeId = owner.backendNodeId;
|
||||
}
|
||||
const graph: CdpFrameGraph = { rootFrameId: frames[0].frameId, frames };
|
||||
const calls: string[] = [];
|
||||
const cdp: CdpRunner = {
|
||||
send: (tabId, method, params) => {
|
||||
calls.push(`${tabId}:${method}`);
|
||||
return send(method, params, rootSession);
|
||||
},
|
||||
sendToTarget: (target, method, params) => {
|
||||
calls.push(`${target.sessionId ?? target.tabId}:${method}`);
|
||||
return send(method, params, sessionFor(target));
|
||||
},
|
||||
getFrameGraph: async () => graph,
|
||||
};
|
||||
const oracles = new Map<string, Oracle>();
|
||||
for (const frame of frames) {
|
||||
const session = sessionFor(frame.target);
|
||||
const { executionContextId } = await send<{ executionContextId: number }>(
|
||||
"Page.createIsolatedWorld",
|
||||
{ frameId: frame.frameId, worldName: "coordinate-oracle" },
|
||||
session,
|
||||
);
|
||||
await expect
|
||||
.poll(
|
||||
async () => {
|
||||
const ready = await send<{ result: { value: boolean } }>(
|
||||
"Runtime.evaluate",
|
||||
{
|
||||
expression: 'document.documentElement.dataset.geometryReady === "true"',
|
||||
contextId: executionContextId,
|
||||
returnByValue: true,
|
||||
},
|
||||
session,
|
||||
);
|
||||
return ready.result.value;
|
||||
},
|
||||
{ timeout: 5_000 },
|
||||
)
|
||||
.toBe(true);
|
||||
// Establish scroll after cross-process target creation/layout settles.
|
||||
const scroll = frame.parentFrameId
|
||||
? names.get(frame.frameId) === "nested"
|
||||
? [10, 20]
|
||||
: [40, 100]
|
||||
: [80, 240];
|
||||
await send(
|
||||
"Runtime.evaluate",
|
||||
{
|
||||
expression: `(async () => { scrollTo(${scroll.join(",")}); await new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve))); })()`,
|
||||
contextId: executionContextId,
|
||||
awaitPromise: true,
|
||||
},
|
||||
session,
|
||||
);
|
||||
const reply = await send<{ result: { value: Oracle } }>(
|
||||
"Runtime.evaluate",
|
||||
{ expression: oracleExpression, contextId: executionContextId, returnByValue: true },
|
||||
session,
|
||||
);
|
||||
oracles.set(frame.frameId, reply.result.value);
|
||||
}
|
||||
const facts = await captureObservationFacts(cdp, 1);
|
||||
expect(facts.issues).toEqual([]);
|
||||
for (const frame of frames) {
|
||||
const node = facts.documents
|
||||
.find((doc) => doc.frame.frameId === frame.frameId)
|
||||
?.domNodes.find((node) => node.attrs.id === "probe");
|
||||
expect(node, `missing probe in ${names.get(frame.frameId) || "root"}`).toBeDefined();
|
||||
const local = oracles.get(frame.frameId)!.probe;
|
||||
const top = { ...local };
|
||||
let current = frame;
|
||||
while (current.parentFrameId) {
|
||||
const owner = oracles.get(current.parentFrameId)!.owners[names.get(current.frameId)!];
|
||||
top.x = owner.x + top.x * owner.scale;
|
||||
top.y = owner.y + top.y * owner.scale;
|
||||
top.w *= owner.scale;
|
||||
top.h *= owner.scale;
|
||||
current = frames.find((item) => item.frameId === current.parentFrameId)!;
|
||||
}
|
||||
expect(
|
||||
node!.rect,
|
||||
JSON.stringify({
|
||||
frame: names.get(frame.frameId),
|
||||
local,
|
||||
top,
|
||||
viewport: facts.viewport,
|
||||
}),
|
||||
).not.toBeNull();
|
||||
for (const key of ["x", "y", "w", "h"] as const) {
|
||||
expect(
|
||||
Math.abs(node!.localRect![key] - local[key]),
|
||||
`${names.get(frame.frameId)} local ${key}`,
|
||||
).toBeLessThan(2);
|
||||
expect(
|
||||
Math.abs(node!.rect![key] - top[key]),
|
||||
`${names.get(frame.frameId)} top ${key}`,
|
||||
).toBeLessThan(2);
|
||||
}
|
||||
}
|
||||
expect(calls.filter((call) => call.endsWith(":Page.getLayoutMetrics"))).toHaveLength(
|
||||
sessions.length,
|
||||
);
|
||||
const metrics = await send<{
|
||||
cssVisualViewport: { zoom: number };
|
||||
visualViewport: { clientWidth: number };
|
||||
}>("Page.getLayoutMetrics", {}, rootSession);
|
||||
expect(metrics.cssVisualViewport.zoom).toBeCloseTo(configuration.zoom, 4);
|
||||
},
|
||||
);
|
||||
} finally {
|
||||
await server.stop();
|
||||
}
|
||||
}, 30_000);
|
||||
});
|
||||
@@ -11,7 +11,7 @@ function fakeAgentWindow(ids: number[]) {
|
||||
return id;
|
||||
},
|
||||
remove: async () => {},
|
||||
ensureActiveTab: async () => {},
|
||||
ensureActiveTab: async () => 1,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { type CdpDebuggerApi, ChromiumCdp } from "@/browser-driver/chromium-cdp";
|
||||
import { SessionManager } from "@/session-manager/manager";
|
||||
import { handleHover } from "../interaction";
|
||||
import {
|
||||
type AgentOverlayResetApi,
|
||||
type ChromeWindowsApi,
|
||||
@@ -23,7 +25,7 @@ function fakeAgentWindow(ids: number[]) {
|
||||
return id;
|
||||
}),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -221,6 +223,7 @@ describe("handleTabCreate", () => {
|
||||
active: true,
|
||||
});
|
||||
expect(res).toMatchObject({ tab_id: 10, window_id: 100 });
|
||||
expect(sm.get("aa11")?.agentCreatedTabs.has(10)).toBe(true);
|
||||
});
|
||||
|
||||
it("forwards explicit url + index + active=false", async () => {
|
||||
@@ -256,6 +259,41 @@ describe("handleTabCreate", () => {
|
||||
const res = await handleTabCreate(sm, { session_id: "aa11", index: -1 }, { tabs: api });
|
||||
expect(res).toMatchObject({ code: "invalid_params" });
|
||||
});
|
||||
|
||||
it("does not claim any tab when chrome.tabs.create fails", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
await sm.start("aa11");
|
||||
const state: FakeTabState = { tabs: new Map(), nextTabId: 10, windowsClosed: new Set() };
|
||||
const { api, spies } = makeTabMutationApi(state);
|
||||
spies.create.mockRejectedValueOnce(new Error("chrome.tabs.create failed"));
|
||||
|
||||
const res = await handleTabCreate(sm, { session_id: "aa11" }, { tabs: api });
|
||||
expect(res).toMatchObject({ code: "protocol_error" });
|
||||
|
||||
expect(sm.get("aa11")?.agentCreatedTabs).toEqual(new Set([1]));
|
||||
});
|
||||
|
||||
it("claims the returned id even when a user tab appears during tab creation", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
const state: FakeTabState = { tabs: new Map(), nextTabId: 10, windowsClosed: new Set() };
|
||||
const { api, spies } = makeTabMutationApi(state);
|
||||
let resolveCreate!: (tab: chrome.tabs.Tab) => void;
|
||||
const createResult = new Promise<chrome.tabs.Tab>((resolve) => {
|
||||
resolveCreate = resolve;
|
||||
});
|
||||
spies.create.mockImplementationOnce(() => createResult);
|
||||
|
||||
const pending = handleTabCreate(sm, { session_id: "aa11" }, { tabs: api });
|
||||
// A user can create an unrelated same-window tab while Chrome is still
|
||||
// resolving the agent request. There is no pending slot for it to steal.
|
||||
expect(ctx.agentCreatedTabs.has(99)).toBe(false);
|
||||
resolveCreate({ id: 10, windowId: 100, active: true } as chrome.tabs.Tab);
|
||||
|
||||
await expect(pending).resolves.toMatchObject({ tab_id: 10 });
|
||||
expect(ctx.agentCreatedTabs.has(10)).toBe(true);
|
||||
expect(ctx.agentCreatedTabs.has(99)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("handleTabClose", () => {
|
||||
@@ -511,6 +549,148 @@ describe("handleTabBorrow", () => {
|
||||
});
|
||||
|
||||
describe("handleTabReturn", () => {
|
||||
it.each([
|
||||
"not_found",
|
||||
"cdp_failed",
|
||||
])("detaches after selector resolution fails with %s", async (code) => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.borrowedTabs.set(7, { tabId: 7, originalWindowId: 200, originalIndex: 4 });
|
||||
const state: FakeTabState = {
|
||||
tabs: new Map([[7, { id: 7, windowId: 100 } as chrome.tabs.Tab]]),
|
||||
nextTabId: 50,
|
||||
windowsClosed: new Set(),
|
||||
};
|
||||
const { api } = makeTabMutationApi(state);
|
||||
const { api: windows } = makeWindowsApi(state);
|
||||
const debuggerApi = {
|
||||
attach: vi.fn(async () => {}),
|
||||
detach: vi.fn(async () => {}),
|
||||
sendCommand: vi.fn(async (_target: unknown, method: string) => {
|
||||
if (method === "DOM.getDocument") return { root: { nodeId: 1 } };
|
||||
if (method === "DOM.querySelector") {
|
||||
if (code === "cdp_failed") throw new Error("selector query failed");
|
||||
return { nodeId: 0 };
|
||||
}
|
||||
return {};
|
||||
}),
|
||||
onEvent: { addListener: vi.fn(), removeListener: vi.fn() },
|
||||
onDetach: { addListener: vi.fn(), removeListener: vi.fn() },
|
||||
};
|
||||
const cdp = new ChromiumCdp(debuggerApi as unknown as CdpDebuggerApi);
|
||||
|
||||
const hover = await handleHover(
|
||||
sm,
|
||||
{ session_id: "aa11", tab_id: 7, selector: "#missing" },
|
||||
{
|
||||
cdp,
|
||||
tabsApi: { get: api.get, query: vi.fn(async () => []) },
|
||||
},
|
||||
);
|
||||
expect(hover).toMatchObject({ code });
|
||||
expect(cdp.isAttached(7)).toBe(true);
|
||||
|
||||
const returned = await handleTabReturn(
|
||||
sm,
|
||||
{ session_id: "aa11", tab_id: 7 },
|
||||
{
|
||||
tabs: api,
|
||||
windows,
|
||||
cdp,
|
||||
agentOverlayReset: { resetAgentOverlays: vi.fn(async () => {}) },
|
||||
},
|
||||
);
|
||||
expect(returned).toMatchObject({ returned_to_window_id: 200 });
|
||||
expect(ctx.borrowedTabs.has(7)).toBe(false);
|
||||
expect(cdp.isAttached(7)).toBe(false);
|
||||
expect(debuggerApi.detach).toHaveBeenCalledExactlyOnceWith({ tabId: 7 });
|
||||
cdp.dispose();
|
||||
});
|
||||
|
||||
it("honours cancellation while preparing a validated return", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.borrowedTabs.set(7, { tabId: 7, originalWindowId: 200, originalIndex: 4 });
|
||||
const state: FakeTabState = {
|
||||
tabs: new Map([[7, { id: 7, windowId: 100 } as chrome.tabs.Tab]]),
|
||||
nextTabId: 50,
|
||||
windowsClosed: new Set(),
|
||||
};
|
||||
const { api, spies } = makeTabMutationApi(state);
|
||||
const { api: windows } = makeWindowsApi(state);
|
||||
const controller = new AbortController();
|
||||
const beforeReturn = vi.fn(async () => {
|
||||
controller.abort();
|
||||
});
|
||||
const result = await handleTabReturn(
|
||||
sm,
|
||||
{ session_id: "aa11", tab_id: 7 },
|
||||
{
|
||||
tabs: api,
|
||||
windows,
|
||||
signal: controller.signal,
|
||||
beforeReturn,
|
||||
},
|
||||
);
|
||||
|
||||
expect(result).toMatchObject({ code: "cancelled" });
|
||||
expect(beforeReturn).toHaveBeenCalledExactlyOnceWith("aa11", 7);
|
||||
expect(spies.move).not.toHaveBeenCalled();
|
||||
expect(ctx.borrowedTabs.has(7)).toBe(true);
|
||||
});
|
||||
|
||||
it.each([
|
||||
"original",
|
||||
"fallback",
|
||||
"cleanup_failed",
|
||||
"failed",
|
||||
"cancelled",
|
||||
])("releases CDP only after a successful return (%s)", async (mode) => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.borrowedTabs.set(7, { tabId: 7, originalWindowId: 200, originalIndex: 4 });
|
||||
const state: FakeTabState = {
|
||||
tabs: new Map([[7, { id: 7, windowId: 100 } as chrome.tabs.Tab]]),
|
||||
nextTabId: 50,
|
||||
windowsClosed: new Set(),
|
||||
};
|
||||
const { api, spies } = makeTabMutationApi(state);
|
||||
const { api: windowsApi } = makeWindowsApi(state, { lastFocused: 500 });
|
||||
const success = mode !== "failed" && mode !== "cancelled";
|
||||
if (mode === "fallback") spies.move.mockRejectedValueOnce(new Error("original move failed"));
|
||||
if (mode === "failed") spies.move.mockRejectedValue(new Error("move failed"));
|
||||
let windowAtRelease: number | undefined;
|
||||
const cdp = {
|
||||
releaseSessionTab: vi.fn(async () => {
|
||||
windowAtRelease = state.tabs.get(7)?.windowId;
|
||||
if (mode === "cleanup_failed") throw new Error("debugger release failed");
|
||||
}),
|
||||
};
|
||||
const controller = new AbortController();
|
||||
if (mode === "cancelled") controller.abort();
|
||||
|
||||
const result = await handleTabReturn(
|
||||
sm,
|
||||
{ session_id: "aa11", tab_id: 7 },
|
||||
{
|
||||
tabs: api,
|
||||
windows: windowsApi,
|
||||
cdp,
|
||||
signal: controller.signal,
|
||||
agentOverlayReset: { resetAgentOverlays: vi.fn(async () => {}) },
|
||||
},
|
||||
);
|
||||
|
||||
expect("code" in result).toBe(!success);
|
||||
expect(ctx.borrowedTabs.has(7)).toBe(!success);
|
||||
expect(cdp.releaseSessionTab).toHaveBeenCalledTimes(success ? 1 : 0);
|
||||
if (success) {
|
||||
expect(cdp.releaseSessionTab).toHaveBeenCalledWith("aa11", 7);
|
||||
expect(windowAtRelease).toBe(mode === "fallback" ? 500 : 200);
|
||||
expect(spies.move).toHaveBeenCalledTimes(mode === "fallback" ? 2 : 1);
|
||||
}
|
||||
});
|
||||
|
||||
it("returns the tab to its original window/index and clears borrowedTabs", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
|
||||
@@ -12,7 +12,7 @@ function fakeAgentWindow(ids: number[]) {
|
||||
return id;
|
||||
}),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -11,7 +11,7 @@ function fakeAgentWindow(ids: number[]) {
|
||||
return id;
|
||||
});
|
||||
const remove = vi.fn(async () => {});
|
||||
const ensureActiveTab = vi.fn(async () => {});
|
||||
const ensureActiveTab = vi.fn(async () => 1);
|
||||
return { create, remove, ensureActiveTab };
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,110 @@
|
||||
import type { RpcError, WaitUntil } from "@/transport/types";
|
||||
import { cdpError } from "./errors";
|
||||
|
||||
interface NavigationEvent {
|
||||
tabId: number;
|
||||
frameId: number;
|
||||
documentId?: string;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
interface NavigationEvents {
|
||||
addListener(listener: (details: NavigationEvent) => void): void;
|
||||
removeListener(listener: (details: NavigationEvent) => void): void;
|
||||
}
|
||||
|
||||
/** Normal browser navigation remains available when chrome.debugger is denied. */
|
||||
export interface BrowserNavigationApi {
|
||||
update(tabId: number, props: { url: string }): Promise<unknown>;
|
||||
reload(tabId: number, props: { bypassCache: boolean }): Promise<void>;
|
||||
onCommitted: NavigationEvents;
|
||||
onDOMContentLoaded: NavigationEvents;
|
||||
onCompleted: NavigationEvents;
|
||||
onErrorOccurred: NavigationEvents;
|
||||
}
|
||||
|
||||
export const chromeBrowserNavigationApi: BrowserNavigationApi = {
|
||||
update: (tabId, props) => chrome.tabs.update(tabId, props),
|
||||
reload: (tabId, props) => chrome.tabs.reload(tabId, props),
|
||||
get onCommitted() {
|
||||
return chrome.webNavigation.onCommitted;
|
||||
},
|
||||
get onDOMContentLoaded() {
|
||||
return chrome.webNavigation.onDOMContentLoaded;
|
||||
},
|
||||
get onCompleted() {
|
||||
return chrome.webNavigation.onCompleted;
|
||||
},
|
||||
get onErrorOccurred() {
|
||||
return chrome.webNavigation.onErrorOccurred;
|
||||
},
|
||||
};
|
||||
|
||||
export type NavigationOutcome =
|
||||
| { reached: "match" | "timeout" | "cancelled"; lastLifecycle?: string }
|
||||
| { reached: "failed"; error: RpcError };
|
||||
|
||||
/** Subscribe before dispatch and ignore the departing document's load events. */
|
||||
export async function navigateWithBrowserApi(
|
||||
api: BrowserNavigationApi,
|
||||
tabId: number,
|
||||
action: () => Promise<unknown>,
|
||||
waitUntil: Exclude<WaitUntil, "networkidle">,
|
||||
timeoutMs: number,
|
||||
signal?: AbortSignal,
|
||||
): Promise<NavigationOutcome> {
|
||||
if (signal?.aborted) return { reached: "cancelled" };
|
||||
if (timeoutMs <= 0) return { reached: "timeout" };
|
||||
let committed = false;
|
||||
let documentId: string | undefined;
|
||||
let lastLifecycle: string | undefined;
|
||||
let finish!: (outcome: NavigationOutcome) => void;
|
||||
const outcome = new Promise<NavigationOutcome>((resolve) => {
|
||||
finish = resolve;
|
||||
});
|
||||
const isMainFrame = (details: NavigationEvent) =>
|
||||
details.tabId === tabId && details.frameId === 0;
|
||||
const onCommitted = (details: NavigationEvent) => {
|
||||
if (!isMainFrame(details)) return;
|
||||
committed = true;
|
||||
documentId = details.documentId;
|
||||
lastLifecycle = "commit";
|
||||
if (waitUntil === "commit") finish({ reached: "match", lastLifecycle });
|
||||
};
|
||||
const onDOMContentLoaded = (details: NavigationEvent) => {
|
||||
if (!isMainFrame(details) || !committed || (documentId && details.documentId !== documentId))
|
||||
return;
|
||||
lastLifecycle = "DOMContentLoaded";
|
||||
if (waitUntil === "domcontentloaded") finish({ reached: "match", lastLifecycle });
|
||||
};
|
||||
const onCompleted = (details: NavigationEvent) => {
|
||||
if (!isMainFrame(details) || !committed || (documentId && details.documentId !== documentId))
|
||||
return;
|
||||
finish({ reached: "match", lastLifecycle: "load" });
|
||||
};
|
||||
const onError = (details: NavigationEvent) => {
|
||||
if (!isMainFrame(details)) return;
|
||||
finish({ reached: "failed", error: cdpError(details.error ?? "browser navigation failed") });
|
||||
};
|
||||
const onAbort = () => finish({ reached: "cancelled", lastLifecycle });
|
||||
const subscriptions = [
|
||||
[api.onCommitted, onCommitted],
|
||||
[api.onDOMContentLoaded, onDOMContentLoaded],
|
||||
[api.onCompleted, onCompleted],
|
||||
[api.onErrorOccurred, onError],
|
||||
] as const;
|
||||
for (const [event, listener] of subscriptions) event.addListener(listener);
|
||||
signal?.addEventListener("abort", onAbort, { once: true });
|
||||
const timer = setTimeout(() => finish({ reached: "timeout", lastLifecycle }), timeoutMs);
|
||||
try {
|
||||
if (signal?.aborted) return { reached: "cancelled" };
|
||||
await action();
|
||||
return await outcome;
|
||||
} catch (error) {
|
||||
return { reached: "failed", error: cdpError(error) };
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
signal?.removeEventListener("abort", onAbort);
|
||||
for (const [event, listener] of subscriptions) event.removeListener(listener);
|
||||
}
|
||||
}
|
||||
@@ -36,6 +36,7 @@ import { isRequestFrame } from "@/transport/types";
|
||||
import { handleConsole } from "./console";
|
||||
import { handleDownload } from "./download";
|
||||
import { type EmulateCdpRunner, handleEmulate } from "./emulate";
|
||||
import { classifyCdpError } from "./errors";
|
||||
import { handleEvaluate } from "./evaluate";
|
||||
import { handleRequestHelp } from "./human-loop";
|
||||
import { handleClick, handleFill, handleHover, handlePress, handleSelect } from "./interaction";
|
||||
@@ -69,6 +70,7 @@ import {
|
||||
import { chromeTabsApi, lookupSession, resolveTargetTab } from "./shared";
|
||||
import {
|
||||
type BorrowConfirmationApprover,
|
||||
chromeTabMutationApi,
|
||||
handleTabBorrow,
|
||||
handleTabClose,
|
||||
handleTabCreate,
|
||||
@@ -118,6 +120,8 @@ export interface DispatcherDeps {
|
||||
onSessionsChanged?: () => void;
|
||||
/** Invoked before a tool that dispatches page input or mutates browser state is forwarded. */
|
||||
onBrowserControlResumed?: (sessionId: string) => void;
|
||||
/** Invoked after a tab is explicitly claimed so its overlay can be refreshed immediately. */
|
||||
onAgentTabClaimed?: (tabId: number, windowId: number) => void;
|
||||
/** User approval for `tool.tab_borrow` (overlay in content script). */
|
||||
approveBorrow?: BorrowConfirmationApprover;
|
||||
/** i18n notification copy for `tool.request_help` (resolved per-call). */
|
||||
@@ -147,6 +151,7 @@ export class ToolDispatcher {
|
||||
private readonly recording?: RecordRuntimeDeps;
|
||||
private readonly onSessionsChanged?: () => void;
|
||||
private readonly onBrowserControlResumed?: (sessionId: string) => void;
|
||||
private readonly onAgentTabClaimed?: (tabId: number, windowId: number) => void;
|
||||
private readonly approveBorrow?: BorrowConfirmationApprover;
|
||||
private readonly helpNotificationCopy?: () => { title: string; body: string };
|
||||
private subscription: { dispose(): void } | null = null;
|
||||
@@ -167,6 +172,7 @@ export class ToolDispatcher {
|
||||
this.recording = deps.recording;
|
||||
this.onSessionsChanged = deps.onSessionsChanged;
|
||||
this.onBrowserControlResumed = deps.onBrowserControlResumed;
|
||||
this.onAgentTabClaimed = deps.onAgentTabClaimed;
|
||||
this.approveBorrow = deps.approveBorrow;
|
||||
this.helpNotificationCopy = deps.helpNotificationCopy;
|
||||
}
|
||||
@@ -234,7 +240,7 @@ export class ToolDispatcher {
|
||||
if (sessionId) this.onBrowserControlResumed?.(sessionId);
|
||||
const result = await this.invoke(req, ac.signal);
|
||||
if (isRpcError(result)) {
|
||||
body = { id: req.id, error: result };
|
||||
body = { id: req.id, error: classifyCdpError(result) };
|
||||
} else {
|
||||
body = { id: req.id, result };
|
||||
if (req.method === "tool.session_start") {
|
||||
@@ -302,13 +308,25 @@ export class ToolDispatcher {
|
||||
await this.releaseHoverLatch((req.params as SessionStopParams).session_id);
|
||||
return handleSessionStop(this.sessions, req.params as SessionStopParams, {
|
||||
cdp: this.cdp,
|
||||
// Must be wired in production: the agent-tab cleanup and the
|
||||
// window-release decision (issue #57) read these deps directly
|
||||
// and silently no-op when they are absent.
|
||||
tabManagement: { tabs: chromeTabMutationApi },
|
||||
tabsQuery: chromeTabsApi,
|
||||
signal,
|
||||
});
|
||||
}
|
||||
case "tool.tab_list":
|
||||
return handleTabList(this.sessions, req.params as TabListParams, chromeTabsApi, signal);
|
||||
case "tool.tab_create":
|
||||
return handleTabCreate(this.sessions, req.params as TabCreateParams, { signal });
|
||||
case "tool.tab_create": {
|
||||
const result = await handleTabCreate(this.sessions, req.params as TabCreateParams, {
|
||||
signal,
|
||||
});
|
||||
if (!isRpcError(result)) {
|
||||
this.onAgentTabClaimed?.(result.tab_id, result.window_id);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
case "tool.tab_close":
|
||||
return this.withHoverReleaseForRequest(
|
||||
req.params as TabCloseParams,
|
||||
@@ -317,13 +335,22 @@ export class ToolDispatcher {
|
||||
);
|
||||
case "tool.tab_select":
|
||||
return handleTabSelect(this.sessions, req.params as TabSelectParams, { signal });
|
||||
case "tool.tab_borrow":
|
||||
return handleTabBorrow(this.sessions, req.params as TabBorrowParams, {
|
||||
case "tool.tab_borrow": {
|
||||
const result = await handleTabBorrow(this.sessions, req.params as TabBorrowParams, {
|
||||
signal,
|
||||
approveBorrow: this.approveBorrow,
|
||||
});
|
||||
if (!isRpcError(result)) {
|
||||
this.onAgentTabClaimed?.(result.tab_id, result.agent_window_id);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
case "tool.tab_return":
|
||||
return handleTabReturn(this.sessions, req.params as TabReturnParams, { signal });
|
||||
return handleTabReturn(this.sessions, req.params as TabReturnParams, {
|
||||
signal,
|
||||
cdp: this.cdp,
|
||||
beforeReturn: (sessionId, tabId) => this.releaseHoverLatch(sessionId, tabId),
|
||||
});
|
||||
case "tool.window_resize":
|
||||
return handleWindowResize(
|
||||
this.sessions,
|
||||
|
||||
@@ -9,8 +9,6 @@ import { type CdpRunner, isRpcError } from "./shared";
|
||||
|
||||
const CORRELATION_GRACE_MS = 750;
|
||||
const UNIQUE_SETTLE_MS = 50;
|
||||
const CLEANUP_RECONCILE_MS = 1_000;
|
||||
const CLEANUP_POLL_MS = 25;
|
||||
const SIZE_POLL_MS = 250;
|
||||
|
||||
type DeterminingFilenameListener = (
|
||||
@@ -112,35 +110,26 @@ function captureError(
|
||||
|
||||
async function cleanupClaimedDownload(downloads: DownloadsApi, downloadId: number): Promise<void> {
|
||||
const lookup = async () => (await downloads.search({ id: downloadId }))[0];
|
||||
let item = await lookup();
|
||||
const item = await lookup();
|
||||
if (!item || item.state === "interrupted") return;
|
||||
if (item.state === "complete") {
|
||||
await downloads.removeFile(downloadId);
|
||||
return;
|
||||
}
|
||||
|
||||
let cancelError: unknown;
|
||||
try {
|
||||
await downloads.cancel(downloadId);
|
||||
} catch (error) {
|
||||
cancelError = error;
|
||||
}
|
||||
|
||||
const deadline = Date.now() + CLEANUP_RECONCILE_MS;
|
||||
while (true) {
|
||||
item = await lookup();
|
||||
if (!item || item.state === "interrupted") return;
|
||||
if (item.state === "complete") {
|
||||
} catch (cancelError) {
|
||||
// Completion can win the race after the lookup but before cancellation.
|
||||
// Reconcile against Chrome's authoritative state before declaring cleanup
|
||||
// failed so every terminal state has one explicit cleanup path.
|
||||
const reconciled = await lookup();
|
||||
if (!reconciled || reconciled.state === "interrupted") return;
|
||||
if (reconciled.state === "complete") {
|
||||
await downloads.removeFile(downloadId);
|
||||
return;
|
||||
}
|
||||
if (Date.now() >= deadline) {
|
||||
if (cancelError instanceof Error) throw cancelError;
|
||||
throw new Error("download remained in progress after cancellation");
|
||||
}
|
||||
await new Promise((resolve) =>
|
||||
setTimeout(resolve, Math.min(CLEANUP_POLL_MS, Math.max(1, deadline - Date.now()))),
|
||||
);
|
||||
throw cancelError;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -13,6 +13,34 @@ import type {
|
||||
|
||||
export type { RpcErrorData, RpcErrorReason };
|
||||
|
||||
/** Chrome checks the whole tab's frame tree, including other extensions' frames. */
|
||||
export function isCdpExtensionAccessDenied(error: unknown): boolean {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
return message.includes("Cannot access a chrome-extension:// URL of different extension");
|
||||
}
|
||||
|
||||
/** Preserve more specific reasons, especially file-transfer outcome/cleanup details. */
|
||||
export function classifyCdpError(error: RpcError): RpcError {
|
||||
if (
|
||||
error.code !== "cdp_failed" ||
|
||||
error.data?.reason ||
|
||||
!isCdpExtensionAccessDenied(error.message)
|
||||
) {
|
||||
return error;
|
||||
}
|
||||
return {
|
||||
...error,
|
||||
data: { ...error.data, reason: "cdp_extension_access_denied" },
|
||||
};
|
||||
}
|
||||
|
||||
export function cdpError(error: unknown): RpcError {
|
||||
return classifyCdpError({
|
||||
code: "cdp_failed",
|
||||
message: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
}
|
||||
|
||||
export interface TransferErrorOptions {
|
||||
effectState: TransferEffectState;
|
||||
phase: string;
|
||||
|
||||
@@ -0,0 +1,319 @@
|
||||
// One native file-drop transaction bound to an explicit page target. CDP drag
|
||||
// events are sent to the target that owns the node: root targets use top-level
|
||||
// coordinates, while OOPIF targets require coordinates local to their session.
|
||||
|
||||
import type { RpcError, TransferEffectState } from "@/transport/types";
|
||||
import { transferError } from "./errors";
|
||||
import { resolveNodeGeometry } from "./frame-geometry";
|
||||
import type { ResolvedActionTarget } from "./interaction";
|
||||
import { type CdpRunner, isRpcError, sendToCdpTarget } from "./shared";
|
||||
import { BoundedWaitError, waitBounded } from "./transfer-transaction";
|
||||
|
||||
const CLEANUP_TIMEOUT_MS = 1_000;
|
||||
|
||||
type FileDropPhase = "resolve_target" | "drag_enter" | "drag_over" | "drop" | "cleanup";
|
||||
|
||||
interface RuntimeReply {
|
||||
result?: { value?: unknown; objectId?: string };
|
||||
exceptionDetails?: { text?: string; exception?: { description?: string } };
|
||||
}
|
||||
|
||||
interface DragData {
|
||||
items: [];
|
||||
files: string[];
|
||||
dragOperationsMask: number;
|
||||
}
|
||||
|
||||
export interface FileDropTransactionOptions {
|
||||
cdp: CdpRunner;
|
||||
actionTarget: ResolvedActionTarget;
|
||||
files: string[];
|
||||
timeoutMs: number;
|
||||
signal?: AbortSignal;
|
||||
withOverlayHidden: <T>(operation: () => Promise<T>) => Promise<T>;
|
||||
}
|
||||
|
||||
export interface FileDropTransactionResult {
|
||||
x: number;
|
||||
y: number;
|
||||
}
|
||||
|
||||
function failure(
|
||||
code: RpcError["code"],
|
||||
reason: "upload_mechanism_unsupported" | "file_drop_target_unavailable" | "file_drop_failed",
|
||||
message: string,
|
||||
effectState: TransferEffectState,
|
||||
phase: FileDropPhase,
|
||||
): RpcError {
|
||||
const error = transferError(code, reason, message, { effectState, phase });
|
||||
error.data = { ...error.data, mechanism: "drop" };
|
||||
return error;
|
||||
}
|
||||
|
||||
function enrichFailure(
|
||||
error: RpcError,
|
||||
effectState: TransferEffectState,
|
||||
phase: FileDropPhase,
|
||||
): RpcError {
|
||||
return {
|
||||
...error,
|
||||
data: {
|
||||
...error.data,
|
||||
mechanism: "drop",
|
||||
effect_state: effectState,
|
||||
phase,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function messageOf(error: unknown): string {
|
||||
return error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
|
||||
function methodUnavailable(error: unknown): boolean {
|
||||
return /(?:method|command).*(?:not found|wasn't found|unsupported)|wasn't found|-32601/i.test(
|
||||
messageOf(error),
|
||||
);
|
||||
}
|
||||
|
||||
function runtimeError(reply: RuntimeReply, fallback: string): Error | null {
|
||||
if (!reply.exceptionDetails) return null;
|
||||
return new Error(
|
||||
reply.exceptionDetails.exception?.description ?? reply.exceptionDetails.text ?? fallback,
|
||||
);
|
||||
}
|
||||
|
||||
async function pointBelongsToTarget(
|
||||
options: FileDropTransactionOptions,
|
||||
x: number,
|
||||
y: number,
|
||||
objectGroup: string,
|
||||
): Promise<boolean> {
|
||||
const target = options.actionTarget.cdpTarget;
|
||||
const hit = await sendToCdpTarget<{ backendNodeId?: number; frameId?: string }>(
|
||||
options.cdp,
|
||||
target,
|
||||
"DOM.getNodeForLocation",
|
||||
{ x: Math.round(x), y: Math.round(y), includeUserAgentShadowDOM: true },
|
||||
);
|
||||
if (typeof hit.backendNodeId !== "number") return false;
|
||||
if (options.actionTarget.frameId && hit.frameId !== options.actionTarget.frameId) return false;
|
||||
|
||||
const [resolvedTarget, resolvedHit] = await Promise.all([
|
||||
sendToCdpTarget<{ object?: { objectId?: string } }>(options.cdp, target, "DOM.resolveNode", {
|
||||
backendNodeId: options.actionTarget.backendNodeId,
|
||||
objectGroup,
|
||||
}),
|
||||
sendToCdpTarget<{ object?: { objectId?: string } }>(options.cdp, target, "DOM.resolveNode", {
|
||||
backendNodeId: hit.backendNodeId,
|
||||
objectGroup,
|
||||
}),
|
||||
]);
|
||||
const targetObjectId = resolvedTarget.object?.objectId;
|
||||
const hitObjectId = resolvedHit.object?.objectId;
|
||||
if (!targetObjectId || !hitObjectId) return false;
|
||||
|
||||
const reply = await sendToCdpTarget<RuntimeReply>(options.cdp, target, "Runtime.callFunctionOn", {
|
||||
objectId: targetObjectId,
|
||||
functionDeclaration: `function(hit) {
|
||||
for (let node = hit; node; ) {
|
||||
if (node === this) return true;
|
||||
if (this.contains?.(node)) return true;
|
||||
const root = node.getRootNode?.();
|
||||
node = node.parentNode || (root && root !== node ? root.host : null);
|
||||
}
|
||||
return false;
|
||||
}`,
|
||||
arguments: [{ objectId: hitObjectId }],
|
||||
returnByValue: true,
|
||||
});
|
||||
const error = runtimeError(reply, "failed to verify file-drop target");
|
||||
if (error) throw error;
|
||||
return reply.result?.value === true;
|
||||
}
|
||||
|
||||
export async function uploadThroughFileDrop(
|
||||
options: FileDropTransactionOptions,
|
||||
): Promise<FileDropTransactionResult | RpcError> {
|
||||
const deadline = Date.now() + options.timeoutMs;
|
||||
const target = options.actionTarget.cdpTarget;
|
||||
const objectGroup = `bsk-file-drop-${crypto.randomUUID()}`;
|
||||
const dragData: DragData = {
|
||||
items: [],
|
||||
files: options.files,
|
||||
dragOperationsMask: 1,
|
||||
};
|
||||
let dragStarted = false;
|
||||
let dropSent = false;
|
||||
let dragPoint = { x: 0, y: 0 };
|
||||
let cleanupFailed = false;
|
||||
let outcome: FileDropTransactionResult | RpcError = failure(
|
||||
"protocol_error",
|
||||
"file_drop_failed",
|
||||
"file-drop transaction ended without an outcome",
|
||||
"none",
|
||||
"cleanup",
|
||||
);
|
||||
|
||||
try {
|
||||
const geometry = await waitBounded(
|
||||
resolveNodeGeometry(
|
||||
options.cdp,
|
||||
options.actionTarget.tab.tabId,
|
||||
{
|
||||
target,
|
||||
backendNodeId: options.actionTarget.backendNodeId,
|
||||
...(options.actionTarget.frameId ? { frameId: options.actionTarget.frameId } : {}),
|
||||
},
|
||||
{ scrollIntoView: true },
|
||||
),
|
||||
deadline,
|
||||
options.signal,
|
||||
"resolving file-drop target timed out",
|
||||
);
|
||||
if (isRpcError(geometry)) {
|
||||
outcome = enrichFailure(geometry, "none", "resolve_target");
|
||||
return outcome;
|
||||
}
|
||||
const point = geometry.targetActionPoint;
|
||||
dragPoint = point;
|
||||
|
||||
outcome = await options.withOverlayHidden(async () => {
|
||||
try {
|
||||
try {
|
||||
const belongs = await waitBounded(
|
||||
pointBelongsToTarget(options, point.x, point.y, objectGroup),
|
||||
deadline,
|
||||
options.signal,
|
||||
"verifying file-drop target timed out",
|
||||
);
|
||||
if (!belongs) {
|
||||
return failure(
|
||||
"permission_denied",
|
||||
"file_drop_target_unavailable",
|
||||
"file-drop target is not the topmost element at its action point",
|
||||
"none",
|
||||
"resolve_target",
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
return failure(
|
||||
error instanceof BoundedWaitError ? "timeout" : "cdp_failed",
|
||||
"file_drop_target_unavailable",
|
||||
messageOf(error),
|
||||
"none",
|
||||
"resolve_target",
|
||||
);
|
||||
}
|
||||
|
||||
for (const [type, phase] of [
|
||||
["dragEnter", "drag_enter"],
|
||||
["dragOver", "drag_over"],
|
||||
] as const) {
|
||||
try {
|
||||
if (type === "dragEnter") dragStarted = true;
|
||||
await waitBounded(
|
||||
sendToCdpTarget(options.cdp, target, "Input.dispatchDragEvent", {
|
||||
type,
|
||||
x: point.x,
|
||||
y: point.y,
|
||||
data: dragData,
|
||||
}),
|
||||
deadline,
|
||||
options.signal,
|
||||
`${type} timed out`,
|
||||
);
|
||||
} catch (error) {
|
||||
const unsupported = methodUnavailable(error);
|
||||
if (type === "dragEnter" && unsupported) dragStarted = false;
|
||||
return failure(
|
||||
unsupported
|
||||
? "unsupported"
|
||||
: error instanceof BoundedWaitError
|
||||
? "timeout"
|
||||
: "cdp_failed",
|
||||
unsupported ? "upload_mechanism_unsupported" : "file_drop_failed",
|
||||
messageOf(error),
|
||||
"none",
|
||||
phase,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
dropSent = true;
|
||||
await waitBounded(
|
||||
sendToCdpTarget(options.cdp, target, "Input.dispatchDragEvent", {
|
||||
type: "drop",
|
||||
x: point.x,
|
||||
y: point.y,
|
||||
data: dragData,
|
||||
}),
|
||||
deadline,
|
||||
options.signal,
|
||||
"file drop timed out",
|
||||
);
|
||||
return { x: point.x, y: point.y };
|
||||
} catch (error) {
|
||||
outcome = failure(
|
||||
error instanceof BoundedWaitError ? "timeout" : "cdp_failed",
|
||||
"file_drop_failed",
|
||||
messageOf(error),
|
||||
"unknown",
|
||||
"drop",
|
||||
);
|
||||
return outcome;
|
||||
}
|
||||
} finally {
|
||||
if (dragStarted && !dropSent) {
|
||||
try {
|
||||
await waitBounded(
|
||||
sendToCdpTarget(options.cdp, target, "Input.dispatchDragEvent", {
|
||||
type: "dragCancel",
|
||||
x: dragPoint.x,
|
||||
y: dragPoint.y,
|
||||
data: dragData,
|
||||
}),
|
||||
Date.now() + CLEANUP_TIMEOUT_MS,
|
||||
undefined,
|
||||
"cancelling file drag timed out",
|
||||
);
|
||||
} catch {
|
||||
cleanupFailed = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
return outcome;
|
||||
} catch (error) {
|
||||
outcome = failure(
|
||||
error instanceof BoundedWaitError ? "timeout" : "cdp_failed",
|
||||
"file_drop_target_unavailable",
|
||||
messageOf(error),
|
||||
"none",
|
||||
"resolve_target",
|
||||
);
|
||||
return outcome;
|
||||
} finally {
|
||||
try {
|
||||
await waitBounded(
|
||||
sendToCdpTarget(options.cdp, target, "Runtime.releaseObjectGroup", { objectGroup }),
|
||||
Date.now() + CLEANUP_TIMEOUT_MS,
|
||||
undefined,
|
||||
"releasing file-drop object group timed out",
|
||||
);
|
||||
} catch {
|
||||
cleanupFailed = true;
|
||||
}
|
||||
|
||||
const effect = isRpcError(outcome)
|
||||
? (outcome.data?.effect_state as TransferEffectState | undefined)
|
||||
: "committed";
|
||||
if (effect === "unknown" || cleanupFailed) {
|
||||
await options.cdp.detach?.(target.tabId);
|
||||
}
|
||||
if (cleanupFailed && isRpcError(outcome)) {
|
||||
outcome.data = { ...outcome.data, cleanup_state: "failed" };
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -7,9 +7,9 @@ import type { ClickResult, RpcError, TransferEffectState } from "@/transport/typ
|
||||
import { transferError } from "./errors";
|
||||
import type { ResolvedActionTarget } from "./interaction";
|
||||
import { type CdpRunner, isRpcError, sendToCdpTarget } from "./shared";
|
||||
import { BoundedWaitError, remainingMs, waitBounded } from "./transfer-transaction";
|
||||
|
||||
const CLEANUP_TIMEOUT_MS = 1_000;
|
||||
const DETACH_TIMEOUT_MS = 500;
|
||||
const ACTIVATION_GRACE_MS = 1_000;
|
||||
const PROBE_INTERVAL_MS = 20;
|
||||
|
||||
@@ -46,51 +46,6 @@ export interface FileInputTransactionResult {
|
||||
multiple: boolean;
|
||||
}
|
||||
|
||||
class BoundedWaitError extends Error {
|
||||
constructor(
|
||||
readonly kind: "timeout" | "aborted",
|
||||
message: string,
|
||||
) {
|
||||
super(message);
|
||||
}
|
||||
}
|
||||
|
||||
function remainingMs(deadline: number): number {
|
||||
return Math.max(0, deadline - Date.now());
|
||||
}
|
||||
|
||||
async function waitBounded<T>(
|
||||
operation: () => Promise<T>,
|
||||
deadline: number,
|
||||
signal: AbortSignal | undefined,
|
||||
timeoutMessage: string,
|
||||
): Promise<T> {
|
||||
const remaining = remainingMs(deadline);
|
||||
if (signal?.aborted) throw new BoundedWaitError("aborted", "upload transaction aborted");
|
||||
if (remaining === 0) throw new BoundedWaitError("timeout", timeoutMessage);
|
||||
|
||||
let timer: ReturnType<typeof setTimeout> | undefined;
|
||||
let onAbort: (() => void) | undefined;
|
||||
const boundary = new Promise<never>((_resolve, reject) => {
|
||||
timer = setTimeout(() => reject(new BoundedWaitError("timeout", timeoutMessage)), remaining);
|
||||
if (signal) {
|
||||
onAbort = () => reject(new BoundedWaitError("aborted", "upload transaction aborted"));
|
||||
signal.addEventListener("abort", onAbort, { once: true });
|
||||
}
|
||||
});
|
||||
try {
|
||||
const work = Promise.resolve().then(() => {
|
||||
if (signal?.aborted) throw new BoundedWaitError("aborted", "upload transaction aborted");
|
||||
if (remainingMs(deadline) === 0) throw new BoundedWaitError("timeout", timeoutMessage);
|
||||
return operation();
|
||||
});
|
||||
return await Promise.race([work, boundary]);
|
||||
} finally {
|
||||
if (timer) clearTimeout(timer);
|
||||
if (signal && onAbort) signal.removeEventListener("abort", onAbort);
|
||||
}
|
||||
}
|
||||
|
||||
function runtimeError(reply: RuntimeReply, fallback: string): Error | null {
|
||||
if (!reply.exceptionDetails) return null;
|
||||
return new Error(
|
||||
@@ -128,18 +83,13 @@ function sameTarget(
|
||||
|
||||
async function delay(ms: number, signal?: AbortSignal): Promise<void> {
|
||||
await waitBounded(
|
||||
() => new Promise((resolve) => setTimeout(resolve, ms)),
|
||||
new Promise((resolve) => setTimeout(resolve, ms)),
|
||||
Date.now() + ms + 1,
|
||||
signal,
|
||||
"upload activation probe timed out",
|
||||
);
|
||||
}
|
||||
|
||||
function waitErrorCode(error: unknown): RpcError["code"] {
|
||||
if (!(error instanceof BoundedWaitError)) return "cdp_failed";
|
||||
return error.kind === "aborted" ? "cancelled" : "timeout";
|
||||
}
|
||||
|
||||
async function callOnTrigger<T = RuntimeReply>(
|
||||
options: FileInputTransactionOptions,
|
||||
objectId: string,
|
||||
@@ -164,8 +114,8 @@ export async function uploadThroughActivatedFileInput(
|
||||
const objectGroup = `bsk-upload-${crypto.randomUUID()}`;
|
||||
const stateKey = `__bskUpload_${crypto.randomUUID().replaceAll("-", "")}`;
|
||||
const chooserEvents: ChooserEvent[] = [];
|
||||
let interceptionState: "not_started" | "possibly_active" | "active" = "not_started";
|
||||
let probeState: "not_started" | "possibly_active" | "active" = "not_started";
|
||||
let interceptionArmed = false;
|
||||
let probeArmed = false;
|
||||
let triggerObjectId: string | undefined;
|
||||
let outcome: FileInputTransactionResult | RpcError = transferFailure(
|
||||
"protocol_error",
|
||||
@@ -183,20 +133,18 @@ export async function uploadThroughActivatedFileInput(
|
||||
|
||||
try {
|
||||
try {
|
||||
interceptionState = "possibly_active";
|
||||
await waitBounded(
|
||||
() =>
|
||||
sendToCdpTarget(options.cdp, target, "Page.setInterceptFileChooserDialog", {
|
||||
enabled: true,
|
||||
}),
|
||||
sendToCdpTarget(options.cdp, target, "Page.setInterceptFileChooserDialog", {
|
||||
enabled: true,
|
||||
}),
|
||||
deadline,
|
||||
options.signal,
|
||||
"arming native file chooser interception timed out",
|
||||
);
|
||||
interceptionState = "active";
|
||||
interceptionArmed = true;
|
||||
} catch (err) {
|
||||
outcome = transferFailure(
|
||||
waitErrorCode(err),
|
||||
err instanceof BoundedWaitError ? "timeout" : "cdp_failed",
|
||||
"file_input_probe_failed",
|
||||
err instanceof Error ? err.message : String(err),
|
||||
"none",
|
||||
@@ -207,16 +155,15 @@ export async function uploadThroughActivatedFileInput(
|
||||
|
||||
try {
|
||||
const resolved = await waitBounded(
|
||||
() =>
|
||||
sendToCdpTarget<{ object?: { objectId?: string } }>(
|
||||
options.cdp,
|
||||
target,
|
||||
"DOM.resolveNode",
|
||||
{
|
||||
backendNodeId: options.actionTarget.backendNodeId,
|
||||
objectGroup,
|
||||
},
|
||||
),
|
||||
sendToCdpTarget<{ object?: { objectId?: string } }>(
|
||||
options.cdp,
|
||||
target,
|
||||
"DOM.resolveNode",
|
||||
{
|
||||
backendNodeId: options.actionTarget.backendNodeId,
|
||||
objectGroup,
|
||||
},
|
||||
),
|
||||
deadline,
|
||||
options.signal,
|
||||
"resolving upload trigger timed out",
|
||||
@@ -224,13 +171,11 @@ export async function uploadThroughActivatedFileInput(
|
||||
triggerObjectId = resolved.object?.objectId;
|
||||
if (!triggerObjectId) throw new Error("DOM.resolveNode returned no trigger objectId");
|
||||
|
||||
probeState = "possibly_active";
|
||||
const armed = await waitBounded(
|
||||
() =>
|
||||
callOnTrigger<RuntimeReply>(
|
||||
options,
|
||||
triggerObjectId as string,
|
||||
`function(key) {
|
||||
callOnTrigger<RuntimeReply>(
|
||||
options,
|
||||
triggerObjectId,
|
||||
`function(key) {
|
||||
const doc = this.ownerDocument;
|
||||
const owner = doc.defaultView;
|
||||
if (!owner) return false;
|
||||
@@ -248,9 +193,9 @@ export async function uploadThroughActivatedFileInput(
|
||||
doc.addEventListener("click", state.listener, true);
|
||||
return true;
|
||||
}`,
|
||||
[stateKey],
|
||||
true,
|
||||
),
|
||||
[stateKey],
|
||||
true,
|
||||
),
|
||||
deadline,
|
||||
options.signal,
|
||||
"arming frame-scoped file input probe timed out",
|
||||
@@ -259,10 +204,10 @@ export async function uploadThroughActivatedFileInput(
|
||||
if (armError || armed.result?.value !== true) {
|
||||
throw armError ?? new Error("file input probe did not arm");
|
||||
}
|
||||
probeState = "active";
|
||||
probeArmed = true;
|
||||
} catch (err) {
|
||||
outcome = transferFailure(
|
||||
waitErrorCode(err),
|
||||
err instanceof BoundedWaitError ? "timeout" : "cdp_failed",
|
||||
"file_input_probe_failed",
|
||||
err instanceof Error ? err.message : String(err),
|
||||
"none",
|
||||
@@ -274,14 +219,14 @@ export async function uploadThroughActivatedFileInput(
|
||||
let click: ClickResult | RpcError;
|
||||
try {
|
||||
click = await waitBounded(
|
||||
options.trigger,
|
||||
options.trigger(),
|
||||
deadline,
|
||||
options.signal,
|
||||
"upload trigger timed out",
|
||||
);
|
||||
} catch (err) {
|
||||
outcome = transferFailure(
|
||||
waitErrorCode(err),
|
||||
err instanceof BoundedWaitError ? "timeout" : "cdp_failed",
|
||||
"file_input_probe_failed",
|
||||
err instanceof Error ? err.message : String(err),
|
||||
"none",
|
||||
@@ -299,23 +244,17 @@ export async function uploadThroughActivatedFileInput(
|
||||
let summary: { count: number; multiple: boolean } = { count: 0, multiple: false };
|
||||
while (Date.now() < activationDeadline) {
|
||||
if (chooserEvents.length > 0) break;
|
||||
const reply = await waitBounded(
|
||||
() =>
|
||||
callOnTrigger<RuntimeReply>(
|
||||
options,
|
||||
triggerObjectId as string,
|
||||
`function(key) {
|
||||
const reply = await callOnTrigger<RuntimeReply>(
|
||||
options,
|
||||
triggerObjectId,
|
||||
`function(key) {
|
||||
const state = this.ownerDocument.defaultView?.[key];
|
||||
return state
|
||||
? { count: state.inputs.length, multiple: state.inputs[0]?.multiple === true }
|
||||
: { count: 0, multiple: false };
|
||||
}`,
|
||||
[stateKey],
|
||||
true,
|
||||
),
|
||||
activationDeadline,
|
||||
options.signal,
|
||||
"inspecting activated file input timed out",
|
||||
[stateKey],
|
||||
true,
|
||||
);
|
||||
const summaryError = runtimeError(reply, "failed to inspect activated file input");
|
||||
if (summaryError) throw summaryError;
|
||||
@@ -378,33 +317,21 @@ export async function uploadThroughActivatedFileInput(
|
||||
);
|
||||
return outcome;
|
||||
}
|
||||
const input = await waitBounded(
|
||||
() =>
|
||||
callOnTrigger<RuntimeReply>(
|
||||
options,
|
||||
triggerObjectId as string,
|
||||
`function(key) { return this.ownerDocument.defaultView?.[key]?.inputs[0]; }`,
|
||||
[stateKey],
|
||||
false,
|
||||
),
|
||||
deadline,
|
||||
options.signal,
|
||||
"resolving activated file input timed out",
|
||||
const input = await callOnTrigger<RuntimeReply>(
|
||||
options,
|
||||
triggerObjectId,
|
||||
`function(key) { return this.ownerDocument.defaultView?.[key]?.inputs[0]; }`,
|
||||
[stateKey],
|
||||
false,
|
||||
);
|
||||
const inputError = runtimeError(input, "failed to resolve activated file input object");
|
||||
if (inputError) throw inputError;
|
||||
if (!input.result?.objectId) throw new Error("activated file input returned no objectId");
|
||||
const described = await waitBounded(
|
||||
() =>
|
||||
sendToCdpTarget<{ node?: { backendNodeId?: number } }>(
|
||||
options.cdp,
|
||||
target,
|
||||
"DOM.describeNode",
|
||||
{ objectId: input.result?.objectId },
|
||||
),
|
||||
deadline,
|
||||
options.signal,
|
||||
"describing activated file input timed out",
|
||||
const described = await sendToCdpTarget<{ node?: { backendNodeId?: number } }>(
|
||||
options.cdp,
|
||||
target,
|
||||
"DOM.describeNode",
|
||||
{ objectId: input.result.objectId },
|
||||
);
|
||||
backendNodeId = described.node?.backendNodeId;
|
||||
if (typeof backendNodeId !== "number") {
|
||||
@@ -423,18 +350,17 @@ export async function uploadThroughActivatedFileInput(
|
||||
|
||||
try {
|
||||
await waitBounded(
|
||||
() =>
|
||||
sendToCdpTarget(options.cdp, target, "DOM.setFileInputFiles", {
|
||||
files: options.files,
|
||||
backendNodeId,
|
||||
}),
|
||||
sendToCdpTarget(options.cdp, target, "DOM.setFileInputFiles", {
|
||||
files: options.files,
|
||||
backendNodeId,
|
||||
}),
|
||||
deadline,
|
||||
options.signal,
|
||||
"setting file input files timed out",
|
||||
);
|
||||
} catch (err) {
|
||||
outcome = transferFailure(
|
||||
waitErrorCode(err),
|
||||
err instanceof BoundedWaitError ? "timeout" : "cdp_failed",
|
||||
"set_file_input_failed",
|
||||
err instanceof Error ? err.message : String(err),
|
||||
"unknown",
|
||||
@@ -446,7 +372,7 @@ export async function uploadThroughActivatedFileInput(
|
||||
return outcome;
|
||||
} catch (err) {
|
||||
outcome = transferFailure(
|
||||
waitErrorCode(err),
|
||||
err instanceof BoundedWaitError ? "timeout" : "cdp_failed",
|
||||
"file_input_probe_failed",
|
||||
err instanceof Error ? err.message : String(err),
|
||||
"none",
|
||||
@@ -456,70 +382,61 @@ export async function uploadThroughActivatedFileInput(
|
||||
}
|
||||
} finally {
|
||||
chooserSubscription?.dispose();
|
||||
const cleanupDeadline = Date.now() + CLEANUP_TIMEOUT_MS;
|
||||
const cleanupTasks: Promise<boolean>[] = [];
|
||||
const cleanup = async (operation: () => Promise<unknown>, message: string) => {
|
||||
let cleanupFailed = false;
|
||||
if (probeArmed && triggerObjectId) {
|
||||
try {
|
||||
await waitBounded(operation, cleanupDeadline, undefined, message);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
};
|
||||
if (probeState !== "not_started" && triggerObjectId) {
|
||||
cleanupTasks.push(
|
||||
cleanup(
|
||||
() =>
|
||||
callOnTrigger(
|
||||
options,
|
||||
triggerObjectId as string,
|
||||
`function(key) {
|
||||
await waitBounded(
|
||||
callOnTrigger(
|
||||
options,
|
||||
triggerObjectId,
|
||||
`function(key) {
|
||||
const owner = this.ownerDocument.defaultView;
|
||||
const state = owner?.[key];
|
||||
if (state?.listener) this.ownerDocument.removeEventListener("click", state.listener, true);
|
||||
if (owner) delete owner[key];
|
||||
}`,
|
||||
[stateKey],
|
||||
true,
|
||||
),
|
||||
[stateKey],
|
||||
true,
|
||||
),
|
||||
Date.now() + CLEANUP_TIMEOUT_MS,
|
||||
undefined,
|
||||
"cleaning file input probe timed out",
|
||||
),
|
||||
);
|
||||
);
|
||||
} catch {
|
||||
cleanupFailed = true;
|
||||
}
|
||||
}
|
||||
if (interceptionState !== "not_started") {
|
||||
cleanupTasks.push(
|
||||
cleanup(
|
||||
() =>
|
||||
sendToCdpTarget(options.cdp, target, "Page.setInterceptFileChooserDialog", {
|
||||
enabled: false,
|
||||
cancel: true,
|
||||
}),
|
||||
if (interceptionArmed) {
|
||||
try {
|
||||
await waitBounded(
|
||||
sendToCdpTarget(options.cdp, target, "Page.setInterceptFileChooserDialog", {
|
||||
enabled: false,
|
||||
cancel: true,
|
||||
}),
|
||||
Date.now() + CLEANUP_TIMEOUT_MS,
|
||||
undefined,
|
||||
"disabling file chooser interception timed out",
|
||||
),
|
||||
);
|
||||
);
|
||||
} catch {
|
||||
cleanupFailed = true;
|
||||
}
|
||||
}
|
||||
let cleanupFailed = (await Promise.all(cleanupTasks)).some((completed) => !completed);
|
||||
if (!cleanupFailed) {
|
||||
cleanupFailed = !(await cleanup(
|
||||
() => sendToCdpTarget(options.cdp, target, "Runtime.releaseObjectGroup", { objectGroup }),
|
||||
try {
|
||||
await waitBounded(
|
||||
sendToCdpTarget(options.cdp, target, "Runtime.releaseObjectGroup", { objectGroup }),
|
||||
Date.now() + CLEANUP_TIMEOUT_MS,
|
||||
undefined,
|
||||
"releasing upload object group timed out",
|
||||
));
|
||||
);
|
||||
} catch {
|
||||
cleanupFailed = true;
|
||||
}
|
||||
|
||||
const effect = isRpcError(outcome)
|
||||
? (outcome.data?.effect_state as TransferEffectState | undefined)
|
||||
: "committed";
|
||||
if (effect === "unknown" || cleanupFailed) {
|
||||
try {
|
||||
await waitBounded(
|
||||
() => options.cdp.detach?.(target.tabId) ?? Promise.resolve(),
|
||||
Date.now() + DETACH_TIMEOUT_MS,
|
||||
undefined,
|
||||
"detaching upload target timed out",
|
||||
);
|
||||
} catch {
|
||||
cleanupFailed = true;
|
||||
}
|
||||
await options.cdp.detach?.(target.tabId);
|
||||
}
|
||||
if (cleanupFailed && isRpcError(outcome)) {
|
||||
outcome.data = { ...outcome.data, cleanup_state: "failed" };
|
||||
|
||||
@@ -1,24 +1,21 @@
|
||||
import { type CdpFrame, type CdpFrameGraph, type CdpTarget } from "@/browser-driver/frame-graph";
|
||||
import { type CdpTarget, cdpTargetKey } from "@/browser-driver/frame-graph";
|
||||
import type { RpcError } from "@/transport/types";
|
||||
import { nodeContentRegion, scrollNodeIntoView } from "./element-geometry";
|
||||
import {
|
||||
clipPolygon,
|
||||
type GeometryProjection,
|
||||
type Point,
|
||||
type Polygon,
|
||||
type ProjectiveEdge,
|
||||
parseCdpQuad,
|
||||
polygonArea,
|
||||
polygonCentroid,
|
||||
projectRegionToViewport,
|
||||
type Quad,
|
||||
type Region,
|
||||
rectPolygon,
|
||||
regionBounds,
|
||||
type Size,
|
||||
type ViewportRect,
|
||||
} from "./geometry";
|
||||
import { type CdpRunner, cdpRunnerForTarget, isRpcError, sendToCdpTarget } from "./shared";
|
||||
import type { CssViewport } from "./geometry/coordinate-types";
|
||||
import { cssViewport, GeometryContext } from "./geometry/frame-context";
|
||||
import { type CdpRunner, cdpRunnerForTarget, isRpcError } from "./shared";
|
||||
|
||||
export interface NodeAddress {
|
||||
target: CdpTarget;
|
||||
@@ -27,154 +24,30 @@ export interface NodeAddress {
|
||||
}
|
||||
|
||||
export interface ResolvedNodeGeometry {
|
||||
/** The same top-level measurement used by projection, for screenshot coordinate adaptation. */
|
||||
topViewport: CssViewport;
|
||||
topVisibleRegions: Region;
|
||||
topBounds: ViewportRect;
|
||||
/** Point in the top-level tab viewport, used by root-target input events. */
|
||||
actionPoint: Point;
|
||||
/** Point in the addressed CDP target's viewport, used by OOPIF-local input events. */
|
||||
targetActionPoint: Point;
|
||||
}
|
||||
|
||||
function geometryError(message: string): RpcError {
|
||||
return { code: "cdp_failed", message };
|
||||
}
|
||||
|
||||
function frameMap(graph: CdpFrameGraph): Map<string, CdpFrame> {
|
||||
return new Map(graph.frames.map((frame) => [frame.frameId, frame]));
|
||||
}
|
||||
|
||||
function targetRootFrame(byId: Map<string, CdpFrame>, frame: CdpFrame): CdpFrame | null {
|
||||
const seen = new Set<string>();
|
||||
let current = frame;
|
||||
while (current.parentFrameId) {
|
||||
if (seen.has(current.frameId)) return null;
|
||||
seen.add(current.frameId);
|
||||
const parent = byId.get(current.parentFrameId);
|
||||
if (!parent || parent.target.sessionId !== current.target.sessionId) break;
|
||||
current = parent;
|
||||
}
|
||||
return current;
|
||||
}
|
||||
|
||||
function frameAncestry(graph: CdpFrameGraph, frameId: string): CdpFrame[] | null {
|
||||
const byId = frameMap(graph);
|
||||
const path: CdpFrame[] = [];
|
||||
const seen = new Set<string>();
|
||||
let current = byId.get(frameId);
|
||||
if (!current) return null;
|
||||
while (current.parentFrameId) {
|
||||
if (seen.has(current.frameId)) return null;
|
||||
seen.add(current.frameId);
|
||||
path.push(current);
|
||||
const parent = byId.get(current.parentFrameId);
|
||||
if (!parent) return null;
|
||||
current = parent;
|
||||
}
|
||||
return path;
|
||||
}
|
||||
|
||||
async function targetViewport(cdp: CdpRunner, target: CdpTarget): Promise<Size | null> {
|
||||
const metrics = await sendToCdpTarget<{
|
||||
cssLayoutViewport?: { clientWidth?: number; clientHeight?: number };
|
||||
layoutViewport?: { clientWidth?: number; clientHeight?: number };
|
||||
}>(cdp, target, "Page.getLayoutMetrics", {});
|
||||
const viewport = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
|
||||
const width = viewport.clientWidth ?? 0;
|
||||
const height = viewport.clientHeight ?? 0;
|
||||
return width > 0 && height > 0 ? { width, height } : null;
|
||||
}
|
||||
|
||||
async function ownerContentQuad(
|
||||
cdp: CdpRunner,
|
||||
parent: CdpFrame,
|
||||
ownerBackendNodeId: number,
|
||||
): Promise<Quad | null> {
|
||||
const result = await sendToCdpTarget<{ model?: { content?: number[] } }>(
|
||||
cdp,
|
||||
parent.target,
|
||||
"DOM.getBoxModel",
|
||||
{ backendNodeId: ownerBackendNodeId },
|
||||
);
|
||||
return parseCdpQuad(result.model?.content);
|
||||
}
|
||||
|
||||
async function sameTargetFrameClips(
|
||||
cdp: CdpRunner,
|
||||
byId: Map<string, CdpFrame>,
|
||||
frame: CdpFrame,
|
||||
targetRoot: CdpFrame,
|
||||
): Promise<Polygon[] | null> {
|
||||
const clips: Polygon[] = [];
|
||||
let current = frame;
|
||||
const seen = new Set<string>();
|
||||
while (current.frameId !== targetRoot.frameId) {
|
||||
if (seen.has(current.frameId) || !current.parentFrameId) return null;
|
||||
seen.add(current.frameId);
|
||||
const parent = byId.get(current.parentFrameId);
|
||||
if (!parent || current.ownerBackendNodeId === undefined) return null;
|
||||
const clip = await ownerContentQuad(cdp, parent, current.ownerBackendNodeId);
|
||||
if (!clip) return null;
|
||||
clips.push(clip);
|
||||
current = parent;
|
||||
}
|
||||
return clips;
|
||||
}
|
||||
|
||||
export async function resolveFrameProjection(
|
||||
cdp: CdpRunner,
|
||||
graph: CdpFrameGraph,
|
||||
frameId: string,
|
||||
): Promise<GeometryProjection | null> {
|
||||
const byId = frameMap(graph);
|
||||
const frame = byId.get(frameId);
|
||||
if (!frame) return null;
|
||||
let targetRoot = targetRootFrame(byId, frame);
|
||||
if (!targetRoot) return null;
|
||||
const sourceViewport = await targetViewport(cdp, targetRoot.target);
|
||||
if (!sourceViewport) return null;
|
||||
const sourceClips = await sameTargetFrameClips(cdp, byId, frame, targetRoot);
|
||||
if (!sourceClips) return null;
|
||||
|
||||
const edges: ProjectiveEdge[] = [];
|
||||
while (targetRoot.parentFrameId) {
|
||||
const parent = byId.get(targetRoot.parentFrameId);
|
||||
if (!parent || targetRoot.ownerBackendNodeId === undefined) return null;
|
||||
const destinationQuad = await ownerContentQuad(cdp, parent, targetRoot.ownerBackendNodeId);
|
||||
if (!destinationQuad) return null;
|
||||
const source =
|
||||
edges.length === 0 ? sourceViewport : await targetViewport(cdp, targetRoot.target);
|
||||
if (!source) return null;
|
||||
const parentTargetRoot = targetRootFrame(byId, parent);
|
||||
if (!parentTargetRoot) return null;
|
||||
const destinationClips = await sameTargetFrameClips(cdp, byId, parent, parentTargetRoot);
|
||||
if (!destinationClips) return null;
|
||||
edges.push({ sourceViewport: source, destinationQuad, destinationClips });
|
||||
targetRoot = parentTargetRoot;
|
||||
}
|
||||
|
||||
const topViewport =
|
||||
edges.length === 0 ? sourceViewport : await targetViewport(cdp, targetRoot.target);
|
||||
return topViewport ? { sourceClips, edges, topViewport } : null;
|
||||
}
|
||||
|
||||
async function loadFrameGraph(cdp: CdpRunner, tabId: number): Promise<CdpFrameGraph | null> {
|
||||
if (!cdp.getFrameGraph) return null;
|
||||
try {
|
||||
return await cdp.getFrameGraph(tabId);
|
||||
} catch (error) {
|
||||
console.debug("[bsk frame-geometry] frame graph resolution failed", error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function scrollFrameOwners(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
graph: CdpFrameGraph,
|
||||
context: GeometryContext,
|
||||
frameId: string,
|
||||
): Promise<RpcError | null> {
|
||||
const byId = frameMap(graph);
|
||||
const ancestry = frameAncestry(graph, frameId);
|
||||
const ancestry = await context.ancestry(frameId);
|
||||
if (!ancestry) return geometryError(`could not resolve frame ancestry for ${frameId}`);
|
||||
for (const child of [...ancestry].reverse()) {
|
||||
const parent = child.parentFrameId ? byId.get(child.parentFrameId) : undefined;
|
||||
const parent = child.parentFrameId ? await context.frame(child.parentFrameId) : undefined;
|
||||
if (!parent || child.ownerBackendNodeId === undefined) {
|
||||
return geometryError(`could not resolve frame owner for ${child.frameId}`);
|
||||
}
|
||||
@@ -188,17 +61,20 @@ async function scrollFrameOwners(
|
||||
return null;
|
||||
}
|
||||
|
||||
async function scrollElementWithFrameGraph(
|
||||
async function scrollElementWithContext(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
target: CdpTarget,
|
||||
backendNodeId: number,
|
||||
frameId: string | undefined,
|
||||
graph: CdpFrameGraph | null,
|
||||
context: GeometryContext,
|
||||
): Promise<RpcError | null> {
|
||||
if (frameId) {
|
||||
if (!graph) return geometryError(`could not resolve frame graph for ${frameId}`);
|
||||
const error = await scrollFrameOwners(cdp, tabId, graph, frameId);
|
||||
const frame = await context.frame(frameId);
|
||||
if (!frame || cdpTargetKey(frame.target) !== cdpTargetKey(target)) {
|
||||
return geometryError("node frame does not belong to its target");
|
||||
}
|
||||
const error = await scrollFrameOwners(cdp, tabId, context, frameId);
|
||||
if (error) return error;
|
||||
} else if (target.sessionId) {
|
||||
return geometryError("an OOPIF node address requires frameId");
|
||||
@@ -213,8 +89,15 @@ export async function scrollElementAndFramesIntoView(
|
||||
backendNodeId: number,
|
||||
frameId?: string,
|
||||
): Promise<RpcError | null> {
|
||||
const graph = frameId ? await loadFrameGraph(cdp, tabId) : null;
|
||||
return scrollElementWithFrameGraph(cdp, tabId, target, backendNodeId, frameId, graph);
|
||||
if (target.tabId !== tabId) return geometryError("node target belongs to another tab");
|
||||
return scrollElementWithContext(
|
||||
cdp,
|
||||
tabId,
|
||||
target,
|
||||
backendNodeId,
|
||||
frameId,
|
||||
new GeometryContext(cdp, tabId),
|
||||
);
|
||||
}
|
||||
|
||||
function largestRegion(regions: Region): Polygon | null {
|
||||
@@ -237,23 +120,32 @@ export async function resolveNodeGeometry(
|
||||
if (address.target.sessionId && !address.frameId) {
|
||||
return geometryError("an OOPIF node address requires frameId");
|
||||
}
|
||||
const graph = address.frameId ? await loadFrameGraph(cdp, tabId) : null;
|
||||
if (address.target.tabId !== tabId) return geometryError("node target belongs to another tab");
|
||||
const context = new GeometryContext(cdp, tabId);
|
||||
const graph = address.frameId ? await context.graph() : null;
|
||||
if (address.frameId && graph) {
|
||||
const frame = await context.frame(address.frameId);
|
||||
if (!frame || cdpTargetKey(frame.target) !== cdpTargetKey(address.target)) {
|
||||
return geometryError("node frame does not belong to its target");
|
||||
}
|
||||
}
|
||||
if (address.frameId && !graph) {
|
||||
return geometryError(`could not resolve frame graph for ${address.frameId}`);
|
||||
}
|
||||
|
||||
if (options.scrollIntoView) {
|
||||
const scrollError = await scrollElementWithFrameGraph(
|
||||
const scrollError = await scrollElementWithContext(
|
||||
cdp,
|
||||
tabId,
|
||||
address.target,
|
||||
address.backendNodeId,
|
||||
address.frameId,
|
||||
graph,
|
||||
context,
|
||||
);
|
||||
if (scrollError) return scrollError;
|
||||
}
|
||||
|
||||
// Only topology was read above. Begin live measurements after scrolling.
|
||||
const localRegion = await nodeContentRegion(
|
||||
cdpRunnerForTarget(cdp, address.target),
|
||||
tabId,
|
||||
@@ -262,13 +154,32 @@ export async function resolveNodeGeometry(
|
||||
if (isRpcError(localRegion)) return localRegion;
|
||||
|
||||
let topVisibleRegions: Region;
|
||||
let targetActionPoint: Point | null = null;
|
||||
if (address.frameId && graph) {
|
||||
const projection = await resolveFrameProjection(cdp, graph, address.frameId);
|
||||
const projection = await context.targetProjection(address.frameId);
|
||||
if (!projection)
|
||||
return geometryError(`could not resolve frame geometry for ${address.frameId}`);
|
||||
topVisibleRegions = projectRegionToViewport(localRegion, projection);
|
||||
if (address.target.sessionId) {
|
||||
const localViewport = projection.edges[0]?.sourceViewport ?? projection.topViewport;
|
||||
const localVisibleRegions = localRegion
|
||||
.map((polygon) =>
|
||||
clipPolygon(
|
||||
polygon,
|
||||
rectPolygon({
|
||||
x: 0,
|
||||
y: 0,
|
||||
w: localViewport.width,
|
||||
h: localViewport.height,
|
||||
}),
|
||||
),
|
||||
)
|
||||
.filter((polygon) => polygon.length >= 3);
|
||||
const localActionRegion = largestRegion(localVisibleRegions);
|
||||
targetActionPoint = localActionRegion ? polygonCentroid(localActionRegion) : null;
|
||||
}
|
||||
} else {
|
||||
const viewport = await targetViewport(cdp, address.target);
|
||||
const viewport = await context.viewport(address.target);
|
||||
if (!viewport) return geometryError("could not resolve top viewport geometry");
|
||||
topVisibleRegions = localRegion
|
||||
.map((polygon) =>
|
||||
@@ -283,7 +194,17 @@ export async function resolveNodeGeometry(
|
||||
if (!topBounds || !actionPoint) {
|
||||
return { code: "permission_denied", message: "element not visible" };
|
||||
}
|
||||
return { topVisibleRegions, topBounds, actionPoint };
|
||||
if (address.target.sessionId && !targetActionPoint) {
|
||||
return { code: "permission_denied", message: "element not visible in its target" };
|
||||
}
|
||||
targetActionPoint ??= actionPoint;
|
||||
return {
|
||||
topVisibleRegions,
|
||||
topBounds,
|
||||
actionPoint,
|
||||
targetActionPoint,
|
||||
topViewport: cssViewport(await context.layoutMetrics({ tabId })),
|
||||
};
|
||||
} catch (error) {
|
||||
return geometryError(error instanceof Error ? error.message : String(error));
|
||||
}
|
||||
|
||||
@@ -245,22 +245,3 @@ export function projectRectToViewport(
|
||||
if (!rect) return null;
|
||||
return regionBounds(projectRegionToViewport([rectPolygon(rect)], projection));
|
||||
}
|
||||
|
||||
export function childFrameProjection(
|
||||
parent: GeometryProjection,
|
||||
ownerRectInParent: { x: number; y: number; w: number; h: number },
|
||||
): GeometryProjection {
|
||||
const destinationQuad = rectPolygon(ownerRectInParent);
|
||||
return {
|
||||
sourceClips: [],
|
||||
edges: [
|
||||
{
|
||||
sourceViewport: { width: ownerRectInParent.w, height: ownerRectInParent.h },
|
||||
destinationQuad,
|
||||
destinationClips: parent.sourceClips,
|
||||
},
|
||||
...parent.edges,
|
||||
],
|
||||
topViewport: parent.topViewport,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -0,0 +1,157 @@
|
||||
import { type CdpTarget, cdpTargetKey } from "@/browser-driver/frame-graph";
|
||||
import type { GeometryProjection, ViewportRect } from "../geometry";
|
||||
import { projectRectToViewport } from "../geometry";
|
||||
|
||||
export interface CoordinateOwner {
|
||||
target: CdpTarget;
|
||||
frameId?: string;
|
||||
}
|
||||
|
||||
export interface FrameViewportRect {
|
||||
space: "frame-viewport-css";
|
||||
owner: CoordinateOwner;
|
||||
rect: ViewportRect;
|
||||
}
|
||||
|
||||
export interface SnapshotProjection {
|
||||
source: CoordinateOwner;
|
||||
geometry: GeometryProjection;
|
||||
}
|
||||
|
||||
/** A failed owner read is different from a successfully projected, clipped-out rect. */
|
||||
export interface UnavailableFrameProjection {
|
||||
status: "unavailable";
|
||||
source: CoordinateOwner;
|
||||
ownerBackendNodeId: number;
|
||||
}
|
||||
|
||||
export interface UnavailableSnapshotCoordinates {
|
||||
status: "unavailable";
|
||||
source: CoordinateOwner;
|
||||
reason: "snapshot-coordinates-unavailable";
|
||||
}
|
||||
|
||||
export type SnapshotProjectionResult =
|
||||
| { status: "available"; projection: SnapshotProjection }
|
||||
| UnavailableFrameProjection;
|
||||
|
||||
/** Descendants inherit the failed boundary without attempting another owner read. */
|
||||
export type FrameProjectionIssue =
|
||||
| UnavailableFrameProjection
|
||||
| UnavailableSnapshotCoordinates
|
||||
| {
|
||||
status: "blocked";
|
||||
source: CoordinateOwner;
|
||||
cause: UnavailableFrameProjection | UnavailableSnapshotCoordinates;
|
||||
};
|
||||
|
||||
export type FrameProjectionState = SnapshotProjectionResult | FrameProjectionIssue;
|
||||
|
||||
export interface SnapshotCoordinates {
|
||||
layoutUnitsPerCssPixel: number;
|
||||
scrollCss: { x: number; y: number };
|
||||
}
|
||||
|
||||
/** Snapshot scroll offsets share the raw layout units of its bounds. Only a
|
||||
* target's root document may fall back to that target's CSS layout viewport. */
|
||||
export function snapshotCoordinates(
|
||||
document: { scrollOffsetX?: number; scrollOffsetY?: number },
|
||||
layoutUnitsPerCssPixel: number | null,
|
||||
rootScrollCss?: { x?: number; y?: number },
|
||||
): SnapshotCoordinates | null {
|
||||
if (
|
||||
layoutUnitsPerCssPixel === null ||
|
||||
!Number.isFinite(layoutUnitsPerCssPixel) ||
|
||||
layoutUnitsPerCssPixel <= 0
|
||||
)
|
||||
return null;
|
||||
const x =
|
||||
document.scrollOffsetX === undefined
|
||||
? rootScrollCss?.x
|
||||
: document.scrollOffsetX / layoutUnitsPerCssPixel;
|
||||
const y =
|
||||
document.scrollOffsetY === undefined
|
||||
? rootScrollCss?.y
|
||||
: document.scrollOffsetY / layoutUnitsPerCssPixel;
|
||||
if (x === undefined || y === undefined || !Number.isFinite(x) || !Number.isFinite(y)) return null;
|
||||
return { layoutUnitsPerCssPixel, scrollCss: { x, y } };
|
||||
}
|
||||
|
||||
/** Convert raw document-relative Blink layout units exactly once, before any
|
||||
* CSS viewport clipping or frame projection. This scale is not screenshot zoom. */
|
||||
export function snapshotViewportRect(
|
||||
bounds: number[],
|
||||
owner: CoordinateOwner,
|
||||
coordinates: SnapshotCoordinates | null,
|
||||
): FrameViewportRect | null {
|
||||
if (!coordinates || bounds.length < 4 || !bounds.slice(0, 4).every(Number.isFinite)) return null;
|
||||
const { layoutUnitsPerCssPixel: scale, scrollCss: scroll } = coordinates;
|
||||
const [x, y, width, height] = bounds;
|
||||
if (
|
||||
!Number.isFinite(scale) ||
|
||||
scale <= 0 ||
|
||||
width <= 0 ||
|
||||
height <= 0 ||
|
||||
!Number.isFinite(scroll.x) ||
|
||||
!Number.isFinite(scroll.y)
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
space: "frame-viewport-css",
|
||||
owner,
|
||||
rect: {
|
||||
x: x / scale - scroll.x,
|
||||
y: y / scale - scroll.y,
|
||||
width: width / scale,
|
||||
height: height / scale,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function projectSnapshotRect(
|
||||
input: FrameViewportRect,
|
||||
projection: SnapshotProjection,
|
||||
): ViewportRect | null {
|
||||
if (
|
||||
input.owner.frameId !== projection.source.frameId ||
|
||||
cdpTargetKey(input.owner.target) !== cdpTargetKey(projection.source.target)
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
const { x, y, width: w, height: h } = input.rect;
|
||||
return projectRectToViewport({ x, y, w, h }, projection.geometry);
|
||||
}
|
||||
|
||||
/** CSS viewport metadata; browser zoom is distinct from raster devicePixelRatio. */
|
||||
export interface CssViewport {
|
||||
width: number;
|
||||
height: number;
|
||||
scrollX: number;
|
||||
scrollY: number;
|
||||
cssToDip: number;
|
||||
}
|
||||
|
||||
/** Page.captureScreenshot takes page-relative DIP, not viewport-relative CSS pixels. */
|
||||
export function screenshotPageRect(
|
||||
rect: ViewportRect,
|
||||
viewport: CssViewport,
|
||||
): { space: "page-dip"; rect: ViewportRect } | null {
|
||||
const { cssToDip, scrollX, scrollY } = viewport;
|
||||
if (
|
||||
![rect.x, rect.y, rect.width, rect.height, cssToDip, scrollX, scrollY].every(Number.isFinite) ||
|
||||
cssToDip <= 0 ||
|
||||
rect.width <= 0 ||
|
||||
rect.height <= 0
|
||||
)
|
||||
return null;
|
||||
return {
|
||||
space: "page-dip",
|
||||
rect: {
|
||||
x: (rect.x + scrollX) * cssToDip,
|
||||
y: (rect.y + scrollY) * cssToDip,
|
||||
width: rect.width * cssToDip,
|
||||
height: rect.height * cssToDip,
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,356 @@
|
||||
import {
|
||||
type CdpFrame,
|
||||
type CdpFrameGraph,
|
||||
type CdpTarget,
|
||||
cdpTargetKey,
|
||||
} from "@/browser-driver/frame-graph";
|
||||
import {
|
||||
type GeometryProjection,
|
||||
type Polygon,
|
||||
type ProjectiveEdge,
|
||||
parseCdpQuad,
|
||||
type Quad,
|
||||
type Size,
|
||||
} from "../geometry";
|
||||
import { type CdpRunner, sendToCdpTarget } from "../shared";
|
||||
import type { CoordinateOwner, CssViewport, SnapshotProjectionResult } from "./coordinate-types";
|
||||
|
||||
import { readSnapshotOwnerSizes } from "./snapshot-owner-sizes";
|
||||
|
||||
export interface LayoutMetrics {
|
||||
cssVisualViewport?: { zoom?: number; clientWidth?: number; clientHeight?: number };
|
||||
visualViewport?: { zoom?: number; clientWidth?: number; clientHeight?: number };
|
||||
cssLayoutViewport?: {
|
||||
clientWidth?: number;
|
||||
clientHeight?: number;
|
||||
pageX?: number;
|
||||
pageY?: number;
|
||||
};
|
||||
layoutViewport?: { clientWidth?: number; clientHeight?: number; pageX?: number; pageY?: number };
|
||||
}
|
||||
|
||||
/** Chromium emits these paired floating-point dimensions before/after dividing
|
||||
* by LayoutZoomFactor. Integer layout viewport dimensions lose precision at
|
||||
* fractional zoom. Neither visualViewport.scale nor .zoom is this conversion.
|
||||
* See InspectorPageAgent::getLayoutMetrics in Chromium. */
|
||||
export function snapshotLayoutScale(metrics: LayoutMetrics): number | null {
|
||||
for (const dimension of ["clientWidth", "clientHeight"] as const) {
|
||||
const raw = metrics.visualViewport?.[dimension];
|
||||
const css = metrics.cssVisualViewport?.[dimension];
|
||||
if (
|
||||
raw !== undefined &&
|
||||
css !== undefined &&
|
||||
Number.isFinite(raw) &&
|
||||
Number.isFinite(css) &&
|
||||
raw > 0 &&
|
||||
css > 0
|
||||
) {
|
||||
const scale = raw / css;
|
||||
if (Number.isFinite(scale) && scale > 0) return scale;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** The legacy field is a compatibility fallback, never a source of raster DPR. */
|
||||
export function cssViewport(metrics: LayoutMetrics): CssViewport {
|
||||
const value = metrics.cssLayoutViewport ?? metrics.layoutViewport;
|
||||
return {
|
||||
width: value?.clientWidth ?? 0,
|
||||
height: value?.clientHeight ?? 0,
|
||||
scrollX: value?.pageX ?? 0,
|
||||
scrollY: value?.pageY ?? 0,
|
||||
cssToDip: metrics.cssVisualViewport?.zoom ?? metrics.visualViewport?.zoom ?? 1,
|
||||
};
|
||||
}
|
||||
|
||||
/** One read-only measurement phase. Discard after scrolling or any later operation. */
|
||||
export class GeometryContext {
|
||||
private readonly metrics = new Map<string, Promise<LayoutMetrics>>();
|
||||
private readonly owners = new Map<string, Promise<Quad | null>>();
|
||||
private readonly projections = new Map<string, Promise<GeometryProjection | null>>();
|
||||
private readonly snapshotOwners = new Map<string, Promise<{ quad: Quad; size: Size } | null>>();
|
||||
private readonly sizeBatches = new Map<string, () => Promise<Map<number, Size>>>();
|
||||
private active = 0;
|
||||
private readonly waiting: Array<() => void> = [];
|
||||
private graphPromise?: Promise<CdpFrameGraph | null>;
|
||||
private framesPromise?: Promise<Map<string, CdpFrame>>;
|
||||
|
||||
constructor(
|
||||
private readonly cdp: CdpRunner,
|
||||
private readonly tabId: number,
|
||||
graph?: CdpFrameGraph,
|
||||
private readonly signal?: AbortSignal,
|
||||
) {
|
||||
if (graph) this.graphPromise = Promise.resolve(graph);
|
||||
}
|
||||
|
||||
private async request<T>(
|
||||
target: CdpTarget,
|
||||
method: string,
|
||||
params: object,
|
||||
cleanup = false,
|
||||
): Promise<T> {
|
||||
if (!cleanup && this.signal?.aborted) throw new DOMException("geometry aborted", "AbortError");
|
||||
if (this.active >= 4) await new Promise<void>((resolve) => this.waiting.push(resolve));
|
||||
else this.active++;
|
||||
try {
|
||||
if (!cleanup && this.signal?.aborted)
|
||||
throw new DOMException("geometry aborted", "AbortError");
|
||||
return await sendToCdpTarget<T>(this.cdp, target, method, params);
|
||||
} finally {
|
||||
const next = this.waiting.shift();
|
||||
if (next) next();
|
||||
else this.active--;
|
||||
}
|
||||
}
|
||||
|
||||
graph(): Promise<CdpFrameGraph | null> {
|
||||
this.graphPromise ??=
|
||||
this.cdp.getFrameGraph?.(this.tabId).catch(() => null) ?? Promise.resolve(null);
|
||||
return this.graphPromise;
|
||||
}
|
||||
|
||||
private frames(): Promise<Map<string, CdpFrame>> {
|
||||
this.framesPromise ??= this.graph().then(
|
||||
(graph) => new Map(graph?.frames.map((frame) => [frame.frameId, frame]) ?? []),
|
||||
);
|
||||
return this.framesPromise;
|
||||
}
|
||||
|
||||
async frame(frameId: string): Promise<CdpFrame | undefined> {
|
||||
return (await this.frames()).get(frameId);
|
||||
}
|
||||
|
||||
/** Topology only: safe to resolve before the caller scrolls frame owners. */
|
||||
async ancestry(frameId: string): Promise<CdpFrame[] | null> {
|
||||
const frames = await this.frames();
|
||||
const path: CdpFrame[] = [];
|
||||
const seen = new Set<string>();
|
||||
let current = frames.get(frameId);
|
||||
if (!current) return null;
|
||||
while (current.parentFrameId) {
|
||||
if (seen.has(current.frameId)) return null;
|
||||
seen.add(current.frameId);
|
||||
path.push(current);
|
||||
const parent = frames.get(current.parentFrameId);
|
||||
if (!parent) return null;
|
||||
current = parent;
|
||||
}
|
||||
return path;
|
||||
}
|
||||
|
||||
layoutMetrics(target: CdpTarget): Promise<LayoutMetrics> {
|
||||
const key = cdpTargetKey(target);
|
||||
let promise = this.metrics.get(key);
|
||||
if (!promise) {
|
||||
promise = this.request<LayoutMetrics>(target, "Page.getLayoutMetrics", {});
|
||||
this.metrics.set(key, promise);
|
||||
}
|
||||
return promise;
|
||||
}
|
||||
|
||||
async viewport(target: CdpTarget): Promise<Size | null> {
|
||||
const { width, height } = cssViewport(await this.layoutMetrics(target));
|
||||
return Number.isFinite(width) && Number.isFinite(height) && width > 0 && height > 0
|
||||
? { width, height }
|
||||
: null;
|
||||
}
|
||||
|
||||
ownerContent(target: CdpTarget, backendNodeId: number): Promise<Quad | null> {
|
||||
const key = `${cdpTargetKey(target)}:${backendNodeId}`;
|
||||
let promise = this.owners.get(key);
|
||||
if (!promise) {
|
||||
promise = this.request<{ model?: { content?: number[] } }>(target, "DOM.getBoxModel", {
|
||||
backendNodeId,
|
||||
}).then((reply) => parseCdpQuad(reply.model?.content));
|
||||
this.owners.set(key, promise);
|
||||
}
|
||||
return promise;
|
||||
}
|
||||
|
||||
/** Same-target owner quads are already target-relative, including nested owners. */
|
||||
async snapshotProjection(
|
||||
source: CoordinateOwner,
|
||||
ownerBackendNodeId: number,
|
||||
ancestorClips: Polygon[],
|
||||
viewport: Size,
|
||||
): Promise<SnapshotProjectionResult> {
|
||||
const key = `${cdpTargetKey(source.target)}:${ownerBackendNodeId}`;
|
||||
let promise = this.snapshotOwners.get(key);
|
||||
if (!promise) {
|
||||
promise = this.snapshotOwner(source.target, ownerBackendNodeId);
|
||||
this.snapshotOwners.set(key, promise);
|
||||
}
|
||||
let owner: { quad: Quad; size: Size } | null;
|
||||
try {
|
||||
owner = await promise;
|
||||
} catch (error) {
|
||||
if (error && typeof error === "object" && "name" in error && error.name === "AbortError")
|
||||
throw error;
|
||||
console.debug("[bsk geometry] frame owner unavailable", error);
|
||||
owner = null;
|
||||
}
|
||||
return owner
|
||||
? {
|
||||
status: "available",
|
||||
projection: {
|
||||
source,
|
||||
geometry: {
|
||||
sourceClips: [],
|
||||
edges: [
|
||||
{
|
||||
sourceViewport: owner.size,
|
||||
destinationQuad: owner.quad,
|
||||
destinationClips: ancestorClips,
|
||||
},
|
||||
],
|
||||
topViewport: viewport,
|
||||
},
|
||||
},
|
||||
}
|
||||
: { status: "unavailable", source, ownerBackendNodeId };
|
||||
}
|
||||
|
||||
/** Register one target's snapshot owners; the batch starts only when a
|
||||
* projection actually needs a size. Single owners keep the cheaper old path. */
|
||||
registerSnapshotOwners(target: CdpTarget, owners: ReadonlySet<number>): void {
|
||||
const key = cdpTargetKey(target);
|
||||
if (owners.size < 2 || this.sizeBatches.has(key)) return;
|
||||
let pending: Promise<Map<number, Size>> | undefined;
|
||||
this.sizeBatches.set(key, () => {
|
||||
pending ??= readSnapshotOwnerSizes(
|
||||
(method, params, cleanup) => this.request(target, method, params, cleanup),
|
||||
owners,
|
||||
).catch((error) => {
|
||||
if (error && typeof error === "object" && "name" in error && error.name === "AbortError")
|
||||
throw error;
|
||||
return new Map<number, Size>();
|
||||
});
|
||||
return pending;
|
||||
});
|
||||
}
|
||||
|
||||
private async snapshotOwner(
|
||||
target: CdpTarget,
|
||||
backendNodeId: number,
|
||||
): Promise<{ quad: Quad; size: Size } | null> {
|
||||
const quad = await this.ownerContent(target, backendNodeId);
|
||||
if (!quad) return null;
|
||||
const batchSize = (await this.sizeBatches.get(cdpTargetKey(target))?.())?.get(backendNodeId);
|
||||
if (this.signal?.aborted) throw new DOMException("geometry aborted", "AbortError");
|
||||
if (batchSize) return { quad, size: batchSize };
|
||||
const resolved = await this.request<{ object?: { objectId?: string } }>(
|
||||
target,
|
||||
"DOM.resolveNode",
|
||||
{ backendNodeId },
|
||||
);
|
||||
const objectId = resolved.object?.objectId;
|
||||
if (!objectId) return null;
|
||||
try {
|
||||
const reply = await this.request<{ result?: { value?: Size }; exceptionDetails?: unknown }>(
|
||||
target,
|
||||
"Runtime.callFunctionOn",
|
||||
{
|
||||
objectId,
|
||||
functionDeclaration: `function() {
|
||||
if (!this.isConnected) return null;
|
||||
const style = getComputedStyle(this);
|
||||
return {
|
||||
width: this.clientWidth - parseFloat(style.paddingLeft) - parseFloat(style.paddingRight),
|
||||
height: this.clientHeight - parseFloat(style.paddingTop) - parseFloat(style.paddingBottom)
|
||||
};
|
||||
}`,
|
||||
returnByValue: true,
|
||||
},
|
||||
);
|
||||
const size = reply.result?.value;
|
||||
return !reply.exceptionDetails &&
|
||||
size &&
|
||||
Number.isFinite(size.width) &&
|
||||
Number.isFinite(size.height) &&
|
||||
size.width > 0 &&
|
||||
size.height > 0
|
||||
? { quad, size }
|
||||
: null;
|
||||
} finally {
|
||||
// Cleanup must still run if cancellation arrived after resolving the object.
|
||||
await this.request(target, "Runtime.releaseObject", { objectId }, true).catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
private targetRoot(frames: Map<string, CdpFrame>, frame: CdpFrame): CdpFrame | null {
|
||||
const seen = new Set<string>();
|
||||
let current = frame;
|
||||
while (current.parentFrameId) {
|
||||
if (seen.has(current.frameId)) return null;
|
||||
seen.add(current.frameId);
|
||||
const parent = frames.get(current.parentFrameId);
|
||||
if (!parent) return null;
|
||||
if (cdpTargetKey(parent.target) !== cdpTargetKey(current.target)) break;
|
||||
current = parent;
|
||||
}
|
||||
return current;
|
||||
}
|
||||
|
||||
private async clips(
|
||||
frames: Map<string, CdpFrame>,
|
||||
frame: CdpFrame,
|
||||
root: CdpFrame,
|
||||
): Promise<Polygon[] | null> {
|
||||
const clips: Polygon[] = [];
|
||||
let current = frame;
|
||||
const seen = new Set<string>();
|
||||
while (current.frameId !== root.frameId) {
|
||||
if (seen.has(current.frameId) || !current.parentFrameId) return null;
|
||||
seen.add(current.frameId);
|
||||
const parent = frames.get(current.parentFrameId);
|
||||
if (!parent || current.ownerBackendNodeId === undefined) return null;
|
||||
const quad = await this.ownerContent(parent.target, current.ownerBackendNodeId);
|
||||
if (!quad) return null;
|
||||
clips.push(quad);
|
||||
current = parent;
|
||||
}
|
||||
return clips;
|
||||
}
|
||||
|
||||
/** Input is already in the frame's CDP target viewport, not its local viewport. */
|
||||
targetProjection(frameId: string): Promise<GeometryProjection | null> {
|
||||
let promise = this.projections.get(frameId);
|
||||
if (!promise) {
|
||||
promise = this.buildTargetProjection(frameId);
|
||||
this.projections.set(frameId, promise);
|
||||
}
|
||||
return promise;
|
||||
}
|
||||
|
||||
private async buildTargetProjection(frameId: string): Promise<GeometryProjection | null> {
|
||||
const frames = await this.frames();
|
||||
const frame = frames.get(frameId);
|
||||
if (!frame) return null;
|
||||
let root = this.targetRoot(frames, frame);
|
||||
if (!root) return null;
|
||||
const sourceViewport = await this.viewport(root.target);
|
||||
if (!sourceViewport) return null;
|
||||
const sourceClips = await this.clips(frames, frame, root);
|
||||
if (!sourceClips) return null;
|
||||
const edges: ProjectiveEdge[] = [];
|
||||
const seen = new Set<string>();
|
||||
while (root.parentFrameId) {
|
||||
if (seen.has(root.frameId)) return null;
|
||||
seen.add(root.frameId);
|
||||
const parent = frames.get(root.parentFrameId);
|
||||
if (!parent || root.ownerBackendNodeId === undefined) return null;
|
||||
const destinationQuad = await this.ownerContent(parent.target, root.ownerBackendNodeId);
|
||||
const source = await this.viewport(root.target);
|
||||
const parentRoot = this.targetRoot(frames, parent);
|
||||
if (!destinationQuad || !source || !parentRoot) return null;
|
||||
const destinationClips = await this.clips(frames, parent, parentRoot);
|
||||
if (!destinationClips) return null;
|
||||
edges.push({ sourceViewport: source, destinationQuad, destinationClips });
|
||||
root = parentRoot;
|
||||
}
|
||||
const topViewport = await this.viewport(root.target);
|
||||
return topViewport ? { sourceClips, edges, topViewport } : null;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
import type { Size } from "../geometry";
|
||||
|
||||
interface SerializedValue {
|
||||
type: string;
|
||||
value?: unknown;
|
||||
}
|
||||
|
||||
type Send = <T>(method: string, params: object, cleanup?: boolean) => Promise<T>;
|
||||
|
||||
/** Batch only accessible frame owners. Missing/shadow/cross-origin owners keep
|
||||
* the existing node-scoped read; no page-wide style columns are requested. */
|
||||
export async function readSnapshotOwnerSizes(
|
||||
send: Send,
|
||||
owners: ReadonlySet<number>,
|
||||
): Promise<Map<number, Size>> {
|
||||
const sizes = new Map<number, Size>();
|
||||
const objectGroup = `bsk-frame-sizes-${crypto.randomUUID()}`;
|
||||
try {
|
||||
const reply = await send<{ result?: { deepSerializedValue?: SerializedValue } }>(
|
||||
"Runtime.evaluate",
|
||||
{
|
||||
expression: `(() => {
|
||||
const result = [];
|
||||
const documents = [document];
|
||||
for (let i = 0; i < documents.length && result.length < ${owners.size}; i++) {
|
||||
for (const owner of documents[i].querySelectorAll('iframe,frame')) {
|
||||
if (result.length >= ${owners.size}) break;
|
||||
if (!owner.isConnected) continue;
|
||||
const style = getComputedStyle(owner);
|
||||
result.push([owner, JSON.stringify({
|
||||
width: owner.clientWidth - parseFloat(style.paddingLeft) - parseFloat(style.paddingRight),
|
||||
height: owner.clientHeight - parseFloat(style.paddingTop) - parseFloat(style.paddingBottom)
|
||||
})]);
|
||||
try { if (owner.contentDocument) documents.push(owner.contentDocument); } catch {}
|
||||
}
|
||||
}
|
||||
return result;
|
||||
})()`,
|
||||
objectGroup,
|
||||
serializationOptions: {
|
||||
serialization: "deep",
|
||||
maxDepth: 3,
|
||||
additionalParameters: { maxNodeDepth: 0, includeShadowTree: "none" },
|
||||
},
|
||||
},
|
||||
);
|
||||
const result = reply.result?.deepSerializedValue;
|
||||
if (result?.type !== "array" || !Array.isArray(result.value)) return sizes;
|
||||
for (const entry of result.value as SerializedValue[]) {
|
||||
if (entry.type !== "array" || !Array.isArray(entry.value)) continue;
|
||||
const [node, data] = entry.value as SerializedValue[];
|
||||
if (node?.type !== "node" || data?.type !== "string" || typeof data.value !== "string")
|
||||
continue;
|
||||
const id = (node.value as { backendNodeId?: number } | undefined)?.backendNodeId;
|
||||
if (id === undefined || !owners.has(id)) continue;
|
||||
try {
|
||||
const size = JSON.parse(data.value) as Size;
|
||||
if (
|
||||
size &&
|
||||
Number.isFinite(size.width) &&
|
||||
Number.isFinite(size.height) &&
|
||||
size.width > 0 &&
|
||||
size.height > 0
|
||||
)
|
||||
sizes.set(id, size);
|
||||
} catch {
|
||||
/* An invalid entry uses the existing individual read. */
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
await send("Runtime.releaseObjectGroup", { objectGroup }, true).catch(() => {});
|
||||
}
|
||||
return sizes;
|
||||
}
|
||||
@@ -183,6 +183,8 @@ export async function resolveBackendNode(
|
||||
}
|
||||
// selector path
|
||||
try {
|
||||
// Selector lookup itself attaches CDP, even when no element is found.
|
||||
cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
const doc = await cdp.send<{ root?: { nodeId?: number } }>(target.tabId, "DOM.getDocument", {
|
||||
depth: 0,
|
||||
});
|
||||
@@ -607,6 +609,53 @@ type SelectMutationResult =
|
||||
// tool.fill
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type FillFailureReason =
|
||||
| "target_not_fillable"
|
||||
| "fill_value_invalid"
|
||||
| "fill_target_changed"
|
||||
| "fill_focus_lost"
|
||||
| "fill_value_mismatch"
|
||||
| "fill_failed";
|
||||
|
||||
type FillPreparation =
|
||||
| { before: string; expected: string }
|
||||
| { reason: FillFailureReason; error: string };
|
||||
|
||||
type FillReadiness = "ready" | "background" | "fill_target_changed" | "fill_focus_lost";
|
||||
|
||||
function fillError(reason: FillFailureReason, message: string): RpcError {
|
||||
return rpcError(
|
||||
reason === "target_not_fillable" || reason === "fill_value_invalid"
|
||||
? "invalid_params"
|
||||
: "cdp_failed",
|
||||
reason,
|
||||
message,
|
||||
);
|
||||
}
|
||||
|
||||
interface FillScriptReply<T> {
|
||||
result?: { value?: T };
|
||||
exceptionDetails?: unknown;
|
||||
}
|
||||
|
||||
const FILL_EDITABLE_FUNCTION = `function() {
|
||||
const tag = this.tagName.toLowerCase();
|
||||
const supported = tag === 'input'
|
||||
? ['text', 'search', 'tel', 'url', 'email', 'password', 'number'].includes(this.type)
|
||||
: tag === 'textarea' || this.isContentEditable;
|
||||
return this.isConnected && supported && !this.readOnly && !this.matches(':disabled');
|
||||
}`;
|
||||
|
||||
// Chrome renders a trailing editable newline with an empty <div><br></div>.
|
||||
// innerText includes the padding break; it is not an extra typed character.
|
||||
const FILL_VALUE_FUNCTION = `function() {
|
||||
if (!this.isContentEditable) return this.value;
|
||||
const value = this.innerText;
|
||||
const tail = this.lastChild;
|
||||
const padding = tail && tail.nodeName === 'DIV' && tail.childNodes.length === 1 && tail.firstChild.nodeName === 'BR';
|
||||
return padding && value.endsWith('\\n\\n') ? value.slice(0, -1) : value;
|
||||
}`;
|
||||
|
||||
export async function handleFill(
|
||||
manager: SessionManager,
|
||||
params: FillParams,
|
||||
@@ -658,12 +707,8 @@ export async function handleFill(
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
await nodeCdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
|
||||
} catch (err) {
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
message: err instanceof Error ? err.message : String(err),
|
||||
};
|
||||
return fillError("fill_failed", err instanceof Error ? err.message : String(err));
|
||||
}
|
||||
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
@@ -676,57 +721,252 @@ export async function handleFill(
|
||||
const clearBefore = params.clear_before ?? true;
|
||||
|
||||
try {
|
||||
if (clearBefore) {
|
||||
// Clear input/textarea value or wipe contenteditable innerText,
|
||||
// then fire `input` so frameworks observe the empty state.
|
||||
await nodeCdp.send(target.tabId, "Runtime.callFunctionOn", {
|
||||
objectId,
|
||||
functionDeclaration: `function() {
|
||||
if (this.isContentEditable) { this.textContent = ''; }
|
||||
else {
|
||||
const proto = this instanceof HTMLTextAreaElement
|
||||
? HTMLTextAreaElement.prototype
|
||||
: HTMLInputElement.prototype;
|
||||
const descriptor = Object.getOwnPropertyDescriptor(proto, 'value');
|
||||
if (descriptor && descriptor.set) descriptor.set.call(this, '');
|
||||
else this.value = '';
|
||||
}
|
||||
this.dispatchEvent(new Event('input', { bubbles: true }));
|
||||
}`,
|
||||
returnByValue: true,
|
||||
});
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
try {
|
||||
await nodeCdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
|
||||
} catch (error) {
|
||||
// Chrome rejects DOM.focus for disabled controls, including inherited
|
||||
// fieldset state. Classify the live target before treating this as a
|
||||
// browser failure; the normal path needs no extra round trip.
|
||||
const editable = await nodeCdp.send<FillScriptReply<boolean>>(
|
||||
target.tabId,
|
||||
"Runtime.callFunctionOn",
|
||||
{ objectId, functionDeclaration: FILL_EDITABLE_FUNCTION, returnByValue: true },
|
||||
);
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
if (!editable.exceptionDetails && editable.result?.value === false) {
|
||||
return fillError(
|
||||
"target_not_fillable",
|
||||
"fill target is not editable or its input type is unsupported",
|
||||
);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
// Check editability before clearing. Keep the expected result tied to
|
||||
// this object, including the existing value on the append path.
|
||||
const prepared = await nodeCdp.send<FillScriptReply<FillPreparation>>(
|
||||
target.tabId,
|
||||
"Runtime.callFunctionOn",
|
||||
{
|
||||
objectId,
|
||||
functionDeclaration: `function(value, clearBefore) {
|
||||
const tag = this.tagName.toLowerCase();
|
||||
const native = tag === 'input' || tag === 'textarea';
|
||||
if (!(${FILL_EDITABLE_FUNCTION}).call(this)) return { reason: 'target_not_fillable', error: 'fill target is not editable or its input type is unsupported' };
|
||||
if (this.getRootNode().activeElement !== this) return { reason: 'fill_focus_lost', error: 'fill target does not have focus' };
|
||||
const before = clearBefore ? '' : (${FILL_VALUE_FUNCTION}).call(this);
|
||||
let expected = before + value;
|
||||
if (native) {
|
||||
// Use the browser's own value sanitization (e.g. textarea
|
||||
// line endings) without changing the live control.
|
||||
const normalizer = this.ownerDocument.createElement(tag);
|
||||
if (tag === 'input') {
|
||||
normalizer.type = this.type;
|
||||
normalizer.multiple = this.multiple;
|
||||
}
|
||||
// insertText treats line breaks in a single-line input as spaces.
|
||||
if (tag === 'input') expected = expected.replace(/\\r\\n|\\r|\\n/g, ' ');
|
||||
normalizer.value = expected;
|
||||
if (expected !== '' && normalizer.value === '' && this.type === 'number') {
|
||||
return { reason: 'fill_value_invalid', error: 'fill value is not valid for a number input' };
|
||||
}
|
||||
expected = normalizer.value;
|
||||
if (this.type !== 'number' && this.maxLength >= 0 && expected.length > this.maxLength) {
|
||||
return { reason: 'fill_value_invalid', error: 'fill value exceeds the target maxlength' };
|
||||
}
|
||||
} else {
|
||||
expected = expected.replace(/\\r\\n?/g, '\\n');
|
||||
}
|
||||
if (clearBefore) {
|
||||
if (native) {
|
||||
const proto = tag === 'textarea' ? HTMLTextAreaElement.prototype : HTMLInputElement.prototype;
|
||||
Object.getOwnPropertyDescriptor(proto, 'value').set.call(this, '');
|
||||
} else {
|
||||
this.textContent = '';
|
||||
}
|
||||
this.dispatchEvent(new Event('input', { bubbles: true }));
|
||||
}
|
||||
return { before, expected };
|
||||
}`,
|
||||
arguments: [{ value: params.value }, { value: clearBefore }],
|
||||
returnByValue: true,
|
||||
},
|
||||
);
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
if (prepared.exceptionDetails) {
|
||||
return fillError("fill_failed", "fill preparation script failed");
|
||||
}
|
||||
const preparation = prepared.result?.value;
|
||||
if (preparation && "error" in preparation) {
|
||||
return fillError(preparation.reason, preparation.error);
|
||||
}
|
||||
if (typeof preparation?.before !== "string" || typeof preparation?.expected !== "string") {
|
||||
return fillError("fill_failed", "fill preparation returned an unexpected result");
|
||||
}
|
||||
// A separate call runs after the clearing event's microtasks drain.
|
||||
// DOM focus works in background tabs. Use document.hasFocus() only to
|
||||
// detect deferred focus events, never to reject background input.
|
||||
const checkReady = async (): Promise<FillReadiness | RpcError> => {
|
||||
const reply = await nodeCdp.send<FillScriptReply<FillReadiness>>(
|
||||
target.tabId,
|
||||
"Runtime.callFunctionOn",
|
||||
{
|
||||
objectId,
|
||||
functionDeclaration: `function(before) {
|
||||
if (!(${FILL_EDITABLE_FUNCTION}).call(this) ||
|
||||
(${FILL_VALUE_FUNCTION}).call(this) !== before) return 'fill_target_changed';
|
||||
if (this.getRootNode().activeElement !== this) return 'fill_focus_lost';
|
||||
return this.ownerDocument.hasFocus() ? 'ready' : 'background';
|
||||
}`,
|
||||
arguments: [{ value: preparation.before }],
|
||||
returnByValue: true,
|
||||
},
|
||||
);
|
||||
const state = reply.result?.value;
|
||||
if (
|
||||
reply.exceptionDetails ||
|
||||
(state !== "ready" &&
|
||||
state !== "background" &&
|
||||
state !== "fill_target_changed" &&
|
||||
state !== "fill_focus_lost")
|
||||
) {
|
||||
return fillError(
|
||||
"fill_failed",
|
||||
"fill readiness script failed or returned an unexpected result",
|
||||
);
|
||||
}
|
||||
return state;
|
||||
};
|
||||
let ready = await checkReady();
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
// Restore focus once only if clearing left this same target editable
|
||||
// and unchanged. Recheck after focus handlers have run.
|
||||
if (clearBefore && ready === "fill_focus_lost") {
|
||||
await nodeCdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
ready = await checkReady();
|
||||
}
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
if (typeof ready !== "string") return ready;
|
||||
if (ready !== "ready" && ready !== "background") {
|
||||
return fillError(ready, "fill target changed or lost focus before typing");
|
||||
}
|
||||
if (params.value !== "" && (!clearBefore || ready === "background")) {
|
||||
// Native editing commands also support number/email inputs, whose
|
||||
// selection APIs cannot set a caret, and multiline contenteditables.
|
||||
// Also do this for background replacement: CDP input focuses the
|
||||
// renderer, delivering deferred focus events before it inserts text.
|
||||
// Use a command-only event so this does not invoke an End shortcut.
|
||||
try {
|
||||
await deps.cdp.send(target.tabId, "Input.dispatchKeyEvent", {
|
||||
type: "rawKeyDown",
|
||||
commands: ["moveToEndOfDocument"],
|
||||
});
|
||||
} finally {
|
||||
await deps.cdp.send(target.tabId, "Input.dispatchKeyEvent", { type: "keyUp" });
|
||||
}
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
// Focus and key handlers can change the target or move focus too.
|
||||
ready = await checkReady();
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
if (typeof ready !== "string") return ready;
|
||||
if (ready !== "ready" && ready !== "background") {
|
||||
return fillError(ready, "fill target changed or lost focus while positioning the caret");
|
||||
}
|
||||
}
|
||||
// CDP `Input.insertText` handles IME / multi-byte input out of the
|
||||
// box, much more reliably than per-key `dispatchKeyEvent`.
|
||||
await deps.cdp.send(target.tabId, "Input.insertText", { text: params.value });
|
||||
if (params.value !== "") {
|
||||
await deps.cdp.send(target.tabId, "Input.insertText", { text: params.value });
|
||||
}
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
// Fire `input` + `change` so React / Vue controlled inputs commit.
|
||||
await nodeCdp.send(target.tabId, "Runtime.callFunctionOn", {
|
||||
// Keep the existing notifications, then read in a separate call so
|
||||
// nested microtasks queued by the page cannot race the verification.
|
||||
const notified = await nodeCdp.send<FillScriptReply<unknown>>(
|
||||
target.tabId,
|
||||
"Runtime.callFunctionOn",
|
||||
{
|
||||
objectId,
|
||||
functionDeclaration: `function() {
|
||||
this.dispatchEvent(new Event('input', { bubbles: true }));
|
||||
this.dispatchEvent(new Event('change', { bubbles: true }));
|
||||
}`,
|
||||
returnByValue: true,
|
||||
},
|
||||
);
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
if (notified.exceptionDetails) {
|
||||
return fillError("fill_failed", "fill notification script failed");
|
||||
}
|
||||
const verified = await nodeCdp.send<
|
||||
FillScriptReply<{ connected: boolean; matches: boolean; valueLength: number }>
|
||||
>(target.tabId, "Runtime.callFunctionOn", {
|
||||
objectId,
|
||||
functionDeclaration: `function() {
|
||||
this.dispatchEvent(new Event('input', { bubbles: true }));
|
||||
this.dispatchEvent(new Event('change', { bubbles: true }));
|
||||
}`,
|
||||
functionDeclaration: `function(expected) {
|
||||
const value = (${FILL_VALUE_FUNCTION}).call(this);
|
||||
return { connected: this.isConnected, matches: value === expected, valueLength: value.length };
|
||||
}`,
|
||||
arguments: [{ value: preparation.expected }],
|
||||
returnByValue: true,
|
||||
});
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
const verification = verified.result?.value;
|
||||
if (
|
||||
verified.exceptionDetails ||
|
||||
typeof verification?.connected !== "boolean" ||
|
||||
typeof verification?.matches !== "boolean" ||
|
||||
typeof verification?.valueLength !== "number"
|
||||
) {
|
||||
return fillError(
|
||||
"fill_failed",
|
||||
"fill verification script failed or returned an unexpected result",
|
||||
);
|
||||
}
|
||||
if (!verification.connected) {
|
||||
return fillError("fill_target_changed", "fill target was removed or replaced during input");
|
||||
}
|
||||
if (!verification.matches) {
|
||||
return fillError(
|
||||
"fill_value_mismatch",
|
||||
"fill could not verify the expected value; observe the page before deciding whether to retry",
|
||||
);
|
||||
}
|
||||
return attachDialogs(deps.cdp, target.tabId, dialogCursor, {
|
||||
tab_id: target.tabId,
|
||||
used_ref: node.usedRef,
|
||||
used_selector: node.usedSelector,
|
||||
value_length: verification.valueLength,
|
||||
});
|
||||
} catch (err) {
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
message: err instanceof Error ? err.message : String(err),
|
||||
};
|
||||
return fillError("fill_failed", err instanceof Error ? err.message : String(err));
|
||||
} finally {
|
||||
await nodeCdp.send(target.tabId, "Runtime.releaseObject", { objectId }).catch(() => undefined);
|
||||
}
|
||||
|
||||
return attachDialogs(deps.cdp, target.tabId, dialogCursor, {
|
||||
tab_id: target.tabId,
|
||||
used_ref: node.usedRef,
|
||||
used_selector: node.usedSelector,
|
||||
value_length: params.value.length,
|
||||
});
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
@@ -28,7 +28,13 @@ import type {
|
||||
RpcError,
|
||||
WaitUntil,
|
||||
} from "@/transport/types";
|
||||
import {
|
||||
type BrowserNavigationApi,
|
||||
chromeBrowserNavigationApi,
|
||||
navigateWithBrowserApi,
|
||||
} from "./browser-navigation";
|
||||
import { attachDialogs, markDialogCursor } from "./dialogs";
|
||||
import { cdpError, isCdpExtensionAccessDenied } from "./errors";
|
||||
import {
|
||||
type CdpRunner,
|
||||
type ChromeTabsApi,
|
||||
@@ -42,6 +48,7 @@ import {
|
||||
export interface NavigationDeps {
|
||||
cdp: CdpRunner;
|
||||
tabsApi: ChromeTabsApi;
|
||||
browserNavigation?: BrowserNavigationApi;
|
||||
/** Optional AbortSignal — M7 abort hook (M10.2 will wire the full chain). */
|
||||
signal?: AbortSignal;
|
||||
/** Override default timeout when the caller omits `timeout_ms`. */
|
||||
@@ -479,6 +486,81 @@ async function readTabUrl(api: ChromeTabsApi, tabId: number): Promise<string | u
|
||||
}
|
||||
}
|
||||
|
||||
/** Recover only a failed preflight; never replay an already-dispatched navigation. */
|
||||
async function recoverBrowserNavigation(
|
||||
deps: NavigationDeps,
|
||||
tabId: number,
|
||||
action: (api: BrowserNavigationApi) => Promise<unknown>,
|
||||
waitUntil: WaitUntil,
|
||||
timeoutMs: number,
|
||||
): Promise<ReloadResult | RpcError> {
|
||||
const abort = linkedAbortSignal(deps.signal);
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
const api = deps.browserNavigation ?? chromeBrowserNavigationApi;
|
||||
try {
|
||||
let outcome = await navigateWithBrowserApi(
|
||||
api,
|
||||
tabId,
|
||||
() => action(api),
|
||||
waitUntil === "networkidle" ? "commit" : waitUntil,
|
||||
timeoutMs,
|
||||
abort.signal,
|
||||
);
|
||||
if (outcome.reached === "failed") return outcome.error;
|
||||
if (outcome.reached === "cancelled" || abort.signal.aborted) {
|
||||
return { code: "cancelled", message: "navigation recovery aborted" };
|
||||
}
|
||||
if (outcome.reached === "match") {
|
||||
// A completed reload is not a recovery if the restricted frame remains.
|
||||
await deps.cdp.send(tabId, "Page.enable", {});
|
||||
if (abort.signal.aborted)
|
||||
return { code: "cancelled", message: "navigation recovery aborted" };
|
||||
if (waitUntil === "networkidle") {
|
||||
// webNavigation cannot prove network idle. Subscribe on the new
|
||||
// document before enabling CDP lifecycle events after its commit.
|
||||
const frame = await readMainFrameInfo(deps.cdp, tabId);
|
||||
if (!frame.frameId) return cdpError("no main frame after navigation");
|
||||
const remaining = deadline - Date.now();
|
||||
if (remaining <= 0) {
|
||||
outcome = { reached: "timeout", lastLifecycle: "commit" };
|
||||
} else {
|
||||
const wait = startLifecycleWait(
|
||||
deps.cdp,
|
||||
tabId,
|
||||
frame.frameId,
|
||||
"networkIdle",
|
||||
remaining,
|
||||
abort.signal,
|
||||
{ loaderId: frame.loaderId ?? undefined },
|
||||
);
|
||||
await deps.cdp.send(tabId, "Page.setLifecycleEventsEnabled", { enabled: true });
|
||||
outcome = await wait.promise;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (outcome.reached === "cancelled" || abort.signal.aborted) {
|
||||
return { code: "cancelled", message: "navigation recovery aborted" };
|
||||
}
|
||||
return {
|
||||
tab_id: tabId,
|
||||
final_url: await readTabUrl(deps.tabsApi, tabId),
|
||||
reached: outcome.reached === "match" ? waitUntil : "timeout",
|
||||
...(outcome.reached === "timeout"
|
||||
? {
|
||||
error_text: `timed out waiting for ${waitUntil} during navigation recovery after ${timeoutMs}ms`,
|
||||
}
|
||||
: {}),
|
||||
};
|
||||
} catch (error) {
|
||||
return abort.signal.aborted
|
||||
? { code: "cancelled", message: "navigation recovery aborted" }
|
||||
: cdpError(error);
|
||||
} finally {
|
||||
abort.abort();
|
||||
abort.cleanup();
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// tool.navigate
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -505,7 +587,20 @@ export async function handleNavigate(
|
||||
|
||||
try {
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
await ensureCdpReady(deps.cdp, target.tabId);
|
||||
try {
|
||||
await ensureCdpReady(deps.cdp, target.tabId);
|
||||
} catch (error) {
|
||||
if (!isCdpExtensionAccessDenied(error)) throw error;
|
||||
const recovered = await recoverBrowserNavigation(
|
||||
deps,
|
||||
target.tabId,
|
||||
(api) => api.update(target.tabId, { url: params.url }),
|
||||
waitUntil,
|
||||
timeoutMs,
|
||||
);
|
||||
if (isRpcError(recovered)) return recovered;
|
||||
return attachDialogs(deps.cdp, target.tabId, dialogCursor, { ...recovered, url: params.url });
|
||||
}
|
||||
const expected = cdpLifecycleName(waitUntil);
|
||||
const beforeReadyState = await probeMainFrameReadyState(deps.cdp, target.tabId);
|
||||
const beforeFrame = await readMainFrameInfo(deps.cdp, target.tabId);
|
||||
@@ -574,10 +669,7 @@ export async function handleNavigate(
|
||||
}`,
|
||||
});
|
||||
} catch (err) {
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
message: err instanceof Error ? err.message : String(err),
|
||||
};
|
||||
return cdpError(err);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -727,7 +819,24 @@ export async function handleReload(
|
||||
|
||||
try {
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
await ensureCdpReady(deps.cdp, target.tabId);
|
||||
try {
|
||||
await ensureCdpReady(deps.cdp, target.tabId);
|
||||
} catch (error) {
|
||||
if (!isCdpExtensionAccessDenied(error)) throw error;
|
||||
const previousUrl = await readTabUrl(deps.tabsApi, target.tabId);
|
||||
const recovered = await recoverBrowserNavigation(
|
||||
deps,
|
||||
target.tabId,
|
||||
(api) => api.reload(target.tabId, { bypassCache: ignoreCache }),
|
||||
waitUntil,
|
||||
timeoutMs,
|
||||
);
|
||||
if (isRpcError(recovered)) return recovered;
|
||||
return attachDialogs(deps.cdp, target.tabId, dialogCursor, {
|
||||
...recovered,
|
||||
previous_url: previousUrl,
|
||||
});
|
||||
}
|
||||
const previousUrl = await readTabUrl(deps.tabsApi, target.tabId);
|
||||
const beforeFrame = await readMainFrameInfo(deps.cdp, target.tabId);
|
||||
const expected = cdpLifecycleName(waitUntil);
|
||||
@@ -776,10 +885,7 @@ export async function handleReload(
|
||||
}`,
|
||||
});
|
||||
} catch (err) {
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
message: err instanceof Error ? err.message : String(err),
|
||||
};
|
||||
return cdpError(err);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import type { CapturedSceneInput } from "./vom/facts";
|
||||
// Observation handlers — `tool.snapshot`, `tool.get_html`, `tool.screenshot`,
|
||||
// and semantic `tool.observe` (design §7). Each handler resolves the target
|
||||
// tab (defaulting to the Agent Window's active tab when omitted) and
|
||||
@@ -17,7 +18,7 @@ import { ChromiumCdp } from "@/browser-driver/chromium-cdp";
|
||||
import type { CdpTarget } from "@/browser-driver/frame-graph";
|
||||
import {
|
||||
type CaptureSuppressSendToTab,
|
||||
withOverlaysHiddenForCapture,
|
||||
withExtensionOverlayHidden,
|
||||
} from "@/lib/capture-suppress-bridge";
|
||||
import type { SessionContext, SessionManager } from "@/session-manager/manager";
|
||||
import type {
|
||||
@@ -34,6 +35,7 @@ import type {
|
||||
import { attachDialogs, markDialogCursor } from "./dialogs";
|
||||
import { rpcError } from "./errors";
|
||||
import { resolveNodeGeometry } from "./frame-geometry";
|
||||
import { screenshotPageRect } from "./geometry/coordinate-types";
|
||||
import {
|
||||
type ChromeTabsApi,
|
||||
enforceToolTargetScope,
|
||||
@@ -47,15 +49,9 @@ import {
|
||||
type ToolEffect,
|
||||
} from "./shared";
|
||||
import { resolveSnapshotRef } from "./snapshot-ref";
|
||||
import {
|
||||
type CapturedNode,
|
||||
type CapturedSurfaceProbe,
|
||||
type CapturedViewModel,
|
||||
captureViewModel,
|
||||
collectOverlayExcludedBackendIds,
|
||||
probeHoverSurfaces,
|
||||
} from "./vom/capture";
|
||||
import { type CapturedFrameDocument, captureFrameData } from "./vom/frame-capture";
|
||||
import { type CapturedNode, type CapturedSurfaceProbe, probeHoverSurfaces } from "./vom/capture";
|
||||
import { captureObservationFacts, semanticCapture } from "./vom/capture-coordinator";
|
||||
import type { FrameDocument as CapturedFrameDocument } from "./vom/frame-document";
|
||||
import { withOverlayBypass } from "./vom/hover-perception";
|
||||
import { probeTooltipNames } from "./vom/name-enrichment";
|
||||
import {
|
||||
@@ -197,15 +193,14 @@ async function captureElementScreenshot(
|
||||
if (isRpcError(geometry)) return geometry;
|
||||
if (signal?.aborted) return cancelled("screenshot");
|
||||
const rect = geometry.topBounds;
|
||||
const clip = screenshotPageRect(rect, geometry.topViewport);
|
||||
if (!clip) return { code: "cdp_failed", message: "invalid screenshot coordinate space" };
|
||||
|
||||
try {
|
||||
const shot = await cdp.send<{ data?: string }>(tabId, "Page.captureScreenshot", {
|
||||
format: "png",
|
||||
clip: {
|
||||
x: rect.x,
|
||||
y: rect.y,
|
||||
width: rect.width,
|
||||
height: rect.height,
|
||||
...clip.rect,
|
||||
scale: 1,
|
||||
},
|
||||
});
|
||||
@@ -324,7 +319,7 @@ export async function handleScreenshot(
|
||||
tabId: target.tabId,
|
||||
...(node.cdpSessionId ? { sessionId: node.cdpSessionId } : {}),
|
||||
};
|
||||
const captured = await withOverlaysHiddenForCapture(
|
||||
const captured = await withExtensionOverlayHidden(
|
||||
target.tabId,
|
||||
() =>
|
||||
captureElementScreenshot(
|
||||
@@ -356,7 +351,7 @@ export async function handleScreenshot(
|
||||
);
|
||||
}
|
||||
|
||||
const captured = await withOverlaysHiddenForCapture(
|
||||
const captured = await withExtensionOverlayHidden(
|
||||
target.tabId,
|
||||
() => captureFullTabPng(deps, ctx, target, signal),
|
||||
deps.sendToTab,
|
||||
@@ -530,7 +525,7 @@ function buildActiveScopeBlocks(nodes: VomNode[], signals: VomNodeDomSignals): A
|
||||
|
||||
function buildConditionalSurfaces(
|
||||
nodes: VomNode[],
|
||||
captured: CapturedViewModel,
|
||||
captured: CapturedSceneInput,
|
||||
probes: CapturedSurfaceProbe[],
|
||||
): CondSurface[] {
|
||||
if (probes.length === 0) return [];
|
||||
@@ -660,161 +655,9 @@ export interface BuildVomSceneOptions {
|
||||
|
||||
export type VomFrameDocument = CapturedFrameDocument<CdpAxNode>;
|
||||
|
||||
function legacyFrameDocuments(
|
||||
axNodes: CdpAxNode[],
|
||||
captured: CapturedViewModel,
|
||||
pageUrl?: string,
|
||||
rootTarget: VomFrameDocument["target"] = { tabId: 0 },
|
||||
): VomFrameDocument[] {
|
||||
const rootFrameId = captured.rootFrameId ?? "root";
|
||||
const documents: VomFrameDocument[] = [
|
||||
{
|
||||
frameId: rootFrameId,
|
||||
contextScopeId: rootFrameId,
|
||||
target: rootTarget,
|
||||
...(pageUrl ? { url: pageUrl } : {}),
|
||||
axNodes: [],
|
||||
domNodes: captured.nodes.map((node) => ({ ...node, frameId: node.frameId ?? rootFrameId })),
|
||||
},
|
||||
];
|
||||
const pending = [...captured.iframeNodes.entries()];
|
||||
let progress = true;
|
||||
let nextSyntheticFrame = 1;
|
||||
while (pending.length > 0 && progress) {
|
||||
progress = false;
|
||||
for (let index = pending.length - 1; index >= 0; index -= 1) {
|
||||
const [ownerBackendNodeId, domNodes] = pending[index];
|
||||
const parent = documents.find((document) =>
|
||||
document.domNodes.some((node) => node.backendNodeId === ownerBackendNodeId),
|
||||
);
|
||||
if (!parent) continue;
|
||||
const frameId =
|
||||
domNodes.find((node) => node.frameId)?.frameId ?? `legacy-frame-${nextSyntheticFrame++}`;
|
||||
documents.push({
|
||||
frameId,
|
||||
parentFrameId: parent.frameId,
|
||||
ownerBackendNodeId,
|
||||
contextScopeId: frameId,
|
||||
target: parent.target,
|
||||
axNodes: [],
|
||||
domNodes: domNodes.map((node) => ({ ...node, frameId: node.frameId ?? frameId })),
|
||||
});
|
||||
pending.splice(index, 1);
|
||||
progress = true;
|
||||
}
|
||||
}
|
||||
|
||||
const documentByFrameId = new Map(documents.map((document) => [document.frameId, document]));
|
||||
const backendFrame = new Map<number, string>();
|
||||
const childFrameByOwner = new Map<number, string>();
|
||||
for (const document of documents) {
|
||||
if (document.ownerBackendNodeId !== undefined) {
|
||||
childFrameByOwner.set(document.ownerBackendNodeId, document.frameId);
|
||||
}
|
||||
for (const node of document.domNodes) backendFrame.set(node.backendNodeId, document.frameId);
|
||||
}
|
||||
const axById = new Map(axNodes.map((node) => [node.nodeId, node]));
|
||||
const ownership = new Map<string, string>();
|
||||
const resolving = new Set<string>();
|
||||
const frameForAx = (node: CdpAxNode): string => {
|
||||
const cached = ownership.get(node.nodeId);
|
||||
if (cached) return cached;
|
||||
let frameId: string | undefined;
|
||||
if (node.frameId && documentByFrameId.has(node.frameId)) frameId = node.frameId;
|
||||
if (!frameId && typeof node.backendDOMNodeId === "number") {
|
||||
frameId = backendFrame.get(node.backendDOMNodeId);
|
||||
}
|
||||
if (!frameId && node.parentId && !resolving.has(node.nodeId)) {
|
||||
const parent = axById.get(node.parentId);
|
||||
if (parent) {
|
||||
frameId =
|
||||
typeof parent.backendDOMNodeId === "number"
|
||||
? childFrameByOwner.get(parent.backendDOMNodeId)
|
||||
: undefined;
|
||||
if (!frameId) {
|
||||
resolving.add(node.nodeId);
|
||||
frameId = frameForAx(parent);
|
||||
resolving.delete(node.nodeId);
|
||||
}
|
||||
}
|
||||
}
|
||||
frameId ??= rootFrameId;
|
||||
ownership.set(node.nodeId, frameId);
|
||||
return frameId;
|
||||
};
|
||||
for (const node of axNodes) frameForAx(node);
|
||||
for (const node of axNodes) {
|
||||
const frameId = ownership.get(node.nodeId) ?? rootFrameId;
|
||||
const document = documentByFrameId.get(frameId) ?? documents[0];
|
||||
document.axNodes.push({
|
||||
...node,
|
||||
frameId,
|
||||
...(node.parentId && ownership.get(node.parentId) === frameId
|
||||
? { parentId: node.parentId }
|
||||
: { parentId: undefined }),
|
||||
...(node.childIds
|
||||
? { childIds: node.childIds.filter((childId) => ownership.get(childId) === frameId) }
|
||||
: {}),
|
||||
});
|
||||
}
|
||||
return documents;
|
||||
}
|
||||
|
||||
function withLegacyBackendIds(scene: VomScene): VomScene {
|
||||
const used = new Set<number>();
|
||||
const idMap = new Map<number, number>();
|
||||
let nextVirtualId = -1;
|
||||
for (const node of scene.nodes) {
|
||||
const preferred = node.backendNodeId;
|
||||
const id = preferred !== undefined && !used.has(preferred) ? preferred : nextVirtualId--;
|
||||
used.add(id);
|
||||
idMap.set(node.id, id);
|
||||
}
|
||||
return {
|
||||
...scene,
|
||||
nodes: scene.nodes.map((node) => ({
|
||||
...node,
|
||||
id: idMap.get(node.id) as number,
|
||||
parentId: node.parentId === null ? null : (idMap.get(node.parentId) ?? null),
|
||||
...(node.domParentId !== undefined
|
||||
? { domParentId: node.domParentId === null ? null : (idMap.get(node.domParentId) ?? null) }
|
||||
: {}),
|
||||
...(node.domAncestorIds
|
||||
? { domAncestorIds: node.domAncestorIds.flatMap((id) => idMap.get(id) ?? []) }
|
||||
: {}),
|
||||
})),
|
||||
...(scene.surfaces
|
||||
? {
|
||||
surfaces: scene.surfaces.map((surface) => ({
|
||||
...surface,
|
||||
triggerId: idMap.get(surface.triggerId) ?? surface.triggerId,
|
||||
})),
|
||||
}
|
||||
: {}),
|
||||
...(scene.activeScopeBlocks
|
||||
? {
|
||||
activeScopeBlocks: scene.activeScopeBlocks.map((block) => ({
|
||||
...block,
|
||||
triggerId: idMap.get(block.triggerId) ?? block.triggerId,
|
||||
})),
|
||||
}
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
|
||||
export function buildVomScene(
|
||||
axNodes: CdpAxNode[],
|
||||
captured: CapturedViewModel,
|
||||
options: BuildVomSceneOptions = {},
|
||||
): VomScene {
|
||||
return withLegacyBackendIds(
|
||||
buildFrameVomScene(legacyFrameDocuments(axNodes, captured, options.pageUrl), captured, options),
|
||||
);
|
||||
}
|
||||
|
||||
export function buildFrameVomScene(
|
||||
documents: VomFrameDocument[],
|
||||
captured: CapturedViewModel,
|
||||
captured: CapturedSceneInput,
|
||||
options: BuildVomSceneOptions = {},
|
||||
): VomScene {
|
||||
const scene = buildSemanticVomScene({
|
||||
@@ -830,7 +673,7 @@ export function buildFrameVomScene(
|
||||
function attachCapturedSceneAnnotations(
|
||||
scene: VomScene,
|
||||
documents: VomFrameDocument[],
|
||||
captured: CapturedViewModel,
|
||||
captured: CapturedSceneInput,
|
||||
surfaceProbes: CapturedSurfaceProbe[],
|
||||
): VomScene {
|
||||
const rootDocument = documents.find((document) => document.frameId === scene.rootFrameId);
|
||||
@@ -980,51 +823,6 @@ export async function handleGetHtml(
|
||||
}
|
||||
}
|
||||
|
||||
function emptyCapturedViewModel(viewport = { width: 0, height: 0 }): CapturedViewModel {
|
||||
return { viewport, nodes: [], iframeNodes: new Map(), excludedBackendNodeIds: new Set() };
|
||||
}
|
||||
|
||||
interface LayoutMetricsViewportReply {
|
||||
cssLayoutViewport?: { clientWidth?: number; clientHeight?: number };
|
||||
layoutViewport?: { clientWidth?: number; clientHeight?: number };
|
||||
}
|
||||
|
||||
async function fallbackCapturedViewModel(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
signal?: AbortSignal,
|
||||
): Promise<CapturedViewModel> {
|
||||
throwIfAborted(signal, "observation");
|
||||
let viewport = { width: 0, height: 0 };
|
||||
try {
|
||||
const metrics = await cdp.send<LayoutMetricsViewportReply>(tabId, "Page.getLayoutMetrics", {});
|
||||
throwIfAborted(signal, "observation");
|
||||
const source = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
|
||||
viewport = {
|
||||
width: source.clientWidth ?? 0,
|
||||
height: source.clientHeight ?? 0,
|
||||
};
|
||||
} catch (error) {
|
||||
if (isAbortError(error)) throw error;
|
||||
}
|
||||
const excludedBackendNodeIds = await collectOverlayExcludedBackendIds(cdp, tabId, signal);
|
||||
throwIfAborted(signal, "observation");
|
||||
return { ...emptyCapturedViewModel(viewport), excludedBackendNodeIds };
|
||||
}
|
||||
|
||||
async function captureForVom(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
options: CaptureVomObservationOptions,
|
||||
): Promise<CapturedViewModel> {
|
||||
try {
|
||||
return await captureViewModel(cdp, tabId, { signal: options.signal });
|
||||
} catch (error) {
|
||||
if (isAbortError(error)) throw error;
|
||||
return fallbackCapturedViewModel(cdp, tabId, options.signal);
|
||||
}
|
||||
}
|
||||
|
||||
export interface HoverProbeOutcome {
|
||||
/** Whether any active hover was dispatched during this observation. */
|
||||
performed: boolean;
|
||||
@@ -1057,7 +855,7 @@ const NO_HOVER_PROBES: HoverProbeOutcome = {
|
||||
async function runHoverProbes(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
captured: CapturedViewModel,
|
||||
captured: CapturedSceneInput,
|
||||
documents: VomFrameDocument[],
|
||||
staticSemantics: ReturnType<typeof resolveSemanticGraph>,
|
||||
options: CaptureVomObservationOptions,
|
||||
@@ -1096,14 +894,9 @@ export async function captureVomObservation(
|
||||
throwIfAborted(options.signal, "observation");
|
||||
await cdp.ensureAttachedToUrl?.(tabId, url);
|
||||
throwIfAborted(options.signal, "observation");
|
||||
const captured = await captureForVom(cdp, tabId, options);
|
||||
const facts = await captureObservationFacts<CdpAxNode>(cdp, tabId, options.signal, url);
|
||||
const { captured, documents: normalizedDocuments } = semanticCapture(facts);
|
||||
throwIfAborted(options.signal, "observation");
|
||||
const documents = await captureFrameData<CdpAxNode>(cdp, tabId, captured, options.signal);
|
||||
throwIfAborted(options.signal, "observation");
|
||||
const normalizedDocuments =
|
||||
documents.length === 1 && captured.iframeNodes.size > 0
|
||||
? legacyFrameDocuments(documents[0].axNodes, captured, url, documents[0].target)
|
||||
: documents;
|
||||
const semanticGraph = buildSemanticGraph({
|
||||
documents: normalizedDocuments,
|
||||
viewport: captured.viewport,
|
||||
@@ -1131,12 +924,31 @@ export async function captureVomObservation(
|
||||
captured,
|
||||
hoverProbes.surfaceProbes,
|
||||
);
|
||||
// Reserve space using the renderer's character-based token estimate. Like VOM
|
||||
// headers, this integrity notice remains visible even under a tiny token budget.
|
||||
const notices: string[] = [];
|
||||
if (facts.issues.some((issue) => issue.stage === "geometry"))
|
||||
notices.push(
|
||||
"@warning geometry incomplete: some page or frame content has no top-level coordinates.",
|
||||
);
|
||||
const incompleteStages = ["dom", "ax", "forms", "ownership"].filter((stage) =>
|
||||
facts.issues.some((issue) => issue.stage === stage),
|
||||
);
|
||||
if (incompleteStages.length)
|
||||
notices.push(
|
||||
`@warning observation incomplete: some ${incompleteStages.join(", ")} data is unavailable or omitted.`,
|
||||
);
|
||||
const captureNotice = notices.join("\n");
|
||||
const rendered = renderVom(decoratedScene, {
|
||||
maxDepth: options.maxDepth,
|
||||
maxTokens: options.maxTokens,
|
||||
maxTokens:
|
||||
!captureNotice || options.maxTokens === undefined
|
||||
? options.maxTokens
|
||||
: Math.max(0, options.maxTokens - Math.ceil((captureNotice.length + 1) / 4)),
|
||||
redactValues: options.redactValues,
|
||||
activeRegionPolicy: options.activeRegionPolicy,
|
||||
});
|
||||
if (captureNotice) rendered.text += `\n${captureNotice}`;
|
||||
throwIfAborted(options.signal, "observation");
|
||||
return projectRecordSafeObservation({
|
||||
rootFrameId: captured.rootFrameId ?? normalizedDocuments[0]?.frameId ?? "root",
|
||||
|
||||
@@ -2,6 +2,7 @@ import { type SessionManager, SessionStartCleanupError } from "@/session-manager
|
||||
import type { RpcError } from "@/transport/types";
|
||||
import { rpcError } from "./errors";
|
||||
import { clearRecordingForSession } from "./record";
|
||||
import type { CdpRunner, ChromeTabsApi } from "./shared";
|
||||
import { isRpcError } from "./shared";
|
||||
import { returnBorrowedTab, type TabManagementDeps } from "./tabs";
|
||||
|
||||
@@ -73,11 +74,17 @@ export interface SessionStopResult {
|
||||
/** Tab ids whose return path failed; those entries remain borrowed so
|
||||
* shutdown can be retried without closing the Agent Window. */
|
||||
return_failures?: Array<{ tab_id: number; code: string; message: string }>;
|
||||
/**
|
||||
* True when the Agent Window was *released* to the user (instead of being
|
||||
* closed) because it still contained user-created tabs after the agent's own
|
||||
* tabs were closed. Single-session scope.
|
||||
*/
|
||||
window_released?: boolean;
|
||||
}
|
||||
|
||||
export interface SessionStopDeps {
|
||||
signal?: AbortSignal;
|
||||
cdp?: {
|
||||
cdp?: Pick<CdpRunner, "releaseSessionTab"> & {
|
||||
detachSession(sessionId: string): Promise<void>;
|
||||
};
|
||||
/**
|
||||
@@ -87,6 +94,13 @@ export interface SessionStopDeps {
|
||||
* without a real browser.
|
||||
*/
|
||||
tabManagement?: TabManagementDeps;
|
||||
/**
|
||||
* Read-only Chrome tabs API used to *query* remaining tabs before deciding
|
||||
* whether to release the window. Kept separate from `tabManagement.tabs`
|
||||
* (a `TabMutationApi` that has no `query`) so the mutation interface stays
|
||||
* pure. Tests can inject a fake `ChromeTabsApi` too.
|
||||
*/
|
||||
tabsQuery?: ChromeTabsApi;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -152,8 +166,10 @@ export async function handleSessionStart(
|
||||
* cleanly (review parity with M6).
|
||||
* 3. Detach CDP sessions the extension still holds for this
|
||||
* session (no-op if M6/M7 didn't attach to any tab).
|
||||
* 4. Close the Agent Window. SessionManager.stop() removes the
|
||||
* Chrome window and forgets the context.
|
||||
* 4. Close the agent-created tabs and the home tab (design §3.2).
|
||||
* 5. If any tab remains — those are user-created tabs — release the
|
||||
* window to the user (dropOnly) instead of closing it, so user
|
||||
* tabs survive. Otherwise close the window as before.
|
||||
*
|
||||
* Failures in step 1 keep the Agent Window open: a failed borrowed tab
|
||||
* may still be there, so closing the window would risk losing user
|
||||
@@ -188,15 +204,19 @@ export async function handleSessionStop(
|
||||
const returnFailures: SessionStopResult["return_failures"] = [];
|
||||
const borrowedIds = Array.from(ctx.borrowedTabs.keys());
|
||||
for (const tabId of borrowedIds) {
|
||||
// A close event may have removed this borrow while another tab returned.
|
||||
if (!ctx.borrowedTabs.has(tabId)) continue;
|
||||
try {
|
||||
const tabManagement = {
|
||||
...(deps.tabManagement ?? {}),
|
||||
cdp: deps.cdp ?? deps.tabManagement?.cdp,
|
||||
signal: deps.signal,
|
||||
isAgentWindowId:
|
||||
deps.tabManagement?.isAgentWindowId ??
|
||||
((windowId: number) => manager.findByWindowId(windowId) !== null),
|
||||
};
|
||||
const outcome = await returnBorrowedTab(ctx, tabId, tabManagement);
|
||||
if (!ctx.borrowedTabs.has(tabId)) continue;
|
||||
if (typeof outcome === "object" && "code" in outcome) {
|
||||
console.warn(`[bsk session_stop] auto-return failed for tab ${tabId}`, outcome);
|
||||
returnFailures?.push({
|
||||
@@ -209,6 +229,7 @@ export async function handleSessionStop(
|
||||
ctx.borrowedTabs.delete(tabId);
|
||||
}
|
||||
} catch (err) {
|
||||
if (!ctx.borrowedTabs.has(tabId)) continue;
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
console.warn(`[bsk session_stop] auto-return threw for tab ${tabId}: ${message}`);
|
||||
returnFailures?.push({
|
||||
@@ -219,8 +240,15 @@ export async function handleSessionStop(
|
||||
}
|
||||
}
|
||||
|
||||
// A previously failed tab may have closed during a later return. Only
|
||||
// confirmed close events remove borrows; ordinary move failures stay retryable.
|
||||
const pendingReturnFailures = returnFailures.filter((failure) =>
|
||||
ctx.borrowedTabs.has(failure.tab_id),
|
||||
);
|
||||
if (deps.signal?.aborted) {
|
||||
const nonCancelFailures = returnFailures?.filter((failure) => failure.code !== "cancelled");
|
||||
const nonCancelFailures = pendingReturnFailures.filter(
|
||||
(failure) => failure.code !== "cancelled",
|
||||
);
|
||||
if (nonCancelFailures && nonCancelFailures.length > 0) {
|
||||
return {
|
||||
...(returnedTabIds.length > 0 ? { returned_tab_ids: returnedTabIds } : {}),
|
||||
@@ -232,8 +260,8 @@ export async function handleSessionStop(
|
||||
|
||||
const result: SessionStopResult = {};
|
||||
if (returnedTabIds.length > 0) result.returned_tab_ids = returnedTabIds;
|
||||
if (returnFailures && returnFailures.length > 0) {
|
||||
result.return_failures = returnFailures;
|
||||
if (pendingReturnFailures.length > 0) {
|
||||
result.return_failures = pendingReturnFailures;
|
||||
// A failed return means at least one borrowed user tab may still be
|
||||
// inside the Agent Window. Keep the session/window alive so the user
|
||||
// can retry `bsk session stop` or explicitly `bsk tab return` after the
|
||||
@@ -253,8 +281,66 @@ export async function handleSessionStop(
|
||||
return { code: "cancelled", message: "session_stop aborted before window close" };
|
||||
}
|
||||
|
||||
// Step 4: close the Agent Window and drop the context.
|
||||
await manager.stop(params.session_id);
|
||||
// Step 4: close every tab explicitly created by the agent, including the
|
||||
// home tab. `tabsApi` is a
|
||||
// TabMutationApi (remove only); `queryApi` is a separate read-only
|
||||
// ChromeTabsApi. If neither is injected, we conservatively fall back to
|
||||
// closing the window (see Step 5).
|
||||
const tabsApi = deps.tabManagement?.tabs;
|
||||
const queryApi = deps.tabsQuery;
|
||||
|
||||
if (tabsApi) {
|
||||
// Close each agent-created tab that still exists.
|
||||
const agentCreatedTabIds = Array.from(ctx.agentCreatedTabs);
|
||||
for (const tabId of agentCreatedTabIds) {
|
||||
try {
|
||||
await tabsApi.remove(tabId);
|
||||
ctx.agentCreatedTabs.delete(tabId);
|
||||
} catch (err) {
|
||||
// Tab may already be gone (closed by the user). Non-fatal.
|
||||
console.warn(`[bsk session_stop] failed to close agent tab ${tabId}`, err);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Step 5: decide whether to release (keep) the window or close it.
|
||||
let shouldRelease = false;
|
||||
if (queryApi) {
|
||||
try {
|
||||
const liveWindowTabs = await queryApi.query({ windowId: ctx.agentWindowId });
|
||||
// Only genuine *user* tabs count toward keeping the window open.
|
||||
// An agent tab that failed to close in Step 4 may still be present
|
||||
// here; if we counted it as a reason to release (dropOnly), the
|
||||
// window would be kept and that agent tab would leak (issue #57
|
||||
// regression). Exclude any id still tracked in agentCreatedTabs.
|
||||
const userTabs = liveWindowTabs.filter((t) => {
|
||||
if (t.id === undefined) return false;
|
||||
return !ctx.agentCreatedTabs.has(t.id);
|
||||
});
|
||||
const leakedAgentTabs = liveWindowTabs.filter(
|
||||
(t) => t.id !== undefined && ctx.agentCreatedTabs.has(t.id),
|
||||
);
|
||||
if (leakedAgentTabs.length > 0) {
|
||||
console.warn(
|
||||
`[bsk session_stop] ${leakedAgentTabs.length} agent tab(s) failed to close; forcing window close instead of release`,
|
||||
leakedAgentTabs.map((t) => t.id),
|
||||
);
|
||||
}
|
||||
shouldRelease = userTabs.length > 0;
|
||||
} catch {
|
||||
// Query failed (e.g. window already gone) — conservatively close it.
|
||||
shouldRelease = false;
|
||||
}
|
||||
}
|
||||
|
||||
if (shouldRelease) {
|
||||
// Keep the window + its user tabs; only drop the session binding.
|
||||
await manager.stop(params.session_id, { dropOnly: true });
|
||||
result.window_released = true;
|
||||
} else {
|
||||
// Window is empty (or we couldn't verify state) — close it.
|
||||
await manager.stop(params.session_id);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
@@ -57,6 +57,7 @@ export interface CdpRunner {
|
||||
getFrameGraph?(tabId: number): Promise<CdpFrameGraph>;
|
||||
ensureAttachedToUrl?(tabId: number, expectedUrl: string | undefined): Promise<void>;
|
||||
trackSessionTab?(sessionId: string, tabId: number): void;
|
||||
releaseSessionTab?(sessionId: string, tabId: number): Promise<void>;
|
||||
onEvent?(handler: (source: CdpDebuggee, method: string, params: unknown) => void): {
|
||||
dispose(): void;
|
||||
};
|
||||
|
||||
@@ -10,7 +10,7 @@ import {
|
||||
import type { SessionContext, SessionManager } from "@/session-manager/manager";
|
||||
import type { RpcError } from "@/transport/types";
|
||||
import { rpcError } from "./errors";
|
||||
import { isRpcError, lookupSession } from "./shared";
|
||||
import { type CdpRunner, isRpcError, lookupSession } from "./shared";
|
||||
|
||||
export type TabScope = "user" | "agent" | "all";
|
||||
|
||||
@@ -287,6 +287,10 @@ export interface TabManagementDeps {
|
||||
approveBorrow?: BorrowConfirmationApprover;
|
||||
/** Clears Agent-scoped overlays after a borrowed tab is returned. */
|
||||
agentOverlayReset?: AgentOverlayResetApi;
|
||||
/** Releases this session's CDP claim after a borrowed tab is returned. */
|
||||
cdp?: Pick<CdpRunner, "releaseSessionTab">;
|
||||
/** Runs after tab_return validation, before moving the borrowed tab. */
|
||||
beforeReturn?: (sessionId: string, tabId: number) => Promise<void>;
|
||||
/**
|
||||
* Reports whether `windowId` is any live session's Agent Window.
|
||||
* `tab_return`'s fallback window picker uses this to avoid moving a
|
||||
@@ -369,6 +373,7 @@ function buildCreateProps(
|
||||
* Returns the created tab on success, or an `RpcError` on failure.
|
||||
*/
|
||||
async function createTabAndCleanup(
|
||||
ctx: SessionContext,
|
||||
deps: TabManagementDeps,
|
||||
createProps: chrome.tabs.CreateProperties,
|
||||
): Promise<CreatedChromeTab | RpcError> {
|
||||
@@ -381,25 +386,28 @@ async function createTabAndCleanup(
|
||||
message: err instanceof Error ? err.message : String(err),
|
||||
};
|
||||
}
|
||||
if (aborted(deps.signal, "tab_create")) {
|
||||
// We already opened the tab; close it on abort so we don't leak.
|
||||
if (typeof tab.id === "number") {
|
||||
try {
|
||||
await getTabsApi(deps).remove(tab.id);
|
||||
} catch (cleanupErr) {
|
||||
return rpcError(
|
||||
"protocol_error",
|
||||
"cleanup_failed",
|
||||
`tab_create aborted but cleanup of tab ${tab.id} failed: ${describeError(cleanupErr)}`,
|
||||
{ resource_type: "tab", resource_id: tab.id },
|
||||
);
|
||||
}
|
||||
}
|
||||
return { code: "cancelled", message: "tab_create aborted" };
|
||||
}
|
||||
if (typeof tab.id !== "number") {
|
||||
return { code: "protocol_error", message: "chrome.tabs.create returned no tab id" };
|
||||
}
|
||||
// Claim the concrete id returned by Chrome. Ownership never depends on
|
||||
// matching this request to an asynchronous onCreated event.
|
||||
ctx.agentCreatedTabs.add(tab.id);
|
||||
if (aborted(deps.signal, "tab_create")) {
|
||||
try {
|
||||
await getTabsApi(deps).remove(tab.id);
|
||||
ctx.agentCreatedTabs.delete(tab.id);
|
||||
} catch (cleanupErr) {
|
||||
// Keep the claim when cleanup fails so session_stop can retry instead
|
||||
// of releasing an agent-owned tab to the user.
|
||||
return rpcError(
|
||||
"protocol_error",
|
||||
"cleanup_failed",
|
||||
`tab_create aborted but cleanup of tab ${tab.id} failed: ${describeError(cleanupErr)}`,
|
||||
{ resource_type: "tab", resource_id: tab.id },
|
||||
);
|
||||
}
|
||||
return { code: "cancelled", message: "tab_create aborted" };
|
||||
}
|
||||
return { ...tab, id: tab.id };
|
||||
}
|
||||
|
||||
@@ -426,7 +434,7 @@ export async function handleTabCreate(
|
||||
const paramErr = validateTabCreateParams(params);
|
||||
if (paramErr) return paramErr;
|
||||
|
||||
const tab = await createTabAndCleanup(deps, buildCreateProps(ctx, params));
|
||||
const tab = await createTabAndCleanup(ctx, deps, buildCreateProps(ctx, params));
|
||||
if (isRpcError(tab)) return tab;
|
||||
|
||||
return {
|
||||
@@ -526,6 +534,9 @@ export async function handleTabClose(
|
||||
}
|
||||
try {
|
||||
await getTabsApi(deps).remove(params.tab_id);
|
||||
// Keep the tracking set accurate so session_stop won't try to close a
|
||||
// tab that's already gone (design §3.1).
|
||||
ctx.agentCreatedTabs.delete(params.tab_id);
|
||||
} catch (err) {
|
||||
return {
|
||||
code: "protocol_error",
|
||||
@@ -753,13 +764,6 @@ async function executeBorrowCore(
|
||||
);
|
||||
if (moveErr) return moveErr;
|
||||
|
||||
// Best-effort activation — failure is non-fatal.
|
||||
try {
|
||||
await p.tabsApi.update(p.tabId, { active: true });
|
||||
} catch (err) {
|
||||
console.debug("[bsk tab_borrow] activate after move failed", err);
|
||||
}
|
||||
|
||||
return { originalWindowId, originalIndex };
|
||||
}
|
||||
|
||||
@@ -823,6 +827,12 @@ export async function handleTabBorrow(
|
||||
message: err instanceof Error ? err.message : String(err),
|
||||
};
|
||||
}
|
||||
// Activate after commit so overlay/event observers see the tab as claimed.
|
||||
try {
|
||||
await tabsApi.update(params.tab_id, { active: true });
|
||||
} catch (err) {
|
||||
console.debug("[bsk tab_borrow] activate after move failed", err);
|
||||
}
|
||||
return {
|
||||
tab_id: params.tab_id,
|
||||
original_window_id: coreResult.originalWindowId,
|
||||
@@ -939,16 +949,22 @@ async function cleanupUnusedFallbackWindow(
|
||||
}
|
||||
}
|
||||
|
||||
function resetAgentOverlaysInReturnedTab(
|
||||
async function releaseReturnedTabState(
|
||||
ctx: SessionContext,
|
||||
tabId: number,
|
||||
deps: TabManagementDeps,
|
||||
): void {
|
||||
): Promise<void> {
|
||||
void getAgentOverlayResetApi(deps)
|
||||
.resetAgentOverlays(tabId, ctx.sessionId)
|
||||
.catch((err) => {
|
||||
console.debug("[bsk tab_return] agent overlay reset failed", err);
|
||||
});
|
||||
try {
|
||||
await deps.cdp?.releaseSessionTab?.(ctx.sessionId, tabId);
|
||||
} catch (err) {
|
||||
// A successful move must not be retried because debugger cleanup failed.
|
||||
console.debug("[bsk tab_return] CDP release failed", err);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -1022,7 +1038,7 @@ export async function returnBorrowedTab(
|
||||
});
|
||||
const movedTab = Array.isArray(moved) ? moved[0] : moved;
|
||||
const finalIndex = typeof movedTab?.index === "number" ? movedTab.index : targetIndex;
|
||||
resetAgentOverlaysInReturnedTab(ctx, tabId, deps);
|
||||
await releaseReturnedTabState(ctx, tabId, deps);
|
||||
return {
|
||||
tabId,
|
||||
toWindowId: targetWindowId,
|
||||
@@ -1064,7 +1080,7 @@ export async function returnBorrowedTab(
|
||||
});
|
||||
const movedTab = Array.isArray(moved) ? moved[0] : moved;
|
||||
const finalIndex = typeof movedTab?.index === "number" ? movedTab.index : target.index;
|
||||
resetAgentOverlaysInReturnedTab(ctx, tabId, deps);
|
||||
await releaseReturnedTabState(ctx, tabId, deps);
|
||||
return {
|
||||
tabId,
|
||||
toWindowId: target.windowId,
|
||||
@@ -1110,6 +1126,7 @@ export async function handleTabReturn(
|
||||
message: `tab_return: tab ${params.tab_id} is not borrowed by this session`,
|
||||
};
|
||||
}
|
||||
await deps.beforeReturn?.(ctx.sessionId, params.tab_id);
|
||||
const outcome = await returnBorrowedTab(ctx, params.tab_id, {
|
||||
...deps,
|
||||
isAgentWindowId:
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
// Shared time boundary for browser-side file-transfer transactions. Concrete
|
||||
// upload/download mechanisms own their state machines and effect semantics.
|
||||
|
||||
export class BoundedWaitError extends Error {
|
||||
constructor(
|
||||
readonly kind: "timeout" | "aborted",
|
||||
message: string,
|
||||
) {
|
||||
super(message);
|
||||
}
|
||||
}
|
||||
|
||||
export function remainingMs(deadline: number): number {
|
||||
return Math.max(0, deadline - Date.now());
|
||||
}
|
||||
|
||||
export async function waitBounded<T>(
|
||||
promise: Promise<T>,
|
||||
deadline: number,
|
||||
signal: AbortSignal | undefined,
|
||||
timeoutMessage: string,
|
||||
): Promise<T> {
|
||||
const remaining = remainingMs(deadline);
|
||||
if (signal?.aborted) throw new BoundedWaitError("aborted", "file transfer aborted");
|
||||
if (remaining === 0) throw new BoundedWaitError("timeout", timeoutMessage);
|
||||
|
||||
let timer: ReturnType<typeof setTimeout> | undefined;
|
||||
let onAbort: (() => void) | undefined;
|
||||
const boundary = new Promise<never>((_resolve, reject) => {
|
||||
timer = setTimeout(() => reject(new BoundedWaitError("timeout", timeoutMessage)), remaining);
|
||||
if (signal) {
|
||||
onAbort = () => reject(new BoundedWaitError("aborted", "file transfer aborted"));
|
||||
signal.addEventListener("abort", onAbort, { once: true });
|
||||
}
|
||||
});
|
||||
try {
|
||||
return await Promise.race([promise, boundary]);
|
||||
} finally {
|
||||
if (timer) clearTimeout(timer);
|
||||
if (signal && onAbort) signal.removeEventListener("abort", onAbort);
|
||||
}
|
||||
}
|
||||
@@ -2,8 +2,13 @@
|
||||
// target, then delegate the browser protocol transaction to the file-input
|
||||
// transaction module.
|
||||
|
||||
import {
|
||||
type CaptureSuppressSendToTab,
|
||||
withExtensionOverlayHidden,
|
||||
} from "@/lib/capture-suppress-bridge";
|
||||
import type { SessionManager } from "@/session-manager/manager";
|
||||
import type { RpcError, UploadParams, UploadResult } from "@/transport/types";
|
||||
import { uploadThroughFileDrop } from "./file-drop-transaction";
|
||||
import { uploadThroughActivatedFileInput } from "./file-input-transaction";
|
||||
import { clickResolvedTarget, type InteractionDeps, resolveActionTarget } from "./interaction";
|
||||
import {
|
||||
@@ -18,6 +23,7 @@ const DEFAULT_TIMEOUT_MS = 120_000;
|
||||
|
||||
export interface UploadDeps extends InteractionDeps {
|
||||
cdp: CdpRunner;
|
||||
sendToTab?: CaptureSuppressSendToTab;
|
||||
}
|
||||
|
||||
export async function handleUpload(
|
||||
@@ -41,19 +47,39 @@ export async function handleUpload(
|
||||
const address = await resolveActionTarget(deps.cdp, ctx, target, params, "upload");
|
||||
if (isRpcError(address)) return address;
|
||||
const timeoutMs = params.timeout_ms ?? DEFAULT_TIMEOUT_MS;
|
||||
const transaction = await uploadThroughActivatedFileInput({
|
||||
cdp: deps.cdp,
|
||||
actionTarget: address,
|
||||
files: params.files.map((file) => file.staged_path as string),
|
||||
timeoutMs,
|
||||
signal: deps.signal,
|
||||
trigger: () => clickResolvedTarget(ctx, address, {}, deps),
|
||||
});
|
||||
if (isRpcError(transaction)) return transaction;
|
||||
const files = params.files.map((file) => file.staged_path as string);
|
||||
const mode = params.mode ?? "input";
|
||||
let usedRef = address.usedRef;
|
||||
let usedSelector = address.usedSelector;
|
||||
if (mode === "drop") {
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
const transaction = await uploadThroughFileDrop({
|
||||
cdp: deps.cdp,
|
||||
actionTarget: address,
|
||||
files,
|
||||
timeoutMs,
|
||||
signal: deps.signal,
|
||||
withOverlayHidden: (operation) =>
|
||||
withExtensionOverlayHidden(target.tabId, operation, deps.sendToTab),
|
||||
});
|
||||
if (isRpcError(transaction)) return transaction;
|
||||
} else {
|
||||
const transaction = await uploadThroughActivatedFileInput({
|
||||
cdp: deps.cdp,
|
||||
actionTarget: address,
|
||||
files,
|
||||
timeoutMs,
|
||||
signal: deps.signal,
|
||||
trigger: () => clickResolvedTarget(ctx, address, {}, deps),
|
||||
});
|
||||
if (isRpcError(transaction)) return transaction;
|
||||
usedRef = transaction.click.used_ref;
|
||||
usedSelector = transaction.click.used_selector;
|
||||
}
|
||||
return {
|
||||
tab_id: target.tabId,
|
||||
used_ref: transaction.click.used_ref,
|
||||
used_selector: transaction.click.used_selector,
|
||||
used_ref: usedRef,
|
||||
used_selector: usedSelector,
|
||||
file_names: params.files.map((file) => file.name),
|
||||
};
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,6 +1,7 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { OVERLAY_HOST_MARKER_ATTR, OVERLAY_HOST_NAME } from "../../../lib/overlay-bridge";
|
||||
import { captureViewModel, collectOverlayExcludedBackendIds, probeHoverSurfaces } from "../capture";
|
||||
import { collectOverlayExcludedBackendIds, probeHoverSurfaces } from "../capture";
|
||||
import { captureObservationFacts, semanticCapture } from "../capture-coordinator";
|
||||
|
||||
// Minimal but format-accurate captureSnapshot reply: a body with one
|
||||
// fixed full-screen overlay div carrying a password input.
|
||||
@@ -26,6 +27,7 @@ function fakeSnapshotReply() {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
frameId: "root",
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1, 2],
|
||||
nodeType: [1, 1, 1, 1],
|
||||
@@ -73,6 +75,7 @@ function hoverTriggerSnapshotReply() {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
frameId: "root",
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1],
|
||||
nodeType: [1, 1, 1],
|
||||
@@ -117,6 +120,7 @@ function twoHoverTriggerSnapshotReply() {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
frameId: "root",
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1, 1],
|
||||
nodeType: [1, 1, 1, 1],
|
||||
@@ -170,6 +174,7 @@ function nestedHoverTriggerSnapshotReply() {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
frameId: "root",
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1, 2, 3],
|
||||
nodeType: [1, 1, 1, 1, 1],
|
||||
@@ -210,38 +215,48 @@ function makeCdp(snapshot: unknown) {
|
||||
if (method === "DOMSnapshot.enable") return {};
|
||||
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 } };
|
||||
return {
|
||||
visualViewport: { clientWidth: 1000 },
|
||||
cssVisualViewport: { clientWidth: 1000 },
|
||||
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
|
||||
};
|
||||
}
|
||||
if (method === "Runtime.evaluate") {
|
||||
return {
|
||||
result: {
|
||||
value: {
|
||||
controls: [
|
||||
deepSerializedValue: {
|
||||
type: "array",
|
||||
value: [
|
||||
{
|
||||
state: "filled",
|
||||
sensitive: true,
|
||||
defaultValue: "hunter2",
|
||||
value: "hunter2",
|
||||
placeholder: "secret",
|
||||
type: "array",
|
||||
value: [
|
||||
{ type: "node", value: { backendNodeId: 13 } },
|
||||
{
|
||||
type: "string",
|
||||
value: JSON.stringify({
|
||||
state: "filled",
|
||||
sensitive: true,
|
||||
placeholder: "secret",
|
||||
}),
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
childFrames: [],
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
if (method === "Runtime.releaseObjectGroup") return {};
|
||||
throw new Error(`unexpected ${method}`);
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
}
|
||||
|
||||
describe("captureViewModel", () => {
|
||||
describe("snapshot capture and hover", () => {
|
||||
it("parses nodes, attrs, rects, paint order and styles", async () => {
|
||||
const { nodes, viewport, iframeNodes } = await captureViewModel(
|
||||
makeCdp(fakeSnapshotReply()),
|
||||
4,
|
||||
);
|
||||
expect(iframeNodes.size).toBe(0);
|
||||
const { nodes, viewport } = semanticCapture(
|
||||
await captureObservationFacts(makeCdp(fakeSnapshotReply()), 4),
|
||||
).captured;
|
||||
expect(viewport).toEqual({ width: 1000, height: 800 });
|
||||
const div = nodes.find((n) => n.backendNodeId === 12);
|
||||
expect(div).toMatchObject({ tag: "div", position: "fixed", parentBackendNodeId: 11 });
|
||||
@@ -253,32 +268,11 @@ describe("captureViewModel", () => {
|
||||
|
||||
it("never hovers the page while capturing", async () => {
|
||||
const cdp = makeCdp(fakeSnapshotReply());
|
||||
await captureViewModel(cdp, 4);
|
||||
semanticCapture(await captureObservationFacts(cdp, 4)).captured;
|
||||
|
||||
expect(cdp.send).not.toHaveBeenCalledWith(4, "Input.dispatchMouseEvent", expect.anything());
|
||||
});
|
||||
|
||||
it("batch-enriches form controls without per-node object resolution", async () => {
|
||||
const cdp = makeCdp(fakeSnapshotReply());
|
||||
const result = await captureViewModel(cdp, 4);
|
||||
const input = result.nodes.find((node) => node.backendNodeId === 13);
|
||||
|
||||
expect(input).toMatchObject({
|
||||
formState: "filled",
|
||||
formPlaceholder: "secret",
|
||||
attrs: { type: "password" },
|
||||
});
|
||||
expect(input?.formValue).toBeUndefined();
|
||||
expect(input?.formDefaultValue).toBeUndefined();
|
||||
expect(cdp.send).toHaveBeenCalledWith(
|
||||
4,
|
||||
"Runtime.evaluate",
|
||||
expect.objectContaining({ returnByValue: true }),
|
||||
);
|
||||
expect(cdp.send).not.toHaveBeenCalledWith(4, "DOM.resolveNode", expect.anything());
|
||||
expect(cdp.send).not.toHaveBeenCalledWith(4, "Runtime.callFunctionOn", expect.anything());
|
||||
});
|
||||
|
||||
it("runs hover surface probes when explicitly enabled", async () => {
|
||||
let hoverStateCalls = 0;
|
||||
const cdp = {
|
||||
@@ -287,6 +281,8 @@ describe("captureViewModel", () => {
|
||||
if (method === "DOMSnapshot.captureSnapshot") return hoverTriggerSnapshotReply();
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return {
|
||||
visualViewport: { clientWidth: 1000 },
|
||||
cssVisualViewport: { clientWidth: 1000 },
|
||||
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
|
||||
};
|
||||
}
|
||||
@@ -318,7 +314,7 @@ describe("captureViewModel", () => {
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
|
||||
const captured = await captureViewModel(cdp, 4);
|
||||
const captured = semanticCapture(await captureObservationFacts(cdp, 4)).captured;
|
||||
const surfaceProbes = await probeHoverSurfaces(cdp, 4, captured.nodes);
|
||||
|
||||
expect(surfaceProbes).toEqual([
|
||||
@@ -350,6 +346,8 @@ describe("captureViewModel", () => {
|
||||
if (method === "DOMSnapshot.captureSnapshot") return twoHoverTriggerSnapshotReply();
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return {
|
||||
visualViewport: { clientWidth: 1000 },
|
||||
cssVisualViewport: { clientWidth: 1000 },
|
||||
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
|
||||
};
|
||||
}
|
||||
@@ -378,7 +376,7 @@ describe("captureViewModel", () => {
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
|
||||
const captured = await captureViewModel(cdp, 4);
|
||||
const captured = semanticCapture(await captureObservationFacts(cdp, 4)).captured;
|
||||
const surfaceProbes = await probeHoverSurfaces(cdp, 4, captured.nodes);
|
||||
|
||||
expect(surfaceProbes).toEqual([
|
||||
@@ -398,6 +396,8 @@ describe("captureViewModel", () => {
|
||||
if (method === "DOMSnapshot.captureSnapshot") return nestedHoverTriggerSnapshotReply();
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return {
|
||||
visualViewport: { clientWidth: 1000 },
|
||||
cssVisualViewport: { clientWidth: 1000 },
|
||||
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
|
||||
};
|
||||
}
|
||||
@@ -423,7 +423,7 @@ describe("captureViewModel", () => {
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
|
||||
const captured = await captureViewModel(cdp, 4);
|
||||
const captured = semanticCapture(await captureObservationFacts(cdp, 4)).captured;
|
||||
await probeHoverSurfaces(cdp, 4, captured.nodes);
|
||||
|
||||
expect(hoverMoves).toBe(1);
|
||||
@@ -455,6 +455,7 @@ describe("captureViewModel", () => {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
frameId: "root",
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1, 0, 3, 4],
|
||||
nodeName: [i("html"), i("body"), i("div"), i(OVERLAY_HOST_NAME), i("div"), i("button")],
|
||||
@@ -485,7 +486,7 @@ describe("captureViewModel", () => {
|
||||
],
|
||||
};
|
||||
|
||||
const { nodes } = await captureViewModel(makeCdp(snapshot), 4);
|
||||
const { nodes } = semanticCapture(await captureObservationFacts(makeCdp(snapshot), 4)).captured;
|
||||
const ids = nodes.map((n) => n.backendNodeId);
|
||||
// Real page nodes survive.
|
||||
expect(ids).toContain(12);
|
||||
@@ -516,6 +517,7 @@ describe("captureViewModel", () => {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
frameId: "root",
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1, 0, 3, 4],
|
||||
nodeName: [i("html"), i("body"), i("div"), i(OVERLAY_HOST_NAME), i("div"), i("button")],
|
||||
@@ -546,7 +548,9 @@ describe("captureViewModel", () => {
|
||||
],
|
||||
};
|
||||
|
||||
const { excludedBackendNodeIds } = await captureViewModel(makeCdp(snapshot), 4);
|
||||
const { excludedBackendNodeIds } = semanticCapture(
|
||||
await captureObservationFacts(makeCdp(snapshot), 4),
|
||||
).captured;
|
||||
expect(excludedBackendNodeIds).toEqual(new Set([13, 14, 15]));
|
||||
});
|
||||
|
||||
@@ -567,6 +571,7 @@ describe("captureViewModel", () => {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
frameId: "root",
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1],
|
||||
nodeName: [i("html"), i("body"), i(OVERLAY_HOST_NAME)],
|
||||
@@ -589,7 +594,9 @@ describe("captureViewModel", () => {
|
||||
],
|
||||
};
|
||||
|
||||
const { nodes, excludedBackendNodeIds } = await captureViewModel(makeCdp(snapshot), 4);
|
||||
const { nodes, excludedBackendNodeIds } = semanticCapture(
|
||||
await captureObservationFacts(makeCdp(snapshot), 4),
|
||||
).captured;
|
||||
expect(nodes.map((n) => n.backendNodeId)).not.toContain(13);
|
||||
expect(excludedBackendNodeIds).toEqual(new Set([13]));
|
||||
});
|
||||
@@ -612,6 +619,7 @@ describe("captureViewModel", () => {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
frameId: "root",
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1],
|
||||
nodeName: [i("html"), i("body"), i("div")],
|
||||
@@ -634,7 +642,9 @@ describe("captureViewModel", () => {
|
||||
],
|
||||
};
|
||||
|
||||
const { nodes, excludedBackendNodeIds } = await captureViewModel(makeCdp(snapshot), 4);
|
||||
const { nodes, excludedBackendNodeIds } = semanticCapture(
|
||||
await captureObservationFacts(makeCdp(snapshot), 4),
|
||||
).captured;
|
||||
expect(nodes.map((n) => n.backendNodeId)).not.toContain(13);
|
||||
expect(excludedBackendNodeIds).toEqual(new Set([13]));
|
||||
});
|
||||
@@ -657,6 +667,7 @@ describe("captureViewModel", () => {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
frameId: "root",
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1],
|
||||
nodeName: [i("html"), i("body"), i("div")],
|
||||
@@ -679,7 +690,7 @@ describe("captureViewModel", () => {
|
||||
},
|
||||
],
|
||||
};
|
||||
const { nodes } = await captureViewModel(makeCdp(snapshot), 4);
|
||||
const { nodes } = semanticCapture(await captureObservationFacts(makeCdp(snapshot), 4)).captured;
|
||||
expect(nodes.find((n) => n.backendNodeId === 12)?.cursor).toBe("pointer");
|
||||
expect(nodes.find((n) => n.backendNodeId === 11)?.cursor).toBe("auto");
|
||||
});
|
||||
@@ -691,6 +702,7 @@ describe("captureViewModel", () => {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
frameId: "root",
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1, 1, 1],
|
||||
nodeName: [i("html"), i("body"), i("button"), i("button"), i("button")],
|
||||
@@ -717,7 +729,7 @@ describe("captureViewModel", () => {
|
||||
],
|
||||
};
|
||||
|
||||
const { nodes } = await captureViewModel(makeCdp(snapshot), 4);
|
||||
const { nodes } = semanticCapture(await captureObservationFacts(makeCdp(snapshot), 4)).captured;
|
||||
|
||||
expect(nodes.find((node) => node.backendNodeId === 12)?.rendered).toBe(true);
|
||||
expect(nodes.find((node) => node.backendNodeId === 13)?.rendered).toBe(false);
|
||||
@@ -731,25 +743,28 @@ describe("captureViewModel", () => {
|
||||
if (method === "DOMSnapshot.captureSnapshot") return fakeSnapshotReply();
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return {
|
||||
visualViewport: { clientWidth: 1000 },
|
||||
cssVisualViewport: { clientWidth: 1000 },
|
||||
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 200 },
|
||||
};
|
||||
}
|
||||
throw new Error(method);
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
const { nodes } = await captureViewModel(cdp, 4);
|
||||
const { nodes } = semanticCapture(await captureObservationFacts(cdp, 4)).captured;
|
||||
const div = nodes.find((n) => n.backendNodeId === 12);
|
||||
expect(div?.localRect?.y).toBe(-200);
|
||||
expect(div?.rect).toMatchObject({ y: 0, h: 600 });
|
||||
});
|
||||
|
||||
it("normalizes device-pixel bounds by devicePixelRatio", async () => {
|
||||
it("normalizes raw snapshot bounds before viewport clipping", async () => {
|
||||
const S = ["html", "body", "div", "position", "fixed", "static", "pointer-events", "auto"];
|
||||
const i = (s: string) => S.indexOf(s);
|
||||
const snapshot = {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
frameId: "root",
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1],
|
||||
nodeName: [i("html"), i("body"), i("div")],
|
||||
@@ -777,6 +792,8 @@ describe("captureViewModel", () => {
|
||||
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return {
|
||||
visualViewport: { clientWidth: 2000 },
|
||||
cssVisualViewport: { clientWidth: 1000 },
|
||||
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
|
||||
layoutViewport: { clientWidth: 2000, clientHeight: 1600 },
|
||||
};
|
||||
@@ -784,13 +801,19 @@ describe("captureViewModel", () => {
|
||||
throw new Error(method);
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
const { nodes } = await captureViewModel(cdp, 4);
|
||||
const { nodes } = semanticCapture(await captureObservationFacts(cdp, 4)).captured;
|
||||
expect(nodes.find((n) => n.backendNodeId === 12)?.rect).toEqual({
|
||||
x: 0,
|
||||
y: 0,
|
||||
w: 1000,
|
||||
h: 800,
|
||||
});
|
||||
expect(nodes.find((n) => n.backendNodeId === 12)?.localRect).toEqual({
|
||||
x: 0,
|
||||
y: 0,
|
||||
w: 1000,
|
||||
h: 800,
|
||||
});
|
||||
});
|
||||
|
||||
it("extracts textContent from #text child nodes via nodeValue", async () => {
|
||||
@@ -814,6 +837,7 @@ describe("captureViewModel", () => {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
frameId: "root",
|
||||
scrollOffsetX: 0,
|
||||
scrollOffsetY: 0,
|
||||
nodes: {
|
||||
@@ -846,13 +870,15 @@ describe("captureViewModel", () => {
|
||||
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return {
|
||||
visualViewport: { clientWidth: 1000 },
|
||||
cssVisualViewport: { clientWidth: 1000 },
|
||||
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
|
||||
};
|
||||
}
|
||||
throw new Error(method);
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
const { nodes } = await captureViewModel(cdp, 4);
|
||||
const { nodes } = semanticCapture(await captureObservationFacts(cdp, 4)).captured;
|
||||
const btn = nodes.find((n) => n.backendNodeId === 12);
|
||||
expect(btn?.textContent).toBe("登录");
|
||||
const link = nodes.find((n) => n.backendNodeId === 14);
|
||||
@@ -862,7 +888,7 @@ describe("captureViewModel", () => {
|
||||
expect(body?.textContent).toBeUndefined();
|
||||
});
|
||||
|
||||
it("parses iframe sub-documents and returns iframeNodes keyed by iframe backendNodeId", async () => {
|
||||
it("retains explicit iframe document ownership and projected nodes", async () => {
|
||||
const S = [
|
||||
"html",
|
||||
"body",
|
||||
@@ -936,39 +962,38 @@ describe("captureViewModel", () => {
|
||||
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return {
|
||||
visualViewport: { clientWidth: 1000 },
|
||||
cssVisualViewport: { clientWidth: 1000 },
|
||||
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
|
||||
};
|
||||
}
|
||||
if (method === "DOM.getBoxModel")
|
||||
return { model: { content: [100, 300, 900, 300, 900, 700, 100, 700] } };
|
||||
if (method === "DOM.resolveNode") return { object: { objectId: "owner" } };
|
||||
if (method === "Runtime.callFunctionOn")
|
||||
return { result: { value: { width: 800, height: 400 } } };
|
||||
if (method === "Runtime.releaseObject") return {};
|
||||
throw new Error(method);
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
const {
|
||||
nodes,
|
||||
iframeNodes,
|
||||
frameNodes,
|
||||
frameOwnerBackendNodeIds,
|
||||
frameParentIds,
|
||||
rootFrameId,
|
||||
} = await captureViewModel(cdp, 4);
|
||||
// Main frame has 4 nodes; iframe is included
|
||||
expect(nodes.find((n) => n.backendNodeId === 13)?.tag).toBe("iframe");
|
||||
// iframeNodes keyed by the <iframe> element's backendNodeId=13
|
||||
expect(iframeNodes.size).toBe(1);
|
||||
expect(iframeNodes.has(13)).toBe(true);
|
||||
const subNodes = iframeNodes.get(13)!;
|
||||
const facts = await captureObservationFacts(cdp, 4);
|
||||
const root = facts.documents.find((doc) => doc.frame.frameId === "main-frame")!;
|
||||
const child = facts.documents.find((doc) => doc.frame.frameId === "child-frame")!;
|
||||
expect(facts.documents).toHaveLength(2);
|
||||
expect(root.domNodes.find((n) => n.backendNodeId === 13)?.tag).toBe("iframe");
|
||||
const subNodes = child.domNodes;
|
||||
const input = subNodes.find((n) => n.backendNodeId === 101);
|
||||
expect(input?.tag).toBe("input");
|
||||
expect(input?.attrs.type).toBe("text");
|
||||
expect(input?.ownerFrameBackendNodeId).toBe(13);
|
||||
expect(input?.localRect).toEqual({ x: 0, y: 0, w: 200, h: 40 });
|
||||
expect(input?.rect).toEqual({ x: 100, y: 300, w: 200, h: 40 });
|
||||
expect(rootFrameId).toBe("main-frame");
|
||||
expect(frameNodes?.get("child-frame")).toBe(subNodes);
|
||||
expect(frameOwnerBackendNodeIds?.get("child-frame")).toBe(13);
|
||||
expect(frameParentIds?.get("child-frame")).toBe("main-frame");
|
||||
expect(facts.rootFrameId).toBe("main-frame");
|
||||
expect(child.frame.ownerBackendNodeId).toBe(13);
|
||||
expect(child.frame.parentFrameId).toBe("main-frame");
|
||||
});
|
||||
|
||||
it("captures iframe documents without requiring owner-frame geometry", async () => {
|
||||
it("projects iframe documents independently of owner snapshot geometry", async () => {
|
||||
const S = [
|
||||
"html",
|
||||
"body",
|
||||
@@ -986,6 +1011,7 @@ describe("captureViewModel", () => {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
frameId: "root",
|
||||
scrollOffsetX: 0,
|
||||
scrollOffsetY: 0,
|
||||
nodes: {
|
||||
@@ -1003,6 +1029,7 @@ describe("captureViewModel", () => {
|
||||
},
|
||||
},
|
||||
{
|
||||
frameId: "child",
|
||||
scrollOffsetX: 0,
|
||||
scrollOffsetY: 0,
|
||||
nodes: {
|
||||
@@ -1026,18 +1053,28 @@ describe("captureViewModel", () => {
|
||||
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return {
|
||||
visualViewport: { clientWidth: 1000 },
|
||||
cssVisualViewport: { clientWidth: 1000 },
|
||||
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
|
||||
};
|
||||
}
|
||||
if (method === "DOM.getBoxModel")
|
||||
return { model: { content: [100, 300, 900, 300, 900, 700, 100, 700] } };
|
||||
if (method === "DOM.resolveNode") return { object: { objectId: "owner" } };
|
||||
if (method === "Runtime.callFunctionOn")
|
||||
return { result: { value: { width: 800, height: 400 } } };
|
||||
if (method === "Runtime.releaseObject") return {};
|
||||
throw new Error(method);
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
|
||||
const { iframeNodes } = await captureViewModel(cdp, 4);
|
||||
const input = iframeNodes.get(13)?.find((n) => n.backendNodeId === 101);
|
||||
const { documents } = await captureObservationFacts(cdp, 4);
|
||||
const input = documents
|
||||
.find((doc) => doc.frame.frameId === "child")
|
||||
?.domNodes?.find((n) => n.backendNodeId === 101);
|
||||
|
||||
expect(input?.localRect).toEqual({ x: 20, y: 20, w: 120, h: 60 });
|
||||
expect(input?.rect).toBeNull();
|
||||
expect(input?.rect).toEqual({ x: 120, y: 320, w: 120, h: 60 });
|
||||
});
|
||||
|
||||
it("recursively normalizes nested iframe sub-documents", async () => {
|
||||
@@ -1058,6 +1095,7 @@ describe("captureViewModel", () => {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
frameId: "root",
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1],
|
||||
nodeName: [i("html"), i("body"), i("iframe")],
|
||||
@@ -1079,6 +1117,9 @@ describe("captureViewModel", () => {
|
||||
},
|
||||
},
|
||||
{
|
||||
scrollOffsetX: 0,
|
||||
scrollOffsetY: 0,
|
||||
frameId: "child",
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1],
|
||||
nodeName: [i("html"), i("body"), i("iframe")],
|
||||
@@ -1100,6 +1141,9 @@ describe("captureViewModel", () => {
|
||||
},
|
||||
},
|
||||
{
|
||||
scrollOffsetX: 0,
|
||||
scrollOffsetY: 0,
|
||||
frameId: "nested",
|
||||
nodes: {
|
||||
parentIndex: [-1, 0],
|
||||
nodeName: [i("body"), i("input")],
|
||||
@@ -1116,21 +1160,54 @@ describe("captureViewModel", () => {
|
||||
],
|
||||
};
|
||||
const cdp = {
|
||||
send: vi.fn(async (_t: number, method: string) => {
|
||||
if (method === "DOMSnapshot.enable") return {};
|
||||
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return {
|
||||
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
|
||||
};
|
||||
}
|
||||
throw new Error(method);
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
send: vi.fn(
|
||||
async (
|
||||
_t: number,
|
||||
method: string,
|
||||
params?: { backendNodeId?: number; objectId?: string },
|
||||
) => {
|
||||
if (method === "DOMSnapshot.enable") return {};
|
||||
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return {
|
||||
visualViewport: { clientWidth: 1000 },
|
||||
cssVisualViewport: { clientWidth: 1000 },
|
||||
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
|
||||
};
|
||||
}
|
||||
if (method === "DOM.getBoxModel")
|
||||
return {
|
||||
model: {
|
||||
content:
|
||||
params?.backendNodeId === 13
|
||||
? [10, 20, 310, 20, 310, 220, 10, 220]
|
||||
: [15, 26, 115, 26, 115, 106, 15, 106],
|
||||
},
|
||||
};
|
||||
if (method === "DOM.resolveNode")
|
||||
return { object: { objectId: String(params?.backendNodeId) } };
|
||||
if (method === "Runtime.callFunctionOn")
|
||||
return {
|
||||
result: {
|
||||
value:
|
||||
params?.objectId === "13"
|
||||
? { width: 300, height: 200 }
|
||||
: { width: 100, height: 80 },
|
||||
},
|
||||
};
|
||||
if (method === "Runtime.releaseObject") return {};
|
||||
throw new Error(method);
|
||||
},
|
||||
) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
|
||||
const { iframeNodes } = await captureViewModel(cdp, 4);
|
||||
const nestedIframe = iframeNodes.get(13)?.find((n) => n.backendNodeId === 23);
|
||||
const input = iframeNodes.get(23)?.find((n) => n.backendNodeId === 31);
|
||||
const { documents } = await captureObservationFacts(cdp, 4);
|
||||
const nestedIframe = documents
|
||||
.find((doc) => doc.frame.frameId === "child")
|
||||
?.domNodes?.find((n) => n.backendNodeId === 23);
|
||||
const input = documents
|
||||
.find((doc) => doc.frame.frameId === "nested")
|
||||
?.domNodes?.find((n) => n.backendNodeId === 31);
|
||||
|
||||
expect(nestedIframe?.localRect).toEqual({ x: 5, y: 6, w: 100, h: 80 });
|
||||
expect(nestedIframe?.rect).toEqual({ x: 15, y: 26, w: 100, h: 80 });
|
||||
@@ -1139,13 +1216,14 @@ describe("captureViewModel", () => {
|
||||
expect(input?.rect).toEqual({ x: 16, y: 28, w: 40, h: 20 });
|
||||
});
|
||||
|
||||
it("normalizes bounds then subtracts CSS scroll at dpr>1", async () => {
|
||||
it("subtracts CSS scroll once without dividing snapshot bounds by a metrics ratio", async () => {
|
||||
const S = ["html", "body", "div", "position", "fixed", "static", "pointer-events", "auto"];
|
||||
const i = (s: string) => S.indexOf(s);
|
||||
const snapshot = {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
frameId: "root",
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1],
|
||||
nodeName: [i("html"), i("body"), i("div")],
|
||||
@@ -1173,6 +1251,8 @@ describe("captureViewModel", () => {
|
||||
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return {
|
||||
visualViewport: { clientWidth: 2000 },
|
||||
cssVisualViewport: { clientWidth: 1000 },
|
||||
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 100 },
|
||||
layoutViewport: { clientWidth: 2000, clientHeight: 1600 },
|
||||
};
|
||||
@@ -1180,7 +1260,7 @@ describe("captureViewModel", () => {
|
||||
throw new Error(method);
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
const { nodes } = await captureViewModel(cdp, 4);
|
||||
const { nodes } = semanticCapture(await captureObservationFacts(cdp, 4)).captured;
|
||||
expect(nodes.find((n) => n.backendNodeId === 12)?.rect?.y).toBe(400 / 2 - 100);
|
||||
});
|
||||
|
||||
|
||||
@@ -0,0 +1,127 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { OVERLAY_HOST_MARKER_ATTR } from "@/lib/overlay-bridge";
|
||||
import { createCaptureCheckpoint } from "../capture-abort";
|
||||
import { buildDocumentIndex, type DecodedNode } from "../facts";
|
||||
import { decodeDocument, REQUESTED_STYLES } from "../snapshot";
|
||||
|
||||
function node(id: number, parent: number | null): DecodedNode {
|
||||
return {
|
||||
backendNodeId: id,
|
||||
parentBackendNodeId: parent,
|
||||
tag: "div",
|
||||
attrs: {},
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
};
|
||||
}
|
||||
|
||||
describe("document facts", () => {
|
||||
it.each([
|
||||
1_000, 10_000, 100_000,
|
||||
])("indexes deep and wide inputs of %i without copying ancestors", async (count) => {
|
||||
for (const wide of [false, true]) {
|
||||
let reads = 0;
|
||||
const input = Array.from({ length: count }, (_, i) => {
|
||||
const item = node(i, i ? (wide ? 0 : i - 1) : null);
|
||||
if (i === 0) item.attrs[OVERLAY_HOST_MARKER_ATTR] = "";
|
||||
Object.defineProperty(item, "parentBackendNodeId", {
|
||||
get: () => {
|
||||
reads++;
|
||||
return i ? (wide ? 0 : i - 1) : null;
|
||||
},
|
||||
});
|
||||
return item;
|
||||
}).reverse(); // parent need not precede child
|
||||
const index = await buildDocumentIndex(input);
|
||||
expect(index.nodes.size).toBe(count);
|
||||
expect(index.excludedBackendNodeIds.size).toBe(count);
|
||||
expect(reads).toBeLessThanOrEqual(count * 8);
|
||||
expect(index.nodes.get(count - 1)).toBe(input[0]);
|
||||
}
|
||||
});
|
||||
|
||||
it("handles cycles and orphans and propagates overlay through shadow roots", async () => {
|
||||
const host = { ...node(1, null), attrs: { [OVERLAY_HOST_MARKER_ATTR]: "" } };
|
||||
const input = [
|
||||
node(3, 2),
|
||||
{ ...node(2, 1), tag: "#document-fragment" },
|
||||
host,
|
||||
node(4, 99),
|
||||
node(5, 6),
|
||||
node(6, 5),
|
||||
node(7, 6),
|
||||
];
|
||||
const index = await buildDocumentIndex(input);
|
||||
expect([...index.excludedBackendNodeIds].sort()).toEqual([1, 2, 3]);
|
||||
expect(index.nodes.size).toBe(input.length);
|
||||
});
|
||||
|
||||
it("does not propagate a descendant overlay backwards into a cycle", async () => {
|
||||
const overlay = { ...node(3, 1), attrs: { [OVERLAY_HOST_MARKER_ATTR]: "" } };
|
||||
const index = await buildDocumentIndex([overlay, node(1, 2), node(2, 1)]);
|
||||
expect([...index.excludedBackendNodeIds]).toEqual([3]);
|
||||
});
|
||||
|
||||
it("decodes current observation bounds, styles and attributes without projecting coordinates", async () => {
|
||||
const strings = ["div", "aria-hidden", "true", "inert", "", "visible", "0", "static", "auto"];
|
||||
const styles = REQUESTED_STYLES.map((style) =>
|
||||
style === "visibility" ? 5 : style === "opacity" ? 6 : style === "position" ? 7 : 8,
|
||||
);
|
||||
const { nodes } = await decodeDocument(
|
||||
{
|
||||
nodes: { backendNodeId: [1], parentIndex: [-1], nodeName: [0], attributes: [[1, 2, 3, 4]] },
|
||||
layout: { nodeIndex: [0], bounds: [[10, 20, 120, 40]], styles: [styles] },
|
||||
},
|
||||
strings,
|
||||
);
|
||||
expect(nodes[0].attrs).toEqual({ "aria-hidden": "true", inert: "" });
|
||||
expect(nodes[0].layout).toEqual({
|
||||
boundsSpace: "snapshot-document-layout",
|
||||
bounds: [10, 20, 120, 40],
|
||||
styles: {
|
||||
position: "static",
|
||||
"pointer-events": "auto",
|
||||
cursor: "auto",
|
||||
visibility: "visible",
|
||||
opacity: "0",
|
||||
},
|
||||
});
|
||||
expect(nodes[0]).not.toHaveProperty("rect");
|
||||
expect(nodes[0]).not.toHaveProperty("excluded");
|
||||
});
|
||||
|
||||
it("keeps short blocks synchronous and checks cancellation after yielding", async () => {
|
||||
const clock = vi.spyOn(performance, "now").mockReturnValue(0);
|
||||
try {
|
||||
const controller = new AbortController();
|
||||
const checkpoint = createCaptureCheckpoint(controller.signal);
|
||||
for (let i = 0; i < 20; i++) expect(checkpoint()).toBeUndefined();
|
||||
clock.mockReturnValue(9);
|
||||
const pending = checkpoint();
|
||||
expect(pending).toBeInstanceOf(Promise);
|
||||
controller.abort();
|
||||
await expect(pending).rejects.toMatchObject({ name: "AbortError" });
|
||||
} finally {
|
||||
clock.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it("checks cancellation at bounded indexing blocks", async () => {
|
||||
const controller = new AbortController();
|
||||
const input = Array.from({ length: 100_000 }, (_, i) => node(i, i ? i - 1 : null));
|
||||
let reads = 0;
|
||||
for (const item of input)
|
||||
Object.defineProperty(item, "attrs", {
|
||||
get: () => {
|
||||
reads++;
|
||||
if (reads === 100) controller.abort();
|
||||
return {};
|
||||
},
|
||||
});
|
||||
await expect(buildDocumentIndex(input, controller.signal)).rejects.toMatchObject({
|
||||
name: "AbortError",
|
||||
});
|
||||
expect(reads).toBeLessThanOrEqual(356);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,298 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { type CdpRunner, cdpRunnerForTarget } from "../../shared";
|
||||
import { enrichFormControlStates } from "../form-capture";
|
||||
import { decodeDocument } from "../snapshot";
|
||||
|
||||
interface Control {
|
||||
id: number;
|
||||
value: string;
|
||||
type?: string;
|
||||
}
|
||||
|
||||
function snapshot(groups: Control[][]) {
|
||||
const strings: string[] = [];
|
||||
const str = (value: string) => {
|
||||
let index = strings.indexOf(value);
|
||||
if (index < 0) index = strings.push(value) - 1;
|
||||
return index;
|
||||
};
|
||||
const documents = groups.map((controls, groupIndex) => ({
|
||||
frameId: `frame-${groupIndex}`,
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, ...controls.map(() => 1)],
|
||||
nodeName: [str("html"), str("body"), ...controls.map(() => str("input"))],
|
||||
backendNodeId: [8000 + groupIndex * 2, 8001 + groupIndex * 2, ...controls.map((c) => c.id)],
|
||||
attributes: [
|
||||
[],
|
||||
[],
|
||||
...controls.map((c) => [
|
||||
str("type"),
|
||||
str(c.type ?? "text"),
|
||||
str("placeholder"),
|
||||
str("snapshot placeholder"),
|
||||
]),
|
||||
],
|
||||
inputValue: {
|
||||
index: controls.map((_, i) => i + 2),
|
||||
value: controls.map((c) => str(c.value)),
|
||||
},
|
||||
contentDocumentIndex: { index: [] as number[], value: [] as number[] },
|
||||
},
|
||||
}));
|
||||
const root = documents[0].nodes;
|
||||
for (let i = 1; i < documents.length; i++) {
|
||||
root.contentDocumentIndex.index.push(root.backendNodeId.length);
|
||||
root.contentDocumentIndex.value.push(i);
|
||||
root.parentIndex.push(1);
|
||||
root.nodeName.push(str("iframe"));
|
||||
root.backendNodeId.push(9000 + i);
|
||||
root.attributes.push([]);
|
||||
}
|
||||
return { strings, documents };
|
||||
}
|
||||
|
||||
function state(value: string) {
|
||||
return {
|
||||
value,
|
||||
defaultValue: "",
|
||||
placeholder: "live placeholder",
|
||||
state: value ? "filled" : "empty",
|
||||
sensitive: false,
|
||||
};
|
||||
}
|
||||
|
||||
function reply(entries: Array<[number, unknown]>) {
|
||||
return {
|
||||
result: {
|
||||
deepSerializedValue: {
|
||||
type: "array",
|
||||
value: entries.map(([backendNodeId, fields]) => ({
|
||||
type: "array",
|
||||
value: [
|
||||
{ type: "node", value: { backendNodeId } },
|
||||
{ type: "string", value: JSON.stringify(fields) },
|
||||
],
|
||||
})),
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function fakeCdp(
|
||||
groups: Control[][],
|
||||
evaluate: (params: Record<string, unknown>) => unknown,
|
||||
) {
|
||||
const snap = snapshot(groups);
|
||||
const send = vi.fn(async (_tabId: number, method: string, params?: object) => {
|
||||
if (method === "Runtime.evaluate") return evaluate(params as Record<string, unknown>);
|
||||
return {};
|
||||
});
|
||||
const nodes = await decodeControls(snap);
|
||||
return { send, cdp: { send: send as CdpRunner["send"] }, nodes };
|
||||
}
|
||||
|
||||
async function decodeControls(snap: ReturnType<typeof snapshot>) {
|
||||
return Promise.all(
|
||||
snap.documents.map(async (doc) =>
|
||||
(await decodeDocument(doc, snap.strings)).nodes.map((node) => ({ ...node, rect: null })),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
describe("form state identity", () => {
|
||||
it("matches reordered controls by identity and preserves controls missing from the batch", async () => {
|
||||
// The snapshot includes a shadow input. The runtime query skips it,
|
||||
// encounters a new input, and sees the remaining inputs in a new order.
|
||||
const { cdp, nodes } = await fakeCdp(
|
||||
[
|
||||
[
|
||||
{ id: 10, value: "" },
|
||||
{ id: 11, value: "old email" },
|
||||
{ id: 12, value: "old city" },
|
||||
],
|
||||
],
|
||||
() =>
|
||||
reply([
|
||||
[99, state("new control")],
|
||||
[12, state("上海")],
|
||||
[11, state("alice@example.com")],
|
||||
]),
|
||||
);
|
||||
await enrichFormControlStates(cdp, 7, nodes);
|
||||
expect(nodes[0].find((n) => n.backendNodeId === 10)).toMatchObject({
|
||||
formValue: "",
|
||||
formState: "empty",
|
||||
formPlaceholder: "snapshot placeholder",
|
||||
});
|
||||
expect(nodes[0].find((n) => n.backendNodeId === 11)).toMatchObject({
|
||||
formValue: "alice@example.com",
|
||||
});
|
||||
expect(nodes[0].find((n) => n.backendNodeId === 12)).toMatchObject({ formValue: "上海" });
|
||||
expect(nodes[0].some((n) => n.backendNodeId === 99)).toBe(false);
|
||||
});
|
||||
|
||||
it("does not shift state between frames when one frame is absent from the runtime batch", async () => {
|
||||
const { cdp, nodes } = await fakeCdp(
|
||||
[
|
||||
[],
|
||||
[{ id: 11, value: "inaccessible frame" }],
|
||||
[{ id: 21, value: "old second" }],
|
||||
[{ id: 31, value: "old third" }],
|
||||
],
|
||||
() =>
|
||||
reply([
|
||||
[31, state("third frame")],
|
||||
[21, state("second frame")],
|
||||
]),
|
||||
);
|
||||
await enrichFormControlStates(cdp, 7, nodes);
|
||||
const controls = nodes.slice(1).flat();
|
||||
expect(controls.find((n) => n.backendNodeId === 11)?.formValue).toBe("inaccessible frame");
|
||||
expect(controls.find((n) => n.backendNodeId === 21)?.formValue).toBe("second frame");
|
||||
expect(controls.find((n) => n.backendNodeId === 31)?.formValue).toBe("third frame");
|
||||
});
|
||||
|
||||
it("keeps identical backend ids in different OOPIF targets separate", async () => {
|
||||
const sendToTarget = vi.fn(async (target, method) => {
|
||||
if (method === "Runtime.evaluate") return reply([[11, state(target.sessionId)]]);
|
||||
return {};
|
||||
});
|
||||
const send = vi.fn(async () => {
|
||||
throw new Error("unexpected top-level target");
|
||||
});
|
||||
const cdp = { send, sendToTarget } as unknown as CdpRunner;
|
||||
for (const sessionId of ["left-frame", "right-frame"]) {
|
||||
const nodes = await decodeControls(snapshot([[{ id: 11, value: "old" }]]));
|
||||
await enrichFormControlStates(cdpRunnerForTarget(cdp, { tabId: 7, sessionId }), 7, nodes);
|
||||
expect(nodes[0].find((n) => n.backendNodeId === 11)?.formValue).toBe(sessionId);
|
||||
expect(sendToTarget).toHaveBeenCalledWith(
|
||||
{ tabId: 7, sessionId },
|
||||
"Runtime.releaseObjectGroup",
|
||||
expect.anything(),
|
||||
);
|
||||
}
|
||||
expect(send).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each([
|
||||
"unsupported",
|
||||
"legacy",
|
||||
"malformed",
|
||||
"invalid-state",
|
||||
])("preserves snapshot data when enrichment is %s", async (mode) => {
|
||||
const { cdp, send, nodes } = await fakeCdp([[{ id: 11, value: "snapshot value" }]], () => {
|
||||
if (mode === "unsupported") throw new Error("serialization unsupported");
|
||||
if (mode === "legacy") return { result: { value: { controls: [state("wrong value")] } } };
|
||||
const result = reply([
|
||||
[11, mode === "invalid-state" ? { value: "wrong value" } : state("wrong value")],
|
||||
]);
|
||||
if (mode === "malformed")
|
||||
result.result.deepSerializedValue.value[0].value[1].value = "not json";
|
||||
return result;
|
||||
});
|
||||
expect(await enrichFormControlStates(cdp, 7, nodes)).toBe(mode !== "unsupported");
|
||||
expect(nodes[0].find((n) => n.backendNodeId === 11)).toMatchObject({
|
||||
formValue: "snapshot value",
|
||||
formPlaceholder: "snapshot placeholder",
|
||||
});
|
||||
expect(send).toHaveBeenCalledWith(7, "Runtime.releaseObjectGroup", expect.anything());
|
||||
});
|
||||
|
||||
it("releases the batch's remote objects when capture is cancelled", async () => {
|
||||
const controller = new AbortController();
|
||||
const { cdp, send, nodes } = await fakeCdp([[{ id: 11, value: "old" }]], () => {
|
||||
controller.abort();
|
||||
return reply([[11, state("new")]]);
|
||||
});
|
||||
await expect(enrichFormControlStates(cdp, 7, nodes, controller.signal)).rejects.toMatchObject({
|
||||
name: "AbortError",
|
||||
});
|
||||
expect(send).toHaveBeenCalledWith(7, "Runtime.releaseObjectGroup", expect.anything());
|
||||
});
|
||||
|
||||
it("does not report success when cancelled during object cleanup", async () => {
|
||||
const controller = new AbortController();
|
||||
const nodes = await decodeControls(snapshot([[{ id: 11, value: "old" }]]));
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async (_tabId, method) => {
|
||||
if (method === "Runtime.evaluate") return reply([[11, state("new")]]);
|
||||
if (method === "Runtime.releaseObjectGroup") controller.abort();
|
||||
return {};
|
||||
}) as CdpRunner["send"],
|
||||
};
|
||||
await expect(enrichFormControlStates(cdp, 7, nodes, controller.signal)).rejects.toMatchObject({
|
||||
name: "AbortError",
|
||||
});
|
||||
expect(cdp.send).toHaveBeenCalledWith(7, "Runtime.releaseObjectGroup", expect.anything());
|
||||
});
|
||||
|
||||
it("skips entries without node identity while still applying valid entries", async () => {
|
||||
const { cdp, nodes } = await fakeCdp(
|
||||
[
|
||||
[
|
||||
{ id: 11, value: "snapshot value" },
|
||||
{ id: 12, value: "old value" },
|
||||
],
|
||||
],
|
||||
() => {
|
||||
const result = reply([
|
||||
[11, state("wrong value")],
|
||||
[12, state("live value")],
|
||||
]);
|
||||
const element = result.result.deepSerializedValue.value[0].value[0];
|
||||
delete (element.value as { backendNodeId?: number }).backendNodeId;
|
||||
return result;
|
||||
},
|
||||
);
|
||||
await enrichFormControlStates(cdp, 7, nodes);
|
||||
expect(nodes[0].find((n) => n.backendNodeId === 11)?.formValue).toBe("snapshot value");
|
||||
expect(nodes[0].find((n) => n.backendNodeId === 12)?.formValue).toBe("live value");
|
||||
});
|
||||
|
||||
it.each([
|
||||
"password",
|
||||
"text",
|
||||
])("redacts a sensitive %s control after identity matching", async (type) => {
|
||||
const { cdp, nodes } = await fakeCdp([[{ id: 11, value: "secret", type }]], () =>
|
||||
reply([[11, { ...state("secret"), sensitive: true }]]),
|
||||
);
|
||||
await enrichFormControlStates(cdp, 7, nodes);
|
||||
const control = nodes[0].find((n) => n.backendNodeId === 11);
|
||||
expect(control?.formState).toBe("filled");
|
||||
expect(control?.formValue).toBeUndefined();
|
||||
expect(control?.formDefaultValue).toBeUndefined();
|
||||
expect(control?.attrs.value).toBeUndefined();
|
||||
});
|
||||
|
||||
it("keeps one bounded batch without per-control CDP queries", async () => {
|
||||
const controls = Array.from({ length: 251 }, (_, i) => ({ id: i + 1, value: "old" }));
|
||||
const elements = controls.map((c) => ({
|
||||
backendNodeId: c.id,
|
||||
tagName: "INPUT",
|
||||
type: "text",
|
||||
value: `live ${c.id}`,
|
||||
defaultValue: "",
|
||||
placeholder: "",
|
||||
}));
|
||||
const { cdp, send, nodes } = await fakeCdp([controls], (params) => {
|
||||
const doc = {
|
||||
querySelectorAll: (selector: string) =>
|
||||
selector === "input,textarea,select" ? elements : [],
|
||||
};
|
||||
const entries = new Function("document", `return ${params.expression}`)(doc) as Array<
|
||||
[{ backendNodeId: number }, string]
|
||||
>;
|
||||
expect(entries).toHaveLength(250);
|
||||
return reply(entries.map(([element, json]) => [element.backendNodeId, JSON.parse(json)]));
|
||||
});
|
||||
await enrichFormControlStates(cdp, 7, nodes);
|
||||
expect(nodes[0].find((n) => n.backendNodeId === 250)?.formValue).toBe("live 250");
|
||||
expect(nodes[0].find((n) => n.backendNodeId === 251)?.formValue).toBe("old");
|
||||
expect(send.mock.calls.filter(([, method]) => method === "Runtime.evaluate")).toHaveLength(1);
|
||||
expect(
|
||||
send.mock.calls.filter(([, method]) => method === "Runtime.releaseObjectGroup"),
|
||||
).toHaveLength(1);
|
||||
expect(send).not.toHaveBeenCalledWith(7, "DOM.resolveNode", expect.anything());
|
||||
expect(send).not.toHaveBeenCalledWith(7, "Runtime.callFunctionOn", expect.anything());
|
||||
});
|
||||
});
|
||||
@@ -1,198 +0,0 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import type { CdpRunner } from "../../shared";
|
||||
import type { CapturedNode, CapturedViewModel } from "../capture";
|
||||
import { captureFrameData } from "../frame-capture";
|
||||
|
||||
function ownerNode(backendNodeId: number, x: number): CapturedNode {
|
||||
return {
|
||||
backendNodeId,
|
||||
parentBackendNodeId: 1,
|
||||
frameId: "main",
|
||||
ownerFrameBackendNodeId: null,
|
||||
tag: "iframe",
|
||||
attrs: {},
|
||||
rect: { x, y: 100, w: 300, h: 200 },
|
||||
localRect: { x, y: 100, w: 300, h: 200 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
};
|
||||
}
|
||||
|
||||
function childSnapshot(frameId: string, backendNodeId: number) {
|
||||
const strings = [frameId, "body", "button", "static", "auto", "pointer"];
|
||||
return {
|
||||
strings,
|
||||
documents: [
|
||||
{
|
||||
frameId,
|
||||
nodes: {
|
||||
parentIndex: [-1, 0],
|
||||
nodeName: [1, 2],
|
||||
backendNodeId: [backendNodeId - 1, backendNodeId],
|
||||
attributes: [[], []],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [0, 1],
|
||||
styles: [
|
||||
[3, 4, 4],
|
||||
[3, 4, 5],
|
||||
],
|
||||
bounds: [
|
||||
[0, 0, 300, 200],
|
||||
[10, 20, 100, 40],
|
||||
],
|
||||
paintOrders: [0, 1],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
describe("captureFrameData", () => {
|
||||
it("captures and positions multiple OOPIF documents missing from the root snapshot", async () => {
|
||||
const leftOwner = ownerNode(10, 50);
|
||||
const rightOwner = ownerNode(20, 500);
|
||||
const captured: CapturedViewModel = {
|
||||
nodes: [leftOwner, rightOwner],
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
frameNodes: new Map([["main", [leftOwner, rightOwner]]]),
|
||||
frameOwnerBackendNodeIds: new Map(),
|
||||
rootFrameId: "main",
|
||||
excludedBackendNodeIds: new Set(),
|
||||
};
|
||||
const sendToTarget = vi.fn(async (target, method) => {
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 300, clientHeight: 200, pageX: 0, pageY: 0 } };
|
||||
}
|
||||
if (method === "DOMSnapshot.enable" || method === "Accessibility.enable") return {};
|
||||
if (method === "DOMSnapshot.captureSnapshot") {
|
||||
return target.sessionId === "left-session"
|
||||
? childSnapshot("left", 101)
|
||||
: childSnapshot("right", 201);
|
||||
}
|
||||
if (method === "Accessibility.getFullAXTree") return { nodes: [] };
|
||||
throw new Error(`unexpected ${method}`);
|
||||
});
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async (_tabId, method, params) => {
|
||||
if (method === "Accessibility.enable") return {};
|
||||
if (method === "Accessibility.getFullAXTree") return { nodes: [] };
|
||||
if (method === "DOM.getBoxModel") {
|
||||
const backendNodeId = (params as { backendNodeId?: number })?.backendNodeId;
|
||||
const x = backendNodeId === 10 ? 50 : 500;
|
||||
return { model: { content: [x, 100, x + 300, 100, x + 300, 300, x, 300] } };
|
||||
}
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } };
|
||||
}
|
||||
throw new Error(`unexpected root ${method}`);
|
||||
}) as CdpRunner["send"],
|
||||
sendToTarget: sendToTarget as unknown as NonNullable<CdpRunner["sendToTarget"]>,
|
||||
getFrameGraph: vi.fn(async () => ({
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "left",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 10,
|
||||
target: { tabId: 4, sessionId: "left-session" },
|
||||
},
|
||||
{
|
||||
frameId: "right",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 20,
|
||||
target: { tabId: 4, sessionId: "right-session" },
|
||||
},
|
||||
],
|
||||
})),
|
||||
};
|
||||
|
||||
const trees = await captureFrameData(cdp, 4, captured);
|
||||
|
||||
expect(trees.map((tree) => tree.frameId)).toEqual(["main", "left", "right"]);
|
||||
expect(
|
||||
captured.frameNodes?.get("left")?.find((node) => node.backendNodeId === 101)?.rect,
|
||||
).toEqual({ x: 60, y: 120, w: 100, h: 40 });
|
||||
expect(
|
||||
captured.frameNodes?.get("right")?.find((node) => node.backendNodeId === 201)?.rect,
|
||||
).toEqual({ x: 510, y: 120, w: 100, h: 40 });
|
||||
expect(captured.frameOwnerBackendNodeIds).toEqual(
|
||||
new Map([
|
||||
["left", 10],
|
||||
["right", 20],
|
||||
]),
|
||||
);
|
||||
expect(captured.frameParentIds).toEqual(
|
||||
new Map([
|
||||
["left", "main"],
|
||||
["right", "main"],
|
||||
]),
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps OOPIF semantics when viewport projection is unavailable", async () => {
|
||||
const owner = { ...ownerNode(10, 50), rect: null, localRect: null };
|
||||
const captured: CapturedViewModel = {
|
||||
nodes: [owner],
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
frameNodes: new Map([["main", [owner]]]),
|
||||
rootFrameId: "main",
|
||||
excludedBackendNodeIds: new Set(),
|
||||
};
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async (_tabId, method) => {
|
||||
if (method === "Accessibility.enable") return {};
|
||||
if (method === "Accessibility.getFullAXTree") return { nodes: [] };
|
||||
if (method === "DOM.getBoxModel") throw new Error("owner geometry unavailable");
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } };
|
||||
}
|
||||
throw new Error(`unexpected root ${method}`);
|
||||
}) as CdpRunner["send"],
|
||||
sendToTarget: vi.fn(async (_target, method) => {
|
||||
if (method === "Page.getLayoutMetrics") throw new Error("viewport unavailable");
|
||||
if (method === "DOMSnapshot.enable" || method === "Accessibility.enable") return {};
|
||||
if (method === "DOMSnapshot.captureSnapshot") return childSnapshot("child", 101);
|
||||
if (method === "Accessibility.getFullAXTree") {
|
||||
return {
|
||||
nodes: [
|
||||
{
|
||||
nodeId: "button",
|
||||
backendDOMNodeId: 101,
|
||||
role: { type: "role", value: "button" },
|
||||
name: { type: "computedString", value: "Continue" },
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
throw new Error(`unexpected ${method}`);
|
||||
}) as unknown as NonNullable<CdpRunner["sendToTarget"]>,
|
||||
getFrameGraph: vi.fn(async () => ({
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "child",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 10,
|
||||
target: { tabId: 4, sessionId: "child-session" },
|
||||
},
|
||||
],
|
||||
})),
|
||||
};
|
||||
|
||||
const documents = await captureFrameData(cdp, 4, captured);
|
||||
const childDocument = documents.find((document) => document.frameId === "child");
|
||||
|
||||
expect(childDocument?.axNodes).toEqual([
|
||||
expect.objectContaining({ backendDOMNodeId: 101, frameId: "child" }),
|
||||
]);
|
||||
expect(childDocument?.domNodes.find((node) => node.backendNodeId === 101)).toEqual(
|
||||
expect.objectContaining({ rect: null, localRect: { x: 10, y: 20, w: 100, h: 40 } }),
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -1,7 +1,12 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import type { CdpFrameGraph } from "@/browser-driver/frame-graph";
|
||||
import type { CapturedNode, CapturedViewModel } from "../capture";
|
||||
import { buildFrameDocuments, type FrameAxBatch, type FrameOwnedAxNode } from "../frame-document";
|
||||
import type { CapturedNode } from "../capture";
|
||||
import {
|
||||
buildFrameDocuments,
|
||||
type FrameAxBatch,
|
||||
type FrameDomInput,
|
||||
type FrameOwnedAxNode,
|
||||
} from "../frame-document";
|
||||
|
||||
function domNode(frameId: string, backendNodeId: number): CapturedNode {
|
||||
return {
|
||||
@@ -17,19 +22,16 @@ function domNode(frameId: string, backendNodeId: number): CapturedNode {
|
||||
};
|
||||
}
|
||||
|
||||
function captured(frameNodes: Map<string, CapturedNode[]>): CapturedViewModel {
|
||||
function captured(frameNodes: Map<string, CapturedNode[]>): FrameDomInput {
|
||||
return {
|
||||
nodes: frameNodes.get("main") ?? [],
|
||||
viewport: { width: 800, height: 600 },
|
||||
iframeNodes: new Map(),
|
||||
frameNodes,
|
||||
rootFrameId: "main",
|
||||
excludedBackendNodeIds: new Set(),
|
||||
};
|
||||
}
|
||||
|
||||
describe("buildFrameDocuments", () => {
|
||||
it("partitions mixed AX results by node ownership and cuts cross-frame parent edges", () => {
|
||||
it("partitions mixed AX results by node ownership and cuts cross-frame parent edges", async () => {
|
||||
const graph: CdpFrameGraph = {
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
@@ -54,7 +56,7 @@ describe("buildFrameDocuments", () => {
|
||||
},
|
||||
];
|
||||
|
||||
const documents = buildFrameDocuments(
|
||||
const documents = await buildFrameDocuments(
|
||||
graph,
|
||||
batches,
|
||||
captured(
|
||||
@@ -72,7 +74,7 @@ describe("buildFrameDocuments", () => {
|
||||
expect(childNodes.find((node) => node.nodeId === "virtual")?.frameId).toBe("child");
|
||||
});
|
||||
|
||||
it("deduplicates overlapping same-target AX results using the strongest ownership", () => {
|
||||
it("deduplicates overlapping same-target AX results using the strongest ownership", async () => {
|
||||
const graph: CdpFrameGraph = {
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
@@ -92,7 +94,7 @@ describe("buildFrameDocuments", () => {
|
||||
},
|
||||
];
|
||||
|
||||
const documents = buildFrameDocuments(
|
||||
const documents = await buildFrameDocuments(
|
||||
graph,
|
||||
batches,
|
||||
captured(
|
||||
@@ -107,7 +109,7 @@ describe("buildFrameDocuments", () => {
|
||||
expect(documents.find((document) => document.frameId === "child")?.axNodes).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("uses target-scoped backend ownership when OOPIF backend ids collide", () => {
|
||||
it("uses target-scoped backend ownership when OOPIF backend ids collide", async () => {
|
||||
const graph: CdpFrameGraph = {
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
@@ -129,7 +131,7 @@ describe("buildFrameDocuments", () => {
|
||||
nodes: [{ nodeId: `${frame.frameId}-node`, backendDOMNodeId: 7 }],
|
||||
}));
|
||||
|
||||
const documents = buildFrameDocuments(
|
||||
const documents = await buildFrameDocuments(
|
||||
graph,
|
||||
batches,
|
||||
captured(
|
||||
@@ -149,7 +151,7 @@ describe("buildFrameDocuments", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("uses captured frame parent identity instead of guessing from backend ids", () => {
|
||||
it("uses captured frame parent identity instead of guessing from backend ids", async () => {
|
||||
const frameNodes = new Map([
|
||||
["main", [domNode("main", 1)]],
|
||||
["left", [domNode("left", 7)]],
|
||||
@@ -157,18 +159,51 @@ describe("buildFrameDocuments", () => {
|
||||
["child", [domNode("child", 9)]],
|
||||
]);
|
||||
const capture = captured(frameNodes);
|
||||
capture.frameOwnerBackendNodeIds = new Map([["child", 7]]);
|
||||
capture.frameParentIds = new Map([["child", "left"]]);
|
||||
const frames = ["main", "left", "right", "child"].map((frameId) => ({
|
||||
frame: { frameId, target: { tabId: 4 } },
|
||||
frame: {
|
||||
frameId,
|
||||
target: { tabId: 4 },
|
||||
...(frameId === "child" ? { parentFrameId: "left", ownerBackendNodeId: 7 } : {}),
|
||||
},
|
||||
nodes: [],
|
||||
}));
|
||||
|
||||
const documents = buildFrameDocuments(null, frames, capture);
|
||||
const documents = await buildFrameDocuments(null, frames, capture);
|
||||
|
||||
expect(documents.find((document) => document.frameId === "child")).toMatchObject({
|
||||
parentFrameId: "left",
|
||||
ownerBackendNodeId: 7,
|
||||
});
|
||||
});
|
||||
it("rejects explicit AX frame ownership in a different target", async () => {
|
||||
const frames = [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{ frameId: "remote", target: { tabId: 4, sessionId: "remote" } },
|
||||
];
|
||||
const unresolved = vi.fn();
|
||||
const documents = await buildFrameDocuments(
|
||||
{ rootFrameId: "main", frames },
|
||||
[{ frame: frames[0], nodes: [{ nodeId: "wrong", frameId: "remote" }] }],
|
||||
captured(new Map()),
|
||||
undefined,
|
||||
unresolved,
|
||||
);
|
||||
expect(documents.every((doc) => doc.axNodes.length === 0)).toBe(true);
|
||||
expect(unresolved).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("resolves a reverse-ordered deep AX parent chain without recursion", async () => {
|
||||
const frame = { frameId: "main", target: { tabId: 4 } };
|
||||
const nodes: FrameOwnedAxNode[] = Array.from({ length: 10000 }, (_, i) => ({
|
||||
nodeId: String(i),
|
||||
...(i ? { parentId: String(i - 1) } : { frameId: "main" }),
|
||||
})).reverse();
|
||||
const documents = await buildFrameDocuments(
|
||||
{ rootFrameId: "main", frames: [frame] },
|
||||
[{ frame, nodes }],
|
||||
captured(new Map()),
|
||||
);
|
||||
expect(documents[0].axNodes).toHaveLength(10000);
|
||||
expect(documents[0].axNodes[0].frameId).toBe("main");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
function captureAbortError(): Error {
|
||||
const error = new Error("observation aborted");
|
||||
error.name = "AbortError";
|
||||
return error;
|
||||
}
|
||||
|
||||
export function isAbortError(error: unknown): boolean {
|
||||
return (
|
||||
typeof error === "object" &&
|
||||
error !== null &&
|
||||
(error as { name?: string }).name === "AbortError"
|
||||
);
|
||||
}
|
||||
|
||||
export function throwIfAborted(signal: AbortSignal | undefined): void {
|
||||
if (signal?.aborted) throw captureAbortError();
|
||||
}
|
||||
|
||||
/** Check at bounded work blocks; yield only after a processing slice is spent.
|
||||
* Each caller owns its deadline, so unrelated captures do not share state. */
|
||||
export function createCaptureCheckpoint(signal?: AbortSignal): () => Promise<void> | undefined {
|
||||
throwIfAborted(signal);
|
||||
let deadline = performance.now() + 8;
|
||||
return () => {
|
||||
throwIfAborted(signal);
|
||||
if (performance.now() < deadline) return;
|
||||
return new Promise<void>((resolve) => {
|
||||
const channel = new MessageChannel();
|
||||
channel.port1.onmessage = () => {
|
||||
channel.port1.close();
|
||||
channel.port2.close();
|
||||
resolve();
|
||||
};
|
||||
channel.port2.postMessage(null);
|
||||
}).then(() => {
|
||||
throwIfAborted(signal);
|
||||
deadline = performance.now() + 8;
|
||||
});
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,357 @@
|
||||
import {
|
||||
type CdpFrame,
|
||||
type CdpFrameGraph,
|
||||
type CdpTarget,
|
||||
cdpTargetKey,
|
||||
} from "@/browser-driver/frame-graph";
|
||||
import { cssViewport, GeometryContext } from "../geometry/frame-context";
|
||||
import { type CdpRunner, cdpRunnerForTarget } from "../shared";
|
||||
import { collectOverlayExcludedBackendIds } from "./capture";
|
||||
import {
|
||||
isAbortError as isCaptureAbort,
|
||||
throwIfAborted as throwCaptureAborted,
|
||||
} from "./capture-abort";
|
||||
import type { CapturedSceneInput } from "./facts";
|
||||
import { buildDocumentIndex, type CaptureIssue, type ObservationFacts } from "./facts";
|
||||
import { enrichFormControlStates } from "./form-capture";
|
||||
import {
|
||||
buildFrameDocuments,
|
||||
type FrameAxBatch,
|
||||
type FrameDocument,
|
||||
type FrameOwnedAxNode,
|
||||
} from "./frame-document";
|
||||
import { type NormalizedFrameDocument, normalizeSnapshot } from "./normalize";
|
||||
import { describeSnapshotFrames, REQUESTED_STYLES, type SnapshotReply } from "./snapshot";
|
||||
|
||||
interface TargetBatch<T extends FrameOwnedAxNode> {
|
||||
target: CdpTarget;
|
||||
frames: CdpFrame[];
|
||||
documents: NormalizedFrameDocument[];
|
||||
ax: FrameAxBatch<T>[];
|
||||
fallbackExcluded: Set<number>;
|
||||
snapshotFrameIds?: Set<string>;
|
||||
rootFrameId?: string;
|
||||
}
|
||||
|
||||
/** A fixed worker pool scoped to one capture; each worker issues at most one
|
||||
* collection request at a time. Geometry has its own measurement-phase bound. */
|
||||
async function collectTasks<T>(
|
||||
items: readonly T[],
|
||||
collect: (item: T) => Promise<void>,
|
||||
signal?: AbortSignal,
|
||||
): Promise<void> {
|
||||
let cursor = 0;
|
||||
let failed = false;
|
||||
let failure: unknown;
|
||||
await Promise.all(
|
||||
Array.from({ length: Math.min(4, items.length) }, async () => {
|
||||
try {
|
||||
while (cursor < items.length && !failed && !signal?.aborted) {
|
||||
const item = items[cursor++];
|
||||
await collect(item);
|
||||
}
|
||||
} catch (error) {
|
||||
if (!failed) failure = error;
|
||||
failed = true;
|
||||
}
|
||||
}),
|
||||
);
|
||||
// A worker's rejection must not let the capture outlive its own task scope.
|
||||
// In-flight reads and their finally blocks settle before cancellation/failure.
|
||||
throwCaptureAborted(signal);
|
||||
if (failed) throw failure;
|
||||
}
|
||||
|
||||
export async function captureObservationFacts<T extends FrameOwnedAxNode>(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
signal?: AbortSignal,
|
||||
pageUrl?: string,
|
||||
): Promise<ObservationFacts<T>> {
|
||||
const startedAt = Date.now();
|
||||
throwCaptureAborted(signal);
|
||||
let graph: CdpFrameGraph | undefined;
|
||||
try {
|
||||
graph = await cdp.getFrameGraph?.(tabId);
|
||||
} catch (error) {
|
||||
if (isCaptureAbort(error)) throw error;
|
||||
}
|
||||
throwCaptureAborted(signal);
|
||||
const geometry = new GeometryContext(cdp, tabId, graph, signal);
|
||||
const issues: CaptureIssue[] = [];
|
||||
const graphFrames = new Map(graph?.frames.map((frame) => [frame.frameId, frame]) ?? []);
|
||||
const groups = new Map<string, TargetBatch<T>>();
|
||||
for (const frame of graph?.frames ?? []) {
|
||||
const key = cdpTargetKey(frame.target);
|
||||
let group = groups.get(key);
|
||||
if (!group) {
|
||||
group = {
|
||||
target: frame.target,
|
||||
frames: [],
|
||||
documents: [],
|
||||
ax: [],
|
||||
fallbackExcluded: new Set(),
|
||||
};
|
||||
groups.set(key, group);
|
||||
}
|
||||
group.frames.push(
|
||||
!frame.target.sessionId && frame.frameId === graph?.rootFrameId && !frame.url && pageUrl
|
||||
? { ...frame, url: pageUrl }
|
||||
: frame,
|
||||
);
|
||||
}
|
||||
if (!groups.has(cdpTargetKey({ tabId })))
|
||||
groups.set(cdpTargetKey({ tabId }), {
|
||||
target: { tabId },
|
||||
frames: [],
|
||||
documents: [],
|
||||
ax: [],
|
||||
fallbackExcluded: new Set(),
|
||||
});
|
||||
const batches = [...groups.values()];
|
||||
const unavailable = (batch: TargetBatch<T>, stage: CaptureIssue["stage"], frameId?: string) =>
|
||||
issues.push({ target: batch.target, frameId, stage, reason: "capture-unavailable" });
|
||||
let firstFailure: unknown;
|
||||
const axReady = new Set<TargetBatch<T>>();
|
||||
await collectTasks(
|
||||
batches,
|
||||
async (batch) => {
|
||||
const scoped = cdpRunnerForTarget(cdp, batch.target);
|
||||
try {
|
||||
throwCaptureAborted(signal);
|
||||
await scoped.send(tabId, "DOMSnapshot.enable", {});
|
||||
throwCaptureAborted(signal);
|
||||
const snapshot = await scoped.send<SnapshotReply>(tabId, "DOMSnapshot.captureSnapshot", {
|
||||
computedStyles: REQUESTED_STYLES,
|
||||
includePaintOrder: true,
|
||||
includeDOMRects: true,
|
||||
});
|
||||
throwCaptureAborted(signal);
|
||||
const observed = describeSnapshotFrames(snapshot, batch.target, graphFrames, pageUrl);
|
||||
batch.snapshotFrameIds = observed.ids;
|
||||
batch.rootFrameId = observed.rootFrameId;
|
||||
// Keep graph-only frames for AX fallback, never in preference to observed membership.
|
||||
batch.frames = [
|
||||
...observed.frames,
|
||||
...batch.frames.filter((frame) => !observed.ids.has(frame.frameId)),
|
||||
];
|
||||
batch.documents = await normalizeSnapshot(
|
||||
snapshot,
|
||||
batch.target,
|
||||
batch.frames.filter((frame) => observed.ids.has(frame.frameId)),
|
||||
geometry,
|
||||
issues,
|
||||
signal,
|
||||
observed.rootFrameId,
|
||||
);
|
||||
const formsAvailable = await enrichFormControlStates(
|
||||
scoped,
|
||||
tabId,
|
||||
batch.documents.map((doc) => doc.nodes),
|
||||
signal,
|
||||
);
|
||||
throwCaptureAborted(signal);
|
||||
if (!formsAvailable) unavailable(batch, "forms");
|
||||
const present = new Set(batch.documents.map((doc) => doc.frame.frameId));
|
||||
for (const frame of batch.frames)
|
||||
if (!present.has(frame.frameId)) unavailable(batch, "dom", frame.frameId);
|
||||
} catch (error) {
|
||||
throwCaptureAborted(signal);
|
||||
if (isCaptureAbort(error)) throw error;
|
||||
firstFailure ??= error;
|
||||
unavailable(batch, "dom");
|
||||
batch.fallbackExcluded = await collectOverlayExcludedBackendIds(scoped, tabId, signal);
|
||||
}
|
||||
if (!batch.frames.length && !graph)
|
||||
batch.frames = [
|
||||
{
|
||||
frameId: "root",
|
||||
target: batch.target,
|
||||
...(!batch.target.sessionId && pageUrl ? { url: pageUrl } : {}),
|
||||
},
|
||||
];
|
||||
try {
|
||||
throwCaptureAborted(signal);
|
||||
await scoped.send(tabId, "Accessibility.enable", {});
|
||||
} catch (error) {
|
||||
throwCaptureAborted(signal);
|
||||
if (isCaptureAbort(error)) throw error;
|
||||
firstFailure ??= error;
|
||||
unavailable(batch, "ax");
|
||||
return;
|
||||
}
|
||||
axReady.add(batch);
|
||||
},
|
||||
signal,
|
||||
);
|
||||
|
||||
// Schedule AX by frame across all ready targets, rather than nesting pools.
|
||||
// Keep result slots in source order even when CDP replies arrive out of order.
|
||||
const axTasks: { batch: TargetBatch<T>; frame: CdpFrame; result?: FrameAxBatch<T> }[] =
|
||||
batches.flatMap((batch) =>
|
||||
axReady.has(batch) ? batch.frames.map((frame) => ({ batch, frame })) : [],
|
||||
);
|
||||
await collectTasks(
|
||||
axTasks,
|
||||
async (task) => {
|
||||
const { batch, frame } = task;
|
||||
try {
|
||||
throwCaptureAborted(signal);
|
||||
const result = await cdpRunnerForTarget(cdp, batch.target).send<{ nodes?: T[] }>(
|
||||
tabId,
|
||||
"Accessibility.getFullAXTree",
|
||||
frame.frameId === "root" ? {} : { frameId: frame.frameId },
|
||||
);
|
||||
throwCaptureAborted(signal);
|
||||
task.result = { frame, nodes: result.nodes ?? [] };
|
||||
} catch (error) {
|
||||
throwCaptureAborted(signal);
|
||||
if (isCaptureAbort(error)) throw error;
|
||||
firstFailure ??= error;
|
||||
unavailable(batch, "ax", frame.frameId);
|
||||
}
|
||||
},
|
||||
signal,
|
||||
);
|
||||
for (const task of axTasks) if (task.result) task.batch.ax.push(task.result);
|
||||
|
||||
let viewport = { width: 0, height: 0 };
|
||||
try {
|
||||
const measured = cssViewport(await geometry.layoutMetrics({ tabId }));
|
||||
viewport = { width: measured.width, height: measured.height };
|
||||
} catch (error) {
|
||||
if (isCaptureAbort(error)) throw error;
|
||||
}
|
||||
|
||||
// A snapshot claim outranks an old graph hint. Conflicting actual claims
|
||||
// cannot be resolved by completion order or by overwriting a frameId map.
|
||||
const claims = new Map<string, string>();
|
||||
const invalid = new Set<string>();
|
||||
for (const batch of batches) {
|
||||
const targetKey = cdpTargetKey(batch.target);
|
||||
for (const id of batch.snapshotFrameIds ?? []) {
|
||||
const previous = claims.get(id);
|
||||
if (previous !== undefined && previous !== targetKey) invalid.add(id);
|
||||
claims.set(id, targetKey);
|
||||
}
|
||||
const accepted = new Set(batch.frames.map((frame) => frame.frameId));
|
||||
for (const id of batch.snapshotFrameIds ?? []) if (!accepted.has(id)) invalid.add(id);
|
||||
}
|
||||
const currentFrames = batches.flatMap((batch) =>
|
||||
batch.frames.filter(
|
||||
(frame) =>
|
||||
!claims.has(frame.frameId) || claims.get(frame.frameId) === cdpTargetKey(batch.target),
|
||||
),
|
||||
);
|
||||
const rootBatch = batches.find((batch) => !batch.target.sessionId);
|
||||
const rootFrameId =
|
||||
rootBatch?.rootFrameId ?? graph?.rootFrameId ?? rootBatch?.frames[0]?.frameId ?? "root";
|
||||
const children = new Map<string, string[]>();
|
||||
for (const frame of currentFrames) {
|
||||
if (!frame.parentFrameId) continue;
|
||||
const siblings = children.get(frame.parentFrameId);
|
||||
if (siblings) siblings.push(frame.frameId);
|
||||
else children.set(frame.parentFrameId, [frame.frameId]);
|
||||
}
|
||||
const blocked = [...invalid];
|
||||
for (let i = 0; i < blocked.length; i++) {
|
||||
for (const child of children.get(blocked[i]) ?? []) {
|
||||
if (!invalid.has(child)) {
|
||||
invalid.add(child);
|
||||
blocked.push(child);
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const frame of currentFrames) {
|
||||
if (invalid.has(frame.frameId))
|
||||
issues.push({
|
||||
target: frame.target,
|
||||
frameId: frame.frameId,
|
||||
stage: "ownership",
|
||||
reason: "frame-ownership-unresolved",
|
||||
});
|
||||
}
|
||||
if (invalid.has(rootFrameId)) throw new Error("observation root document ownership is ambiguous");
|
||||
const frames = currentFrames.filter((frame) => !invalid.has(frame.frameId));
|
||||
const frameById = new Map(frames.map((frame) => [frame.frameId, frame]));
|
||||
const belongs = (frame: CdpFrame) => {
|
||||
const current = frameById.get(frame.frameId);
|
||||
return current && cdpTargetKey(current.target) === cdpTargetKey(frame.target);
|
||||
};
|
||||
const decoded = new Map(
|
||||
batches
|
||||
.flatMap((batch) => batch.documents)
|
||||
.filter((doc) => belongs(doc.frame))
|
||||
.map((doc) => [doc.frame.frameId, doc]),
|
||||
);
|
||||
const ax = batches.flatMap((batch) => batch.ax).filter((batch) => belongs(batch.frame));
|
||||
const documents = await buildFrameDocuments(
|
||||
{ rootFrameId, frames },
|
||||
ax,
|
||||
{
|
||||
nodes: [],
|
||||
rootFrameId,
|
||||
frameNodes: new Map([...decoded].map(([id, doc]) => [id, doc.nodes])),
|
||||
frameExcludedBackendNodeIds: new Map(
|
||||
[...decoded].map(([id, doc]) => [id, doc.index.excludedBackendNodeIds]),
|
||||
),
|
||||
},
|
||||
signal,
|
||||
(frame) =>
|
||||
issues.push({
|
||||
target: frame.target,
|
||||
frameId: frame.frameId,
|
||||
stage: "ownership",
|
||||
reason: "frame-ownership-unresolved",
|
||||
}),
|
||||
);
|
||||
const facts: ObservationFacts<T>["documents"][number][] = [];
|
||||
for (const document of documents) {
|
||||
const doc = decoded.get(document.frameId);
|
||||
const index = doc?.index ?? (await buildDocumentIndex([], signal));
|
||||
const fallback = groups.get(cdpTargetKey(document.target))?.fallbackExcluded;
|
||||
const {
|
||||
domNodes,
|
||||
axNodes,
|
||||
contextScopeId: _scope,
|
||||
excludedBackendNodeIds: _excluded,
|
||||
...frame
|
||||
} = document;
|
||||
facts.push({
|
||||
frame,
|
||||
index: fallback?.size ? { ...index, excludedBackendNodeIds: fallback } : index,
|
||||
domNodes,
|
||||
axNodes,
|
||||
});
|
||||
}
|
||||
throwCaptureAborted(signal);
|
||||
if (firstFailure && facts.every((doc) => !doc.domNodes.length && !doc.axNodes.length))
|
||||
throw firstFailure;
|
||||
return { rootFrameId, viewport, documents: facts, issues, startedAt, finishedAt: Date.now() };
|
||||
}
|
||||
|
||||
/** Existing semantic consumers get a narrow view; no raw snapshot
|
||||
* data can leak through recording's allowlisted output DTO. */
|
||||
export function semanticCapture<T extends FrameOwnedAxNode>(
|
||||
facts: ObservationFacts<T>,
|
||||
): { captured: CapturedSceneInput; documents: FrameDocument<T>[] } {
|
||||
const documents = facts.documents.map((doc) => ({
|
||||
...doc.frame,
|
||||
contextScopeId: doc.frame.frameId,
|
||||
domNodes: doc.domNodes.filter(
|
||||
(node) => !doc.index.excludedBackendNodeIds.has(node.backendNodeId),
|
||||
),
|
||||
axNodes: doc.axNodes,
|
||||
excludedBackendNodeIds: doc.index.excludedBackendNodeIds,
|
||||
}));
|
||||
const root = documents.find((doc) => doc.frameId === facts.rootFrameId);
|
||||
return {
|
||||
documents,
|
||||
captured: {
|
||||
nodes: root?.domNodes ?? [],
|
||||
viewport: facts.viewport,
|
||||
rootFrameId: facts.rootFrameId,
|
||||
excludedBackendNodeIds: root?.excludedBackendNodeIds ?? new Set(),
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
import type { Rect } from "@browser-skill/vom";
|
||||
|
||||
export interface CapturedNode {
|
||||
backendNodeId: number;
|
||||
parentBackendNodeId: number | null;
|
||||
frameId?: string;
|
||||
/** Owning iframe backend node id; `null` for the top-level document. */
|
||||
ownerFrameBackendNodeId?: number | null;
|
||||
tag: string;
|
||||
attrs: Record<string, string>;
|
||||
/** Top-level viewport-relative CSS px, clipped to the owning frame viewport. */
|
||||
rect: Rect | null;
|
||||
/** Frame-local viewport-relative CSS px before top-level projection. */
|
||||
localRect?: Rect | null;
|
||||
paintOrder: number;
|
||||
position: string;
|
||||
pointerEvents: string;
|
||||
/**
|
||||
* computed `cursor`. `cursor: pointer` is the strongest CDP-free signal
|
||||
* that a non-semantic element (a `<div>`/`<span>` with a click handler)
|
||||
* is actually an interactive control — used by the adapter to surface
|
||||
* custom buttons/checkboxes the AX tree drops as `generic`. Optional like
|
||||
* `textContent`: the live parser always sets it, hand-built fixtures may not.
|
||||
*/
|
||||
cursor?: string;
|
||||
/**
|
||||
* Whether the live DOM snapshot provides a painted, non-hidden box for this
|
||||
* node. Semantic resolution uses this only for DOM fallback nodes; AX-backed
|
||||
* nodes remain authoritative even when they are outside the viewport.
|
||||
*/
|
||||
rendered?: boolean;
|
||||
textContent?: string;
|
||||
formState?: "empty" | "filled" | "default";
|
||||
formValue?: string;
|
||||
formDefaultValue?: string;
|
||||
formPlaceholder?: string;
|
||||
}
|
||||
@@ -1,298 +1,19 @@
|
||||
// CDP capture adapter: DOMSnapshot.captureSnapshot + Page.getLayoutMetrics
|
||||
// → CapturedNode[] + Viewport. This is the ONLY VOM module that touches
|
||||
// raw CDP. The captureSnapshot reply is columnar (parallel arrays + a
|
||||
// shared string table); requested computed styles are decoded through
|
||||
// STYLE_COL so capture and visibility policy share one explicit contract.
|
||||
|
||||
import type { Rect, Viewport } from "@browser-skill/vom";
|
||||
// Browser-side hover probes and overlay exclusion fallback. Static collection
|
||||
// lives in capture-coordinator; snapshot decoding has no browser dependency.
|
||||
import { evaluateHoverTrigger } from "@/lib/hover-trigger-policy";
|
||||
import { isOverlayHostNode, OVERLAY_HOST_SELECTOR } from "../../lib/overlay-bridge";
|
||||
import { childFrameProjection, type GeometryProjection, projectRectToViewport } from "../geometry";
|
||||
import { OVERLAY_HOST_SELECTOR } from "../../lib/overlay-bridge";
|
||||
import type { CdpRunner } from "../shared";
|
||||
import type { CapturedNode } from "./capture-types";
|
||||
import type { CapturedSurfaceProbe } from "./facts";
|
||||
|
||||
export type {
|
||||
CapturedNode,
|
||||
CapturedSurfaceProbe,
|
||||
} from "./facts";
|
||||
|
||||
import { isAbortError, throwIfAborted } from "./capture-abort";
|
||||
import { clearHover, ProbeBudget, waitForHover } from "./hover-perception";
|
||||
|
||||
const REQUESTED_STYLES = ["position", "pointer-events", "cursor", "visibility", "opacity"] as const;
|
||||
const STYLE_COL = Object.fromEntries(
|
||||
REQUESTED_STYLES.map((name, index) => [name, index]),
|
||||
) as Record<(typeof REQUESTED_STYLES)[number], number>;
|
||||
|
||||
export interface CapturedNode {
|
||||
backendNodeId: number;
|
||||
parentBackendNodeId: number | null;
|
||||
frameId?: string;
|
||||
/** Owning iframe backend node id; `null` for the top-level document. */
|
||||
ownerFrameBackendNodeId?: number | null;
|
||||
tag: string;
|
||||
attrs: Record<string, string>;
|
||||
/** Top-level viewport-relative CSS px, clipped to the owning frame viewport. */
|
||||
rect: Rect | null;
|
||||
/** Frame-local viewport-relative CSS px before top-level projection. */
|
||||
localRect?: Rect | null;
|
||||
paintOrder: number;
|
||||
position: string;
|
||||
pointerEvents: string;
|
||||
/**
|
||||
* computed `cursor`. `cursor: pointer` is the strongest CDP-free signal
|
||||
* that a non-semantic element (a `<div>`/`<span>` with a click handler)
|
||||
* is actually an interactive control — used by the adapter to surface
|
||||
* custom buttons/checkboxes the AX tree drops as `generic`. Optional like
|
||||
* `textContent`: the live parser always sets it, hand-built fixtures may not.
|
||||
*/
|
||||
cursor?: string;
|
||||
/**
|
||||
* Whether the live DOM snapshot provides a painted, non-hidden box for this
|
||||
* node. Semantic resolution uses this only for DOM fallback nodes; AX-backed
|
||||
* nodes remain authoritative even when they are outside the viewport.
|
||||
*/
|
||||
rendered?: boolean;
|
||||
textContent?: string;
|
||||
formState?: "empty" | "filled" | "default";
|
||||
formValue?: string;
|
||||
formDefaultValue?: string;
|
||||
formPlaceholder?: string;
|
||||
}
|
||||
|
||||
export type CapturedIframeNodes = Map<number, CapturedNode[]>;
|
||||
|
||||
export interface CapturedSurfaceProbe {
|
||||
triggerBackendNodeId: number;
|
||||
triggerPoint?: { x: number; y: number };
|
||||
triggerAction: "hover" | "focus" | string;
|
||||
subItems: string[];
|
||||
confidence?: "high" | "medium" | "low";
|
||||
}
|
||||
|
||||
export interface CapturedViewModel {
|
||||
nodes: CapturedNode[];
|
||||
viewport: Viewport;
|
||||
iframeNodes: CapturedIframeNodes;
|
||||
frameNodes?: Map<string, CapturedNode[]>;
|
||||
frameOwnerBackendNodeIds?: Map<string, number>;
|
||||
/** Explicit DOMSnapshot frame ancestry; never inferred from backend node ids. */
|
||||
frameParentIds?: Map<string, string>;
|
||||
rootFrameId?: string;
|
||||
/** Backend node ids belonging to the agent overlay host + its shadow subtree. */
|
||||
excludedBackendNodeIds: Set<number>;
|
||||
}
|
||||
|
||||
export interface CaptureViewModelOptions {
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
function captureAbortError(): Error {
|
||||
const error = new Error("observation aborted");
|
||||
error.name = "AbortError";
|
||||
return error;
|
||||
}
|
||||
|
||||
function isAbortError(error: unknown): boolean {
|
||||
return (
|
||||
typeof error === "object" &&
|
||||
error !== null &&
|
||||
(error as { name?: string }).name === "AbortError"
|
||||
);
|
||||
}
|
||||
|
||||
function throwIfAborted(signal: AbortSignal | undefined): void {
|
||||
if (signal?.aborted) throw captureAbortError();
|
||||
}
|
||||
|
||||
/** Sparse array format Chrome uses for infrequently-set per-node fields. */
|
||||
interface SparseArray {
|
||||
index: number[];
|
||||
value: number[];
|
||||
}
|
||||
|
||||
interface RareBooleanData {
|
||||
index: number[];
|
||||
}
|
||||
|
||||
interface SnapshotDocument {
|
||||
frameId?: string | number;
|
||||
scrollOffsetX?: number;
|
||||
scrollOffsetY?: number;
|
||||
nodes?: {
|
||||
parentIndex?: number[];
|
||||
nodeName?: number[];
|
||||
backendNodeId?: number[];
|
||||
attributes?: number[][];
|
||||
/**
|
||||
* Per-node text value (index into strings), set for `#text` / CDATA nodes.
|
||||
* Element nodes carry -1. Same length as `backendNodeId`.
|
||||
*/
|
||||
nodeValue?: number[];
|
||||
/** Maps node array index → index into `documents[]` for frame content. */
|
||||
contentDocumentIndex?: SparseArray;
|
||||
inputValue?: SparseArray;
|
||||
textValue?: SparseArray;
|
||||
inputChecked?: RareBooleanData;
|
||||
optionSelected?: RareBooleanData;
|
||||
};
|
||||
layout?: {
|
||||
nodeIndex?: number[];
|
||||
styles?: number[][];
|
||||
bounds?: number[][];
|
||||
paintOrders?: number[];
|
||||
};
|
||||
}
|
||||
|
||||
function snapshotFrameId(document: SnapshotDocument, strings: string[]): string | undefined {
|
||||
if (typeof document.frameId === "string") return document.frameId || undefined;
|
||||
if (typeof document.frameId === "number") return str(strings, document.frameId) || undefined;
|
||||
return undefined;
|
||||
}
|
||||
|
||||
interface SnapshotReply {
|
||||
strings?: string[];
|
||||
documents?: SnapshotDocument[];
|
||||
}
|
||||
|
||||
interface LayoutMetricsReply {
|
||||
cssLayoutViewport?: {
|
||||
clientWidth?: number;
|
||||
clientHeight?: number;
|
||||
pageX?: number;
|
||||
pageY?: number;
|
||||
};
|
||||
layoutViewport?: { clientWidth?: number; clientHeight?: number; pageX?: number; pageY?: number };
|
||||
}
|
||||
|
||||
function isFormControlTag(tag: string): boolean {
|
||||
return tag === "input" || tag === "textarea" || tag === "select";
|
||||
}
|
||||
|
||||
function isSensitiveFormControl(node: Pick<CapturedNode, "tag" | "attrs">): boolean {
|
||||
return node.tag === "input" && (node.attrs.type ?? "").toLowerCase() === "password";
|
||||
}
|
||||
|
||||
function snapshotFormState(
|
||||
value: string | undefined,
|
||||
defaultValue: string,
|
||||
hasDefaultValue: boolean,
|
||||
sensitive: boolean,
|
||||
): CapturedNode["formState"] {
|
||||
if (sensitive) {
|
||||
if (value === undefined && !hasDefaultValue) return undefined;
|
||||
return (value ?? defaultValue) === "" ? "empty" : "filled";
|
||||
}
|
||||
if (value === undefined) return undefined;
|
||||
if (value === "") return "empty";
|
||||
return value === defaultValue ? "default" : "filled";
|
||||
}
|
||||
|
||||
const MAX_FORM_ENRICH_CONTROLS = 250;
|
||||
|
||||
interface CapturedFormState {
|
||||
value?: string;
|
||||
defaultValue?: string;
|
||||
placeholder?: string;
|
||||
state?: "empty" | "filled" | "default";
|
||||
sensitive?: boolean;
|
||||
}
|
||||
|
||||
interface CapturedFrameFormState {
|
||||
controls: CapturedFormState[];
|
||||
childFrames: CapturedFrameFormState[];
|
||||
}
|
||||
|
||||
function formStateBatchExpression(maxControls: number): string {
|
||||
return `(() => {
|
||||
const maxControls = ${JSON.stringify(maxControls)};
|
||||
let remaining = maxControls;
|
||||
const controlSelector = "input,textarea,select";
|
||||
const controlState = (el) => {
|
||||
const tag = el.tagName.toLowerCase();
|
||||
const type = tag === "input" ? String(el.type || "text").toLowerCase() : tag;
|
||||
const sensitive = type === "password" || type === "credit-card";
|
||||
const rawValue = typeof el.value === "string" ? el.value : "";
|
||||
const defaultValue = typeof el.defaultValue === "string" ? el.defaultValue : "";
|
||||
const placeholder = typeof el.placeholder === "string" ? el.placeholder : "";
|
||||
const state = rawValue === "" ? "empty" : rawValue === defaultValue ? "default" : "filled";
|
||||
return {
|
||||
state,
|
||||
sensitive,
|
||||
placeholder,
|
||||
...(sensitive ? {} : { value: rawValue, defaultValue }),
|
||||
};
|
||||
};
|
||||
const collect = (doc) => {
|
||||
const controls = [];
|
||||
for (const el of Array.from(doc.querySelectorAll(controlSelector))) {
|
||||
if (remaining <= 0) break;
|
||||
controls.push(controlState(el));
|
||||
remaining -= 1;
|
||||
}
|
||||
const childFrames = [];
|
||||
for (const frame of Array.from(doc.querySelectorAll("iframe"))) {
|
||||
let childDoc = null;
|
||||
try { childDoc = frame.contentDocument; } catch { childDoc = null; }
|
||||
childFrames.push(childDoc && remaining > 0 ? collect(childDoc) : { controls: [], childFrames: [] });
|
||||
}
|
||||
return { controls, childFrames };
|
||||
};
|
||||
return collect(document);
|
||||
})()`;
|
||||
}
|
||||
|
||||
function flattenFrameFormStates(
|
||||
frame: CapturedFrameFormState | undefined,
|
||||
out: CapturedFormState[][] = [],
|
||||
): CapturedFormState[][] {
|
||||
if (!frame) return out;
|
||||
out.push(Array.isArray(frame.controls) ? frame.controls : []);
|
||||
for (const child of Array.isArray(frame.childFrames) ? frame.childFrames : []) {
|
||||
flattenFrameFormStates(child, out);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function applyFormStates(nodes: CapturedNode[], states: CapturedFormState[]): void {
|
||||
let index = 0;
|
||||
for (const node of nodes) {
|
||||
if (!isFormControlTag(node.tag)) continue;
|
||||
const state = states[index];
|
||||
index += 1;
|
||||
if (!state) continue;
|
||||
node.formState = state.state;
|
||||
node.formPlaceholder = state.placeholder ?? "";
|
||||
if (state.sensitive || isSensitiveFormControl(node)) {
|
||||
delete node.formValue;
|
||||
delete node.formDefaultValue;
|
||||
delete node.attrs.value;
|
||||
} else {
|
||||
node.formDefaultValue = state.defaultValue ?? "";
|
||||
if (state.value !== undefined) node.formValue = state.value;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function enrichFormControlStates(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
frameNodeGroups: CapturedNode[][],
|
||||
signal?: AbortSignal,
|
||||
): Promise<void> {
|
||||
const hasControls = frameNodeGroups.some((nodes) =>
|
||||
nodes.some((node) => isFormControlTag(node.tag)),
|
||||
);
|
||||
if (!hasControls) return;
|
||||
try {
|
||||
throwIfAborted(signal);
|
||||
const result = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
|
||||
expression: formStateBatchExpression(MAX_FORM_ENRICH_CONTROLS),
|
||||
returnByValue: true,
|
||||
});
|
||||
throwIfAborted(signal);
|
||||
const frameStates = flattenFrameFormStates(runtimeValue<CapturedFrameFormState>(result));
|
||||
for (let i = 0; i < frameNodeGroups.length; i += 1) {
|
||||
applyFormStates(frameNodeGroups[i], frameStates[i] ?? []);
|
||||
}
|
||||
} catch (error) {
|
||||
if (isAbortError(error)) throw error;
|
||||
// Best-effort enrichment. DOMSnapshot/AX data still carries the nodes.
|
||||
}
|
||||
}
|
||||
|
||||
interface RuntimeEvaluateReply {
|
||||
result?: {
|
||||
value?: unknown;
|
||||
@@ -314,44 +35,6 @@ interface CdpDomNode {
|
||||
shadowRoots?: CdpDomNode[];
|
||||
}
|
||||
|
||||
interface ParseDocumentResult {
|
||||
nodes: CapturedNode[];
|
||||
excludedBackendNodeIds: Set<number>;
|
||||
}
|
||||
|
||||
interface FrameContext {
|
||||
frameId?: string;
|
||||
ownerFrameBackendNodeId: number | null;
|
||||
projection: GeometryProjection | null;
|
||||
scrollX: number;
|
||||
scrollY: number;
|
||||
}
|
||||
|
||||
function str(strings: string[], idx: number | undefined): string {
|
||||
if (idx === undefined || idx < 0) return "";
|
||||
return strings[idx] ?? "";
|
||||
}
|
||||
|
||||
function devicePixelRatio(metrics: LayoutMetricsReply): number {
|
||||
const layoutW = metrics.layoutViewport?.clientWidth ?? 0;
|
||||
const cssW = metrics.cssLayoutViewport?.clientWidth ?? 0;
|
||||
if (!layoutW || !cssW) return 1;
|
||||
const dpr = layoutW / cssW;
|
||||
if (!Number.isFinite(dpr) || dpr <= 0) return 1;
|
||||
return dpr >= 1 ? dpr : 1;
|
||||
}
|
||||
|
||||
function sparseIndexMap(sparse: SparseArray | undefined): Map<number, number> {
|
||||
const out = new Map<number, number>();
|
||||
if (!sparse?.index || !sparse.value) return out;
|
||||
for (let i = 0; i < sparse.index.length; i++) {
|
||||
const nodeIndex = sparse.index[i];
|
||||
const docIndex = sparse.value[i];
|
||||
if (nodeIndex !== undefined && docIndex !== undefined) out.set(nodeIndex, docIndex);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function collectBackendIdsFromDomNode(node: CdpDomNode | undefined, out: Set<number>): void {
|
||||
if (!node) return;
|
||||
if (typeof node.backendNodeId === "number") {
|
||||
@@ -760,358 +443,3 @@ export async function collectOverlayExcludedBackendIds(
|
||||
}
|
||||
return excluded;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse one DOMSnapshot document into CapturedNode[].
|
||||
*
|
||||
* @param doc - the raw DOMSnapshot document object
|
||||
* @param strings - the shared string table for the whole snapshot
|
||||
* @param dpr - device-pixel-ratio from Page.getLayoutMetrics
|
||||
* @param context - frame coordinate context; output rects are top viewport-relative
|
||||
*/
|
||||
function parseDocumentNodes(
|
||||
doc: SnapshotDocument,
|
||||
strings: string[],
|
||||
dpr: number,
|
||||
context: FrameContext,
|
||||
): ParseDocumentResult {
|
||||
const dn = doc.nodes;
|
||||
const dl = doc.layout;
|
||||
if (!dn?.backendNodeId) {
|
||||
return { nodes: [], excludedBackendNodeIds: new Set() };
|
||||
}
|
||||
|
||||
const count = dn.backendNodeId.length;
|
||||
const layoutByNode = new Map<number, number>();
|
||||
(dl?.nodeIndex ?? []).forEach((nodeIdx, layoutIdx) => layoutByNode.set(nodeIdx, layoutIdx));
|
||||
|
||||
// The agent's own overlay (WXT shadow host, marked with OVERLAY_HOST_MARKER_ATTR)
|
||||
// is injected into the page and its open shadow root
|
||||
// is inlined by DOMSnapshot as descendants of the host. Its fixed
|
||||
// full-viewport click-blocker would otherwise dominate occlusion detection
|
||||
// and hide the real page, so mark the host + its whole subtree excluded.
|
||||
// parentIndex always references an earlier node, so one forward pass suffices.
|
||||
const excluded = new Array<boolean>(count).fill(false);
|
||||
const excludedBackendNodeIds = new Set<number>();
|
||||
for (let n = 0; n < count; n++) {
|
||||
const parentIdx = dn.parentIndex?.[n] ?? -1;
|
||||
const inherited = parentIdx >= 0 && excluded[parentIdx];
|
||||
let isOverlayHost = inherited;
|
||||
if (!isOverlayHost) {
|
||||
const pairs = dn.attributes?.[n] ?? [];
|
||||
const attrNames: string[] = [];
|
||||
for (let a = 0; a + 1 < pairs.length; a += 2) {
|
||||
attrNames.push(str(strings, pairs[a]));
|
||||
}
|
||||
isOverlayHost = isOverlayHostNode(str(strings, dn.nodeName?.[n]), attrNames);
|
||||
}
|
||||
if (!isOverlayHost) continue;
|
||||
excluded[n] = true;
|
||||
const backendNodeId = dn.backendNodeId[n];
|
||||
if (backendNodeId !== undefined) {
|
||||
excludedBackendNodeIds.add(backendNodeId);
|
||||
}
|
||||
}
|
||||
|
||||
const posCol = STYLE_COL.position;
|
||||
const peCol = STYLE_COL["pointer-events"];
|
||||
const cursorCol = STYLE_COL.cursor;
|
||||
const visibilityCol = STYLE_COL.visibility;
|
||||
const opacityCol = STYLE_COL.opacity;
|
||||
const inputValues = sparseIndexMap(dn.inputValue);
|
||||
const textValues = sparseIndexMap(dn.textValue);
|
||||
const checkedInputs = new Set(dn.inputChecked?.index ?? []);
|
||||
const selectedOptions = new Set(dn.optionSelected?.index ?? []);
|
||||
|
||||
// Collect visible text from #text child nodes so element CapturedNodes
|
||||
// carry a textContent value usable as a button/link label fallback.
|
||||
// nodeValue is a parallel array: string index for text nodes, -1 otherwise.
|
||||
const nodeTextContent = new Map<number, string>();
|
||||
if (dn.nodeValue) {
|
||||
for (let n = 0; n < count; n++) {
|
||||
const nvIdx = dn.nodeValue[n] ?? -1;
|
||||
if (nvIdx < 0) continue;
|
||||
const text = str(strings, nvIdx).trim();
|
||||
if (!text) continue;
|
||||
const parentIdx = dn.parentIndex?.[n] ?? -1;
|
||||
if (parentIdx >= 0) {
|
||||
const existing = nodeTextContent.get(parentIdx);
|
||||
nodeTextContent.set(parentIdx, existing ? `${existing} ${text}` : text);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Build CapturedNodes.
|
||||
const nodes: CapturedNode[] = [];
|
||||
for (let n = 0; n < count; n++) {
|
||||
if (excluded[n]) continue;
|
||||
const backendNodeId = dn.backendNodeId[n];
|
||||
const parentIdx = dn.parentIndex?.[n] ?? -1;
|
||||
const parentBackendNodeId = parentIdx >= 0 ? (dn.backendNodeId[parentIdx] ?? null) : null;
|
||||
const tag = str(strings, dn.nodeName?.[n]).toLowerCase();
|
||||
|
||||
const attrs: Record<string, string> = {};
|
||||
const pairs = dn.attributes?.[n] ?? [];
|
||||
for (let a = 0; a + 1 < pairs.length; a += 2) {
|
||||
attrs[str(strings, pairs[a]).toLowerCase()] = str(strings, pairs[a + 1]);
|
||||
}
|
||||
|
||||
let rect: Rect | null = null;
|
||||
let localRect: Rect | null = null;
|
||||
let paintOrder = 0;
|
||||
let position = "static";
|
||||
let pointerEvents = "auto";
|
||||
let cursor = "auto";
|
||||
let rendered = false;
|
||||
const li = layoutByNode.get(n);
|
||||
if (li !== undefined) {
|
||||
const b = dl?.bounds?.[li];
|
||||
if (b && b.length >= 4 && b[2] > 0 && b[3] > 0) {
|
||||
localRect = {
|
||||
x: b[0] / dpr - context.scrollX,
|
||||
y: b[1] / dpr - context.scrollY,
|
||||
w: b[2] / dpr,
|
||||
h: b[3] / dpr,
|
||||
};
|
||||
const bounds = context.projection
|
||||
? projectRectToViewport(localRect, context.projection)
|
||||
: null;
|
||||
rect = bounds ? { x: bounds.x, y: bounds.y, w: bounds.width, h: bounds.height } : null;
|
||||
}
|
||||
paintOrder = dl?.paintOrders?.[li] ?? 0;
|
||||
const styleRow = dl?.styles?.[li] ?? [];
|
||||
position = str(strings, styleRow[posCol]) || "static";
|
||||
pointerEvents = str(strings, styleRow[peCol]) || "auto";
|
||||
cursor = str(strings, styleRow[cursorCol]) || "auto";
|
||||
const visibility = str(strings, styleRow[visibilityCol]) || "visible";
|
||||
const opacity = str(strings, styleRow[opacityCol]) || "1";
|
||||
rendered =
|
||||
localRect !== null &&
|
||||
visibility !== "hidden" &&
|
||||
visibility !== "collapse" &&
|
||||
(Number.parseFloat(opacity) || 0) > 0;
|
||||
}
|
||||
|
||||
// Skip non-element nodes (#text, #cdata-section, etc.) — they carry no
|
||||
// geometry and are never queried by callers.
|
||||
if (tag.startsWith("#")) continue;
|
||||
|
||||
const textContent = nodeTextContent.get(n);
|
||||
|
||||
const rawFormValueIndex = tag === "textarea" ? textValues.get(n) : inputValues.get(n);
|
||||
const rawFormValue =
|
||||
rawFormValueIndex !== undefined ? str(strings, rawFormValueIndex) : undefined;
|
||||
const sensitive = isSensitiveFormControl({ tag, attrs });
|
||||
const formDefaultValue = attrs.value ?? "";
|
||||
const formValue = sensitive ? undefined : rawFormValue;
|
||||
const formState = snapshotFormState(
|
||||
rawFormValue,
|
||||
formDefaultValue,
|
||||
Object.prototype.hasOwnProperty.call(attrs, "value"),
|
||||
sensitive,
|
||||
);
|
||||
if (sensitive) delete attrs.value;
|
||||
|
||||
nodes.push({
|
||||
backendNodeId,
|
||||
parentBackendNodeId,
|
||||
...(context.frameId ? { frameId: context.frameId } : {}),
|
||||
ownerFrameBackendNodeId: context.ownerFrameBackendNodeId,
|
||||
tag,
|
||||
attrs,
|
||||
rect,
|
||||
localRect,
|
||||
paintOrder,
|
||||
position,
|
||||
pointerEvents,
|
||||
cursor,
|
||||
rendered,
|
||||
textContent,
|
||||
...(formValue !== undefined ? { formValue } : {}),
|
||||
...(tag === "input" || tag === "textarea"
|
||||
? {
|
||||
formPlaceholder: attrs.placeholder ?? "",
|
||||
...(!sensitive ? { formDefaultValue } : {}),
|
||||
...(formState ? { formState } : {}),
|
||||
}
|
||||
: {}),
|
||||
...(checkedInputs.has(n) ? { formValue: "true", formState: "filled" } : {}),
|
||||
...(selectedOptions.has(n) ? { formValue: attrs.value ?? textContent ?? "" } : {}),
|
||||
});
|
||||
}
|
||||
return { nodes, excludedBackendNodeIds };
|
||||
}
|
||||
|
||||
interface ParseFrameDocumentsResult {
|
||||
iframeNodes: CapturedIframeNodes;
|
||||
frameOwnerBackendNodeIds: Map<string, number>;
|
||||
frameParentIds: Map<string, string>;
|
||||
excludedBackendNodeIds: Set<number>;
|
||||
}
|
||||
|
||||
function parseChildFrameDocuments(
|
||||
documents: SnapshotDocument[],
|
||||
strings: string[],
|
||||
dpr: number,
|
||||
parentDocIndex: number,
|
||||
parentNodes: CapturedNode[],
|
||||
parentContext: FrameContext,
|
||||
visited = new Set<number>(),
|
||||
): ParseFrameDocumentsResult {
|
||||
const iframeNodes: CapturedIframeNodes = new Map();
|
||||
const frameOwnerBackendNodeIds = new Map<string, number>();
|
||||
const frameParentIds = new Map<string, string>();
|
||||
const excludedBackendNodeIds = new Set<number>();
|
||||
const parentDoc = documents[parentDocIndex];
|
||||
const cdi = sparseIndexMap(parentDoc?.nodes?.contentDocumentIndex);
|
||||
if (cdi.size === 0) {
|
||||
return { iframeNodes, frameOwnerBackendNodeIds, frameParentIds, excludedBackendNodeIds };
|
||||
}
|
||||
|
||||
const parentBackendIds = parentDoc?.nodes?.backendNodeId ?? [];
|
||||
const parentNodeByBackendId = new Map(parentNodes.map((node) => [node.backendNodeId, node]));
|
||||
|
||||
for (const [nodeArrayIdx, childDocIndex] of cdi) {
|
||||
if (visited.has(childDocIndex)) continue;
|
||||
const childDoc = documents[childDocIndex];
|
||||
if (!childDoc) continue;
|
||||
const iframeBackendId = parentBackendIds[nodeArrayIdx];
|
||||
if (iframeBackendId === undefined) continue;
|
||||
const iframeNode = parentNodeByBackendId.get(iframeBackendId);
|
||||
const projection =
|
||||
parentContext.projection && iframeNode?.localRect
|
||||
? childFrameProjection(parentContext.projection, iframeNode.localRect)
|
||||
: null;
|
||||
|
||||
const childContext: FrameContext = {
|
||||
frameId: snapshotFrameId(childDoc, strings),
|
||||
ownerFrameBackendNodeId: iframeBackendId,
|
||||
projection,
|
||||
scrollX: childDoc.scrollOffsetX ?? 0,
|
||||
scrollY: childDoc.scrollOffsetY ?? 0,
|
||||
};
|
||||
const nextVisited = new Set(visited);
|
||||
nextVisited.add(childDocIndex);
|
||||
const parsed = parseDocumentNodes(childDoc, strings, dpr, childContext);
|
||||
iframeNodes.set(iframeBackendId, parsed.nodes);
|
||||
if (childContext.frameId) {
|
||||
frameOwnerBackendNodeIds.set(childContext.frameId, iframeBackendId);
|
||||
if (parentContext.frameId) frameParentIds.set(childContext.frameId, parentContext.frameId);
|
||||
}
|
||||
for (const id of parsed.excludedBackendNodeIds) excludedBackendNodeIds.add(id);
|
||||
|
||||
const nested = parseChildFrameDocuments(
|
||||
documents,
|
||||
strings,
|
||||
dpr,
|
||||
childDocIndex,
|
||||
parsed.nodes,
|
||||
childContext,
|
||||
nextVisited,
|
||||
);
|
||||
for (const [nestedFrameId, nestedNodes] of nested.iframeNodes) {
|
||||
iframeNodes.set(nestedFrameId, nestedNodes);
|
||||
}
|
||||
for (const [frameId, ownerBackendNodeId] of nested.frameOwnerBackendNodeIds) {
|
||||
frameOwnerBackendNodeIds.set(frameId, ownerBackendNodeId);
|
||||
}
|
||||
for (const [frameId, parentFrameId] of nested.frameParentIds) {
|
||||
frameParentIds.set(frameId, parentFrameId);
|
||||
}
|
||||
for (const id of nested.excludedBackendNodeIds) excludedBackendNodeIds.add(id);
|
||||
}
|
||||
|
||||
return { iframeNodes, frameOwnerBackendNodeIds, frameParentIds, excludedBackendNodeIds };
|
||||
}
|
||||
|
||||
export async function captureViewModel(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
options: CaptureViewModelOptions = {},
|
||||
): Promise<CapturedViewModel> {
|
||||
throwIfAborted(options.signal);
|
||||
let metrics: LayoutMetricsReply = {};
|
||||
try {
|
||||
metrics = await cdp.send<LayoutMetricsReply>(tabId, "Page.getLayoutMetrics", {});
|
||||
} catch (error) {
|
||||
if (isAbortError(error)) throw error;
|
||||
console.debug("[bsk capture] layout metrics unavailable", error);
|
||||
}
|
||||
throwIfAborted(options.signal);
|
||||
const dpr = devicePixelRatio(metrics);
|
||||
const vpSrc = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
|
||||
const viewport: Viewport = {
|
||||
width: vpSrc.clientWidth ?? 0,
|
||||
height: vpSrc.clientHeight ?? 0,
|
||||
};
|
||||
const scrollX = vpSrc.pageX ?? 0;
|
||||
const scrollY = vpSrc.pageY ?? 0;
|
||||
|
||||
await cdp.send(tabId, "DOMSnapshot.enable", {});
|
||||
throwIfAborted(options.signal);
|
||||
const snap = await cdp.send<SnapshotReply>(tabId, "DOMSnapshot.captureSnapshot", {
|
||||
computedStyles: REQUESTED_STYLES,
|
||||
includePaintOrder: true,
|
||||
includeDOMRects: true,
|
||||
});
|
||||
throwIfAborted(options.signal);
|
||||
|
||||
const strings = snap.strings ?? [];
|
||||
const documents = snap.documents ?? [];
|
||||
const doc0 = documents[0];
|
||||
if (!doc0?.nodes?.backendNodeId) {
|
||||
return {
|
||||
nodes: [],
|
||||
viewport,
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
};
|
||||
}
|
||||
|
||||
const topContext: FrameContext = {
|
||||
frameId: snapshotFrameId(doc0, strings),
|
||||
ownerFrameBackendNodeId: null,
|
||||
projection: {
|
||||
sourceClips: [],
|
||||
edges: [],
|
||||
topViewport: viewport,
|
||||
},
|
||||
scrollX,
|
||||
scrollY,
|
||||
};
|
||||
const mainParsed = parseDocumentNodes(doc0, strings, dpr, topContext);
|
||||
const nodes = mainParsed.nodes;
|
||||
const excludedBackendNodeIds = new Set(mainParsed.excludedBackendNodeIds);
|
||||
|
||||
const frameParsed = parseChildFrameDocuments(documents, strings, dpr, 0, nodes, topContext);
|
||||
const iframeNodes = frameParsed.iframeNodes;
|
||||
for (const id of frameParsed.excludedBackendNodeIds) {
|
||||
excludedBackendNodeIds.add(id);
|
||||
}
|
||||
|
||||
await enrichFormControlStates(cdp, tabId, [nodes, ...iframeNodes.values()], options.signal);
|
||||
throwIfAborted(options.signal);
|
||||
|
||||
const frameNodes = new Map<string, CapturedNode[]>();
|
||||
if (topContext.frameId) frameNodes.set(topContext.frameId, nodes);
|
||||
for (const iframeFrameNodes of iframeNodes.values()) {
|
||||
const frameId = iframeFrameNodes.find((node) => node.frameId)?.frameId;
|
||||
if (frameId) frameNodes.set(frameId, iframeFrameNodes);
|
||||
}
|
||||
for (const [frameId, ownerBackendNodeId] of frameParsed.frameOwnerBackendNodeIds) {
|
||||
if (!frameNodes.has(frameId)) {
|
||||
frameNodes.set(frameId, iframeNodes.get(ownerBackendNodeId) ?? []);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
nodes,
|
||||
viewport,
|
||||
iframeNodes,
|
||||
frameNodes,
|
||||
frameOwnerBackendNodeIds: frameParsed.frameOwnerBackendNodeIds,
|
||||
frameParentIds: frameParsed.frameParentIds,
|
||||
...(topContext.frameId ? { rootFrameId: topContext.frameId } : {}),
|
||||
excludedBackendNodeIds,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -0,0 +1,141 @@
|
||||
import type { Viewport } from "@browser-skill/vom";
|
||||
import type { CdpFrame, CdpTarget } from "@/browser-driver/frame-graph";
|
||||
import { isOverlayHostNode } from "@/lib/overlay-bridge";
|
||||
import type { FrameProjectionIssue } from "../geometry/coordinate-types";
|
||||
import { createCaptureCheckpoint } from "./capture-abort";
|
||||
import type { CapturedNode } from "./capture-types";
|
||||
import type { FrameOwnedAxNode } from "./frame-document";
|
||||
|
||||
/** Bounds retain their raw snapshot document layout units until normalization. */
|
||||
export interface SnapshotLayout {
|
||||
readonly boundsSpace: "snapshot-document-layout";
|
||||
bounds?: number[];
|
||||
styles: Readonly<Record<string, string>>;
|
||||
}
|
||||
|
||||
export interface DecodedNode extends Omit<CapturedNode, "rect" | "localRect" | "rendered"> {
|
||||
nodeType?: number;
|
||||
layout?: SnapshotLayout;
|
||||
}
|
||||
|
||||
export interface NodeFacts extends CapturedNode {
|
||||
nodeType?: number;
|
||||
layout?: SnapshotLayout;
|
||||
}
|
||||
|
||||
export interface DocumentIndex<T extends DecodedNode = NodeFacts> {
|
||||
readonly nodes: ReadonlyMap<number, T>;
|
||||
readonly excludedBackendNodeIds: ReadonlySet<number>;
|
||||
}
|
||||
|
||||
export interface DecodedDocument {
|
||||
nodes: DecodedNode[];
|
||||
}
|
||||
|
||||
/** All snapshot nodes participate, including document and shadow roots. This
|
||||
* preserves overlay propagation when the semantic adapter omits those nodes. */
|
||||
export async function buildDocumentIndex<T extends DecodedNode>(
|
||||
input: readonly T[],
|
||||
signal?: AbortSignal,
|
||||
): Promise<DocumentIndex<T>> {
|
||||
const checkpoint = createCaptureCheckpoint(signal);
|
||||
const nodes = new Map<number, T>();
|
||||
const overlayByNode = new Map<number, boolean>();
|
||||
const excludedBackendNodeIds = new Set<number>();
|
||||
for (let i = 0; i < input.length; i++) {
|
||||
if (i % 256 === 0) {
|
||||
const pending = checkpoint();
|
||||
if (pending) await pending;
|
||||
}
|
||||
const node = input[i];
|
||||
nodes.set(node.backendNodeId, node);
|
||||
}
|
||||
let work = 0;
|
||||
for (const node of input) {
|
||||
if (work++ % 256 === 0) {
|
||||
const pending = checkpoint();
|
||||
if (pending) await pending;
|
||||
}
|
||||
if (overlayByNode.has(node.backendNodeId)) continue;
|
||||
const path: DecodedNode[] = [];
|
||||
const visiting = new Set<number>();
|
||||
let current: DecodedNode | undefined = node;
|
||||
let overlay = false;
|
||||
while (current && !overlayByNode.has(current.backendNodeId)) {
|
||||
if (work++ % 256 === 0) {
|
||||
const pending = checkpoint();
|
||||
if (pending) await pending;
|
||||
}
|
||||
if (visiting.has(current.backendNodeId)) {
|
||||
// Only an overlay inside the cycle may mark the cycle and its descendants.
|
||||
overlay = path
|
||||
.slice(path.findIndex((item) => item.backendNodeId === current!.backendNodeId))
|
||||
.some((item) => isOverlayHostNode(item.tag, Object.keys(item.attrs)));
|
||||
break;
|
||||
}
|
||||
visiting.add(current.backendNodeId);
|
||||
path.push(current);
|
||||
if (current.parentBackendNodeId === null) {
|
||||
current = undefined;
|
||||
break;
|
||||
}
|
||||
current = nodes.get(current.parentBackendNodeId);
|
||||
}
|
||||
if (current && overlayByNode.has(current.backendNodeId))
|
||||
overlay = overlayByNode.get(current.backendNodeId)!;
|
||||
for (let i = path.length - 1; i >= 0; i--) {
|
||||
if (work++ % 256 === 0) {
|
||||
const pending = checkpoint();
|
||||
if (pending) await pending;
|
||||
}
|
||||
const item = path[i];
|
||||
overlay = overlay || isOverlayHostNode(item.tag, Object.keys(item.attrs));
|
||||
overlayByNode.set(item.backendNodeId, overlay);
|
||||
if (overlay) excludedBackendNodeIds.add(item.backendNodeId);
|
||||
}
|
||||
}
|
||||
return { nodes, excludedBackendNodeIds };
|
||||
}
|
||||
|
||||
/** Narrow input shared by the existing semantic scene/hover consumers. */
|
||||
export interface CapturedSceneInput {
|
||||
nodes: CapturedNode[];
|
||||
viewport: Viewport;
|
||||
rootFrameId?: string;
|
||||
excludedBackendNodeIds: ReadonlySet<number>;
|
||||
}
|
||||
|
||||
export interface CaptureIssue {
|
||||
/** Owner failure or inherited blockage; absent for other geometry failures. */
|
||||
projectionIssue?: FrameProjectionIssue;
|
||||
target: CdpTarget;
|
||||
frameId?: string;
|
||||
stage: "dom" | "ax" | "ownership" | "geometry" | "forms";
|
||||
reason: "capture-unavailable" | "frame-ownership-unresolved" | "geometry-unavailable";
|
||||
}
|
||||
|
||||
export interface DocumentFacts<T extends FrameOwnedAxNode> {
|
||||
readonly frame: CdpFrame;
|
||||
readonly index: DocumentIndex;
|
||||
readonly domNodes: CapturedNode[];
|
||||
readonly axNodes: T[];
|
||||
}
|
||||
|
||||
export interface ObservationFacts<T extends FrameOwnedAxNode> {
|
||||
readonly rootFrameId: string;
|
||||
readonly viewport: Viewport;
|
||||
readonly documents: readonly DocumentFacts<T>[];
|
||||
readonly issues: readonly CaptureIssue[];
|
||||
readonly startedAt: number;
|
||||
readonly finishedAt: number;
|
||||
}
|
||||
|
||||
export interface CapturedSurfaceProbe {
|
||||
triggerBackendNodeId: number;
|
||||
triggerPoint?: { x: number; y: number };
|
||||
triggerAction: "hover" | "focus" | string;
|
||||
subItems: string[];
|
||||
confidence?: "high" | "medium" | "low";
|
||||
}
|
||||
|
||||
export type { CapturedNode } from "./capture-types";
|
||||
@@ -0,0 +1,155 @@
|
||||
import type { CdpRunner } from "../shared";
|
||||
import { isAbortError, throwIfAborted } from "./capture-abort";
|
||||
import type { CapturedNode } from "./capture-types";
|
||||
|
||||
interface RuntimeEvaluateReply {
|
||||
result?: { deepSerializedValue?: DeepSerializedValue };
|
||||
}
|
||||
const MAX_FORM_ENRICH_CONTROLS = 250;
|
||||
|
||||
interface CapturedFormState {
|
||||
value?: string;
|
||||
defaultValue?: string;
|
||||
placeholder?: string;
|
||||
state?: "empty" | "filled" | "default";
|
||||
sensitive?: boolean;
|
||||
}
|
||||
|
||||
interface DeepSerializedValue {
|
||||
type: string;
|
||||
value?: unknown;
|
||||
}
|
||||
|
||||
function formStateBatchExpression(maxControls: number): string {
|
||||
return `(() => {
|
||||
const maxControls = ${JSON.stringify(maxControls)};
|
||||
let remaining = maxControls;
|
||||
const controlSelector = "input,textarea,select";
|
||||
const controlState = (el) => {
|
||||
const tag = el.tagName.toLowerCase();
|
||||
const type = tag === "input" ? String(el.type || "text").toLowerCase() : tag;
|
||||
const sensitive = type === "password" || type === "credit-card";
|
||||
const rawValue = typeof el.value === "string" ? el.value : "";
|
||||
const defaultValue = typeof el.defaultValue === "string" ? el.defaultValue : "";
|
||||
const placeholder = typeof el.placeholder === "string" ? el.placeholder : "";
|
||||
const state = rawValue === "" ? "empty" : rawValue === defaultValue ? "default" : "filled";
|
||||
return {
|
||||
state,
|
||||
sensitive,
|
||||
placeholder,
|
||||
...(sensitive ? {} : { value: rawValue, defaultValue }),
|
||||
};
|
||||
};
|
||||
const controls = [];
|
||||
const collect = (doc) => {
|
||||
for (const el of Array.from(doc.querySelectorAll(controlSelector))) {
|
||||
if (remaining <= 0) break;
|
||||
// Deep serialization supplies the node's backend id. Keep the
|
||||
// state as JSON so decoding needs no general-purpose V8 deserializer.
|
||||
controls.push([el, JSON.stringify(controlState(el))]);
|
||||
remaining -= 1;
|
||||
}
|
||||
for (const frame of Array.from(doc.querySelectorAll("iframe"))) {
|
||||
if (remaining <= 0) break;
|
||||
let childDoc = null;
|
||||
try { childDoc = frame.contentDocument; } catch { childDoc = null; }
|
||||
if (childDoc) collect(childDoc);
|
||||
}
|
||||
};
|
||||
collect(document);
|
||||
return controls;
|
||||
})()`;
|
||||
}
|
||||
|
||||
function formStatesByBackendId(result: RuntimeEvaluateReply): Map<number, CapturedFormState> {
|
||||
const states = new Map<number, CapturedFormState>();
|
||||
const serialized = result.result?.deepSerializedValue;
|
||||
if (serialized?.type !== "array" || !Array.isArray(serialized.value)) return states;
|
||||
for (const entry of serialized.value as DeepSerializedValue[]) {
|
||||
if (entry?.type !== "array" || !Array.isArray(entry.value)) continue;
|
||||
const [element, json] = entry.value as DeepSerializedValue[];
|
||||
if (element?.type !== "node" || json?.type !== "string" || typeof json.value !== "string") {
|
||||
continue;
|
||||
}
|
||||
const backendNodeId = (element.value as { backendNodeId?: number } | undefined)?.backendNodeId;
|
||||
if (typeof backendNodeId !== "number" || !Number.isSafeInteger(backendNodeId)) continue;
|
||||
try {
|
||||
const state = JSON.parse(json.value) as CapturedFormState | null;
|
||||
if (
|
||||
!state ||
|
||||
!["empty", "filled", "default"].includes(state.state ?? "") ||
|
||||
typeof state.sensitive !== "boolean" ||
|
||||
typeof state.placeholder !== "string" ||
|
||||
(!state.sensitive &&
|
||||
(typeof state.value !== "string" || typeof state.defaultValue !== "string"))
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
states.set(backendNodeId, state);
|
||||
} catch {
|
||||
// A malformed entry must not overwrite the snapshot's own state.
|
||||
}
|
||||
}
|
||||
return states;
|
||||
}
|
||||
|
||||
function applyFormStates(nodes: CapturedNode[], states: Map<number, CapturedFormState>): void {
|
||||
for (const node of nodes) {
|
||||
if (!["input", "textarea", "select"].includes(node.tag)) continue;
|
||||
const state = states.get(node.backendNodeId);
|
||||
if (!state) continue;
|
||||
node.formState = state.state;
|
||||
node.formPlaceholder = state.placeholder ?? "";
|
||||
if (
|
||||
state.sensitive ||
|
||||
(node.tag === "input" && (node.attrs.type ?? "").toLowerCase() === "password")
|
||||
) {
|
||||
delete node.formValue;
|
||||
delete node.formDefaultValue;
|
||||
delete node.attrs.value;
|
||||
} else {
|
||||
node.formDefaultValue = state.defaultValue ?? "";
|
||||
if (state.value !== undefined) node.formValue = state.value;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function enrichFormControlStates(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
frameNodeGroups: CapturedNode[][],
|
||||
signal?: AbortSignal,
|
||||
): Promise<boolean> {
|
||||
const hasControls = frameNodeGroups.some((nodes) =>
|
||||
nodes.some((node) => ["input", "textarea", "select"].includes(node.tag)),
|
||||
);
|
||||
if (!hasControls) return true;
|
||||
const objectGroup = `bsk-vom-forms-${crypto.randomUUID()}`;
|
||||
try {
|
||||
throwIfAborted(signal);
|
||||
const result = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
|
||||
expression: formStateBatchExpression(MAX_FORM_ENRICH_CONTROLS),
|
||||
objectGroup,
|
||||
serializationOptions: {
|
||||
serialization: "deep",
|
||||
maxDepth: 3,
|
||||
additionalParameters: { maxNodeDepth: 0, includeShadowTree: "none" },
|
||||
},
|
||||
});
|
||||
throwIfAborted(signal);
|
||||
// Backend ids are scoped to this capture's CDP target. OOPIF targets
|
||||
// have separate enrichment calls and never share this lookup.
|
||||
const states = formStatesByBackendId(result);
|
||||
for (const nodes of frameNodeGroups) {
|
||||
applyFormStates(nodes, states);
|
||||
}
|
||||
return true;
|
||||
} catch (error) {
|
||||
if (isAbortError(error)) throw error;
|
||||
// Best-effort enrichment. DOMSnapshot/AX data still carries the nodes.
|
||||
return false;
|
||||
} finally {
|
||||
await cdp.send(tabId, "Runtime.releaseObjectGroup", { objectGroup }).catch(() => undefined);
|
||||
throwIfAborted(signal);
|
||||
}
|
||||
}
|
||||
@@ -1,181 +0,0 @@
|
||||
import type { CdpFrame, CdpFrameGraph } from "@/browser-driver/frame-graph";
|
||||
import { resolveFrameProjection } from "../frame-geometry";
|
||||
import { type GeometryProjection, projectRectToViewport } from "../geometry";
|
||||
import { type CdpRunner, cdpRunnerForTarget, sendToCdpTarget } from "../shared";
|
||||
import { type CapturedNode, type CapturedViewModel, captureViewModel } from "./capture";
|
||||
import {
|
||||
buildFrameDocuments,
|
||||
type FrameAxBatch,
|
||||
type FrameDocument,
|
||||
type FrameOwnedAxNode,
|
||||
} from "./frame-document";
|
||||
|
||||
export type FrameAxNode = FrameOwnedAxNode;
|
||||
export type CapturedFrameDocument<T extends FrameAxNode> = FrameDocument<T>;
|
||||
|
||||
function throwIfAborted(signal?: AbortSignal): void {
|
||||
if (signal?.aborted) throw new DOMException("observation aborted", "AbortError");
|
||||
}
|
||||
|
||||
function isAbortError(err: unknown): boolean {
|
||||
return err instanceof Error && err.name === "AbortError";
|
||||
}
|
||||
|
||||
async function discoverFrameGraph(cdp: CdpRunner, tabId: number): Promise<CdpFrameGraph | null> {
|
||||
if (!cdp.getFrameGraph) return null;
|
||||
try {
|
||||
return await cdp.getFrameGraph(tabId);
|
||||
} catch (err) {
|
||||
console.debug("[bsk observation] frame graph capture failed", err);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function transformFrameNodes(
|
||||
nodes: CapturedNode[],
|
||||
projection: GeometryProjection | null,
|
||||
): CapturedNode[] {
|
||||
return nodes.map((node) => ({
|
||||
...node,
|
||||
rect: projection
|
||||
? (() => {
|
||||
const bounds = projectRectToViewport(node.rect, projection);
|
||||
return bounds ? { x: bounds.x, y: bounds.y, w: bounds.width, h: bounds.height } : null;
|
||||
})()
|
||||
: null,
|
||||
}));
|
||||
}
|
||||
|
||||
async function captureMissingFrameDocuments(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
graph: CdpFrameGraph,
|
||||
captured: CapturedViewModel,
|
||||
signal?: AbortSignal,
|
||||
): Promise<void> {
|
||||
if (!captured.frameNodes) captured.frameNodes = new Map();
|
||||
if (!captured.frameOwnerBackendNodeIds) captured.frameOwnerBackendNodeIds = new Map();
|
||||
if (!captured.frameParentIds) captured.frameParentIds = new Map();
|
||||
|
||||
for (const frame of graph.frames) {
|
||||
if (!frame.target.sessionId || captured.frameNodes.has(frame.frameId)) continue;
|
||||
try {
|
||||
const child = await captureViewModel(cdpRunnerForTarget(cdp, frame.target), tabId, {
|
||||
signal,
|
||||
});
|
||||
let projection: GeometryProjection | null = null;
|
||||
try {
|
||||
projection = await resolveFrameProjection(cdp, graph, frame.frameId);
|
||||
} catch (err) {
|
||||
if (isAbortError(err)) throw err;
|
||||
console.debug("[bsk observation] child frame geometry projection failed", {
|
||||
frameId: frame.frameId,
|
||||
err,
|
||||
});
|
||||
}
|
||||
const rootFrameId = child.rootFrameId ?? frame.frameId;
|
||||
const childFrames = child.frameNodes ?? new Map<string, CapturedNode[]>();
|
||||
if (!childFrames.has(rootFrameId)) childFrames.set(rootFrameId, child.nodes);
|
||||
if (rootFrameId !== frame.frameId && !childFrames.has(frame.frameId)) {
|
||||
childFrames.set(
|
||||
frame.frameId,
|
||||
child.nodes.map((node) => ({ ...node, frameId: frame.frameId })),
|
||||
);
|
||||
}
|
||||
for (const [childFrameId, nodes] of childFrames) {
|
||||
captured.frameNodes.set(childFrameId, transformFrameNodes(nodes, projection));
|
||||
}
|
||||
if (frame.ownerBackendNodeId !== undefined) {
|
||||
captured.iframeNodes.set(
|
||||
frame.ownerBackendNodeId,
|
||||
captured.frameNodes.get(frame.frameId) ?? [],
|
||||
);
|
||||
captured.frameOwnerBackendNodeIds.set(frame.frameId, frame.ownerBackendNodeId);
|
||||
}
|
||||
if (frame.parentFrameId) captured.frameParentIds.set(frame.frameId, frame.parentFrameId);
|
||||
for (const [childFrameId, ownerBackendNodeId] of child.frameOwnerBackendNodeIds ?? []) {
|
||||
captured.frameOwnerBackendNodeIds.set(childFrameId, ownerBackendNodeId);
|
||||
}
|
||||
for (const [childFrameId, parentFrameId] of child.frameParentIds ?? []) {
|
||||
captured.frameParentIds.set(childFrameId, parentFrameId);
|
||||
}
|
||||
} catch (err) {
|
||||
if (isAbortError(err)) throw err;
|
||||
console.debug("[bsk observation] child frame DOM capture failed", {
|
||||
frameId: frame.frameId,
|
||||
err,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function captureAxTrees<T extends FrameAxNode>(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
discoveredFrames: CdpFrame[],
|
||||
captured: CapturedViewModel,
|
||||
signal?: AbortSignal,
|
||||
): Promise<FrameAxBatch<T>[]> {
|
||||
let frames = [...discoveredFrames];
|
||||
const capturedFrameIds = [...(captured.frameNodes?.keys() ?? [])];
|
||||
if (frames.length === 0) {
|
||||
const rootFrameId = captured.rootFrameId ?? capturedFrameIds[0] ?? "root";
|
||||
frames = [{ frameId: rootFrameId, target: { tabId } }];
|
||||
}
|
||||
for (const frameId of capturedFrameIds) {
|
||||
if (!frames.some((frame) => frame.frameId === frameId)) {
|
||||
frames.push({ frameId, target: { tabId } });
|
||||
}
|
||||
}
|
||||
|
||||
let firstFailure: unknown;
|
||||
const trees = await Promise.all(
|
||||
frames.map(async (frame): Promise<FrameAxBatch<T>> => {
|
||||
try {
|
||||
throwIfAborted(signal);
|
||||
await sendToCdpTarget(cdp, frame.target, "Accessibility.enable", {});
|
||||
const result = await sendToCdpTarget<{ nodes?: T[] }>(
|
||||
cdp,
|
||||
frame.target,
|
||||
"Accessibility.getFullAXTree",
|
||||
frame.frameId === "root" ? {} : { frameId: frame.frameId },
|
||||
);
|
||||
throwIfAborted(signal);
|
||||
return {
|
||||
frame,
|
||||
nodes: result.nodes ?? [],
|
||||
};
|
||||
} catch (err) {
|
||||
if (isAbortError(err)) throw err;
|
||||
firstFailure ??= err;
|
||||
console.debug("[bsk observation] frame AX capture failed", {
|
||||
frameId: frame.frameId,
|
||||
err,
|
||||
});
|
||||
return { frame, nodes: [] };
|
||||
}
|
||||
}),
|
||||
);
|
||||
const capturedNodeCount = [...captured.iframeNodes.values()].reduce(
|
||||
(count, nodes) => count + nodes.length,
|
||||
captured.nodes.length,
|
||||
);
|
||||
if (firstFailure && capturedNodeCount === 0 && trees.every((tree) => tree.nodes.length === 0)) {
|
||||
throw firstFailure;
|
||||
}
|
||||
return trees;
|
||||
}
|
||||
|
||||
export async function captureFrameData<T extends FrameAxNode>(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
captured: CapturedViewModel,
|
||||
signal?: AbortSignal,
|
||||
): Promise<CapturedFrameDocument<T>[]> {
|
||||
const graph = await discoverFrameGraph(cdp, tabId);
|
||||
const frames = graph?.frames ?? [];
|
||||
if (graph) await captureMissingFrameDocuments(cdp, tabId, graph, captured, signal);
|
||||
throwIfAborted(signal);
|
||||
const batches = await captureAxTrees<T>(cdp, tabId, frames, captured, signal);
|
||||
return buildFrameDocuments(graph, batches, captured);
|
||||
}
|
||||
@@ -4,7 +4,8 @@ import {
|
||||
type CdpTarget,
|
||||
cdpTargetKey,
|
||||
} from "@/browser-driver/frame-graph";
|
||||
import type { CapturedNode, CapturedViewModel } from "./capture";
|
||||
import { createCaptureCheckpoint } from "./capture-abort";
|
||||
import type { CapturedNode } from "./capture-types";
|
||||
|
||||
export interface FrameOwnedAxNode {
|
||||
nodeId: string;
|
||||
@@ -23,6 +24,15 @@ export interface FrameDocument<T extends FrameOwnedAxNode> extends CdpFrame {
|
||||
contextScopeId: string;
|
||||
axNodes: T[];
|
||||
domNodes: CapturedNode[];
|
||||
excludedBackendNodeIds?: ReadonlySet<number>;
|
||||
}
|
||||
|
||||
/** Explicit snapshot document data used to join DOM and AX ownership. */
|
||||
export interface FrameDomInput {
|
||||
nodes: CapturedNode[];
|
||||
rootFrameId?: string;
|
||||
frameNodes?: ReadonlyMap<string, CapturedNode[]>;
|
||||
frameExcludedBackendNodeIds?: ReadonlyMap<string, ReadonlySet<number>>;
|
||||
}
|
||||
|
||||
interface Ownership {
|
||||
@@ -47,35 +57,25 @@ function targetBackendKey(target: CdpTarget, backendNodeId: number): string {
|
||||
function frameList<T extends FrameOwnedAxNode>(
|
||||
graph: CdpFrameGraph | null,
|
||||
batches: FrameAxBatch<T>[],
|
||||
captured: CapturedViewModel,
|
||||
): CdpFrame[] {
|
||||
const frames = new Map<string, CdpFrame>();
|
||||
for (const frame of graph?.frames ?? []) frames.set(frame.frameId, frame);
|
||||
for (const batch of batches) {
|
||||
if (!frames.has(batch.frame.frameId)) frames.set(batch.frame.frameId, batch.frame);
|
||||
}
|
||||
for (const frameId of captured.frameNodes?.keys() ?? []) {
|
||||
if (!frames.has(frameId))
|
||||
frames.set(frameId, { frameId, target: { tabId: batches[0]?.frame.target.tabId ?? 0 } });
|
||||
}
|
||||
return [...frames.values()].map((frame) => {
|
||||
const ownerBackendNodeId =
|
||||
frame.ownerBackendNodeId ?? captured.frameOwnerBackendNodeIds?.get(frame.frameId);
|
||||
const parentFrameId = frame.parentFrameId ?? captured.frameParentIds?.get(frame.frameId);
|
||||
return {
|
||||
...frame,
|
||||
...(ownerBackendNodeId !== undefined ? { ownerBackendNodeId } : {}),
|
||||
...(parentFrameId ? { parentFrameId } : {}),
|
||||
};
|
||||
});
|
||||
return [...frames.values()];
|
||||
}
|
||||
|
||||
export function buildFrameDocuments<T extends FrameOwnedAxNode>(
|
||||
export async function buildFrameDocuments<T extends FrameOwnedAxNode>(
|
||||
graph: CdpFrameGraph | null,
|
||||
batches: FrameAxBatch<T>[],
|
||||
captured: CapturedViewModel,
|
||||
): FrameDocument<T>[] {
|
||||
const frames = frameList(graph, batches, captured);
|
||||
captured: FrameDomInput,
|
||||
signal?: AbortSignal,
|
||||
unresolved?: (frame: CdpFrame) => void,
|
||||
): Promise<FrameDocument<T>[]> {
|
||||
const checkpoint = createCaptureCheckpoint(signal);
|
||||
let work = 0;
|
||||
const frames = frameList(graph, batches);
|
||||
const frameById = new Map(frames.map((frame) => [frame.frameId, frame]));
|
||||
const rootFrameId = graph?.rootFrameId ?? captured.rootFrameId ?? frames[0]?.frameId;
|
||||
const domNodesForFrame = (frameId: string): CapturedNode[] =>
|
||||
@@ -83,54 +83,80 @@ export function buildFrameDocuments<T extends FrameOwnedAxNode>(
|
||||
const backendOwner = new Map<string, string>();
|
||||
for (const frame of frames) {
|
||||
for (const node of domNodesForFrame(frame.frameId)) {
|
||||
if (work++ % 256 === 0) {
|
||||
const pending = checkpoint();
|
||||
if (pending) await pending;
|
||||
}
|
||||
backendOwner.set(targetBackendKey(frame.target, node.backendNodeId), frame.frameId);
|
||||
}
|
||||
}
|
||||
|
||||
const candidates = new Map<string, OwnedCandidate<T>>();
|
||||
for (const batch of batches) {
|
||||
const nodeById = new Map(batch.nodes.map((node) => [node.nodeId, node]));
|
||||
const ownershipByNodeId = new Map<string, Ownership>();
|
||||
const resolving = new Set<string>();
|
||||
const resolveOwnership = (node: T): Ownership => {
|
||||
const cached = ownershipByNodeId.get(node.nodeId);
|
||||
if (cached) return cached;
|
||||
if (node.frameId && frameById.has(node.frameId)) {
|
||||
const ownership = { frameId: node.frameId, strength: 4 };
|
||||
ownershipByNodeId.set(node.nodeId, ownership);
|
||||
return ownership;
|
||||
}
|
||||
if (typeof node.backendDOMNodeId === "number") {
|
||||
const frameId = backendOwner.get(
|
||||
targetBackendKey(batch.frame.target, node.backendDOMNodeId),
|
||||
);
|
||||
if (frameId) {
|
||||
const ownership = { frameId, strength: 3 };
|
||||
ownershipByNodeId.set(node.nodeId, ownership);
|
||||
return ownership;
|
||||
}
|
||||
}
|
||||
if (node.parentId && !resolving.has(node.nodeId)) {
|
||||
const parent = nodeById.get(node.parentId);
|
||||
if (parent) {
|
||||
resolving.add(node.nodeId);
|
||||
const inherited = resolveOwnership(parent);
|
||||
resolving.delete(node.nodeId);
|
||||
const ownership = {
|
||||
frameId: inherited.frameId,
|
||||
strength: Math.min(2, inherited.strength),
|
||||
};
|
||||
ownershipByNodeId.set(node.nodeId, ownership);
|
||||
return ownership;
|
||||
}
|
||||
}
|
||||
const ownership = { frameId: batch.frame.frameId, strength: 1 };
|
||||
ownershipByNodeId.set(node.nodeId, ownership);
|
||||
return ownership;
|
||||
};
|
||||
|
||||
const nodeById = new Map<string, T>();
|
||||
for (const node of batch.nodes) {
|
||||
const ownership = resolveOwnership(node);
|
||||
if (work++ % 256 === 0) {
|
||||
const pending = checkpoint();
|
||||
if (pending) await pending;
|
||||
}
|
||||
nodeById.set(node.nodeId, node);
|
||||
}
|
||||
let reported = false;
|
||||
const ownershipByNodeId = new Map<string, Ownership>();
|
||||
for (const node of batch.nodes) {
|
||||
const path: T[] = [];
|
||||
const visiting = new Set<string>();
|
||||
let current: T | undefined = node;
|
||||
let ownership: Ownership = { frameId: batch.frame.frameId, strength: 1 };
|
||||
while (current) {
|
||||
if (work++ % 256 === 0) {
|
||||
const pending = checkpoint();
|
||||
if (pending) await pending;
|
||||
}
|
||||
const cached = ownershipByNodeId.get(current.nodeId);
|
||||
if (cached) {
|
||||
ownership = cached;
|
||||
break;
|
||||
}
|
||||
if (visiting.has(current.nodeId)) break;
|
||||
visiting.add(current.nodeId);
|
||||
if (current.frameId) {
|
||||
const frame = frameById.get(current.frameId);
|
||||
// An explicit contradiction must not fall through to a guessed owner.
|
||||
if (!frame || cdpTargetKey(frame.target) !== cdpTargetKey(batch.frame.target)) {
|
||||
if (!reported) {
|
||||
unresolved?.(batch.frame);
|
||||
reported = true;
|
||||
}
|
||||
ownership = { frameId: "", strength: 0 };
|
||||
break;
|
||||
}
|
||||
ownership = { frameId: frame.frameId, strength: 4 };
|
||||
ownershipByNodeId.set(current.nodeId, ownership);
|
||||
break;
|
||||
}
|
||||
const frameId =
|
||||
typeof current.backendDOMNodeId === "number"
|
||||
? backendOwner.get(targetBackendKey(batch.frame.target, current.backendDOMNodeId))
|
||||
: undefined;
|
||||
if (frameId) {
|
||||
ownership = { frameId, strength: 3 };
|
||||
ownershipByNodeId.set(current.nodeId, ownership);
|
||||
break;
|
||||
}
|
||||
path.push(current);
|
||||
current = current.parentId ? nodeById.get(current.parentId) : undefined;
|
||||
}
|
||||
for (let i = path.length - 1; i >= 0; i--) {
|
||||
if (work++ % 256 === 0) {
|
||||
const pending = checkpoint();
|
||||
if (pending) await pending;
|
||||
}
|
||||
ownership = { frameId: ownership.frameId, strength: Math.min(2, ownership.strength) };
|
||||
ownershipByNodeId.set(path[i].nodeId, ownership);
|
||||
}
|
||||
if (!ownershipByNodeId.has(node.nodeId)) ownershipByNodeId.set(node.nodeId, ownership);
|
||||
if (!ownership.frameId) continue;
|
||||
const key = targetNodeKey(batch.frame.target, node.nodeId);
|
||||
const existing = candidates.get(key);
|
||||
if (!existing || ownership.strength > existing.ownership.strength) {
|
||||
@@ -144,6 +170,10 @@ export function buildFrameDocuments<T extends FrameOwnedAxNode>(
|
||||
);
|
||||
const axNodesByFrame = new Map<string, T[]>();
|
||||
for (const candidate of candidates.values()) {
|
||||
if (work++ % 256 === 0) {
|
||||
const pending = checkpoint();
|
||||
if (pending) await pending;
|
||||
}
|
||||
const { node, target, ownership } = candidate;
|
||||
const parentFrameId = node.parentId
|
||||
? ownershipByTargetNode.get(targetNodeKey(target, node.parentId))
|
||||
@@ -169,5 +199,6 @@ export function buildFrameDocuments<T extends FrameOwnedAxNode>(
|
||||
contextScopeId: frame.frameId,
|
||||
axNodes: axNodesByFrame.get(frame.frameId) ?? [],
|
||||
domNodes: domNodesForFrame(frame.frameId),
|
||||
excludedBackendNodeIds: captured.frameExcludedBackendNodeIds?.get(frame.frameId),
|
||||
}));
|
||||
}
|
||||
|
||||
@@ -0,0 +1,318 @@
|
||||
import type { Viewport } from "@browser-skill/vom";
|
||||
import { type CdpFrame, type CdpTarget, cdpTargetKey } from "@/browser-driver/frame-graph";
|
||||
import { type GeometryProjection, projectRectToViewport } from "../geometry";
|
||||
import {
|
||||
type FrameProjectionState,
|
||||
projectSnapshotRect,
|
||||
type SnapshotCoordinates,
|
||||
snapshotCoordinates,
|
||||
snapshotViewportRect,
|
||||
} from "../geometry/coordinate-types";
|
||||
import {
|
||||
GeometryContext,
|
||||
type LayoutMetrics,
|
||||
snapshotLayoutScale,
|
||||
} from "../geometry/frame-context";
|
||||
import {
|
||||
createCaptureCheckpoint,
|
||||
isAbortError as isCaptureAbort,
|
||||
throwIfAborted as throwCaptureAborted,
|
||||
} from "./capture-abort";
|
||||
import { buildDocumentIndex, type CaptureIssue, type DocumentIndex, type NodeFacts } from "./facts";
|
||||
import {
|
||||
decodeDocument,
|
||||
type SnapshotDocument,
|
||||
type SnapshotReply,
|
||||
snapshotFrameId,
|
||||
} from "./snapshot";
|
||||
|
||||
export interface FrameContext {
|
||||
frameId?: string;
|
||||
ownerFrameBackendNodeId: number | null;
|
||||
projection: FrameProjectionState | null;
|
||||
targetProjection?: GeometryProjection | null;
|
||||
target: CdpTarget;
|
||||
coordinates: SnapshotCoordinates | null;
|
||||
}
|
||||
|
||||
export interface NormalizedDocument {
|
||||
nodes: NodeFacts[];
|
||||
index: DocumentIndex;
|
||||
}
|
||||
|
||||
/** Interpret one document using a supplied projection. No live reads or frame scheduling. */
|
||||
export async function normalizeDocument(
|
||||
doc: SnapshotDocument,
|
||||
strings: string[],
|
||||
context: FrameContext,
|
||||
signal?: AbortSignal,
|
||||
): Promise<NormalizedDocument> {
|
||||
const decoded = await decodeDocument(doc, strings, signal);
|
||||
const checkpoint = createCaptureCheckpoint(signal);
|
||||
const nodes: NodeFacts[] = [];
|
||||
for (let i = 0; i < decoded.nodes.length; i++) {
|
||||
if (i % 256 === 0) {
|
||||
const pending = checkpoint();
|
||||
if (pending) await pending;
|
||||
}
|
||||
const node = decoded.nodes[i];
|
||||
const bounds = node.layout?.bounds ?? [];
|
||||
const input = snapshotViewportRect(
|
||||
bounds,
|
||||
{ target: context.target, frameId: context.frameId },
|
||||
context.coordinates,
|
||||
);
|
||||
const local = input?.rect;
|
||||
let rect =
|
||||
input && context.projection?.status === "available"
|
||||
? projectSnapshotRect(input, context.projection.projection)
|
||||
: null;
|
||||
if (rect && context.targetProjection !== undefined)
|
||||
rect = context.targetProjection
|
||||
? projectRectToViewport(
|
||||
{ x: rect.x, y: rect.y, w: rect.width, h: rect.height },
|
||||
context.targetProjection,
|
||||
)
|
||||
: null;
|
||||
const visibility = node.layout?.styles.visibility || "visible";
|
||||
const opacity = node.layout?.styles.opacity || "1";
|
||||
nodes.push({
|
||||
...node,
|
||||
...(context.frameId ? { frameId: context.frameId } : {}),
|
||||
ownerFrameBackendNodeId: context.ownerFrameBackendNodeId,
|
||||
localRect: local ? { x: local.x, y: local.y, w: local.width, h: local.height } : null,
|
||||
rect: rect ? { x: rect.x, y: rect.y, w: rect.width, h: rect.height } : null,
|
||||
rendered:
|
||||
bounds.length >= 4 &&
|
||||
bounds.slice(0, 4).every(Number.isFinite) &&
|
||||
bounds[2] > 0 &&
|
||||
bounds[3] > 0 &&
|
||||
visibility !== "hidden" &&
|
||||
visibility !== "collapse" &&
|
||||
(Number.parseFloat(opacity) || 0) > 0,
|
||||
});
|
||||
}
|
||||
const index = await buildDocumentIndex(nodes, signal);
|
||||
return {
|
||||
nodes: nodes.filter(
|
||||
(node) => !node.tag.startsWith("#") && !index.excludedBackendNodeIds.has(node.backendNodeId),
|
||||
),
|
||||
index,
|
||||
};
|
||||
}
|
||||
|
||||
export interface NormalizedFrameDocument extends NormalizedDocument {
|
||||
frame: CdpFrame;
|
||||
}
|
||||
|
||||
/** One target's snapshot. Source ownership is checked before interpreting any
|
||||
* coordinates. Snapshot and live quad coordinates are deliberately not conflated. */
|
||||
export async function normalizeSnapshot(
|
||||
snapshot: SnapshotReply,
|
||||
target: CdpTarget,
|
||||
frames: readonly CdpFrame[],
|
||||
geometry: GeometryContext,
|
||||
issues: CaptureIssue[],
|
||||
signal?: AbortSignal,
|
||||
rootFrameId?: string,
|
||||
): Promise<NormalizedFrameDocument[]> {
|
||||
const strings = snapshot.strings ?? [];
|
||||
const raw = snapshot.documents ?? [];
|
||||
const ownerIds = new Set<number>();
|
||||
for (const document of raw) {
|
||||
for (const index of document.nodes?.contentDocumentIndex?.index ?? []) {
|
||||
const id = document.nodes?.backendNodeId?.[index];
|
||||
if (id !== undefined) ownerIds.add(id);
|
||||
}
|
||||
}
|
||||
geometry.registerSnapshotOwners(target, ownerIds);
|
||||
const frameById = new Map(frames.map((frame) => [frame.frameId, frame]));
|
||||
const sources = new Map<string, SnapshotDocument>();
|
||||
for (const doc of raw) {
|
||||
const id = snapshotFrameId(doc, strings);
|
||||
if (!id || !frameById.has(id) || sources.has(id)) {
|
||||
issues.push({
|
||||
target,
|
||||
frameId: id,
|
||||
stage: "ownership",
|
||||
reason: "frame-ownership-unresolved",
|
||||
});
|
||||
continue;
|
||||
}
|
||||
sources.set(id, doc);
|
||||
}
|
||||
let metrics: LayoutMetrics = {};
|
||||
try {
|
||||
metrics = await geometry.layoutMetrics(target);
|
||||
} catch (error) {
|
||||
if (isCaptureAbort(error)) throw error;
|
||||
}
|
||||
const viewport: Viewport = {
|
||||
width: metrics.cssLayoutViewport?.clientWidth ?? 0,
|
||||
height: metrics.cssLayoutViewport?.clientHeight ?? 0,
|
||||
};
|
||||
const layoutUnitsPerCssPixel = snapshotLayoutScale(metrics);
|
||||
const viewportAvailable = [viewport.width, viewport.height].every(
|
||||
(size) => Number.isFinite(size) && size > 0,
|
||||
);
|
||||
|
||||
const children = new Map<string, CdpFrame[]>();
|
||||
const pending: CdpFrame[] = [];
|
||||
for (const frame of frames) {
|
||||
if (!frame.parentFrameId || !frameById.has(frame.parentFrameId)) pending.push(frame);
|
||||
else {
|
||||
const siblings = children.get(frame.parentFrameId);
|
||||
if (siblings) siblings.push(frame);
|
||||
else children.set(frame.parentFrameId, [frame]);
|
||||
}
|
||||
}
|
||||
const projections = new Map<string, FrameProjectionState | null>();
|
||||
const result: NormalizedFrameDocument[] = [];
|
||||
let targetProjection: GeometryProjection | null = null;
|
||||
const targetRoot = rootFrameId ? frameById.get(rootFrameId) : undefined;
|
||||
if (target.sessionId && targetRoot) {
|
||||
try {
|
||||
const known = await geometry.frame(targetRoot.frameId);
|
||||
if (known && cdpTargetKey(known.target) === cdpTargetKey(target))
|
||||
targetProjection = await geometry.targetProjection(targetRoot.frameId);
|
||||
} catch (error) {
|
||||
if (isCaptureAbort(error)) throw error;
|
||||
}
|
||||
}
|
||||
const checkpoint = createCaptureCheckpoint(signal);
|
||||
const visited = new Set<string>();
|
||||
const remaining = frames[Symbol.iterator]();
|
||||
for (let cursor = 0; visited.size < frames.length; cursor++) {
|
||||
if (cursor >= pending.length) {
|
||||
let next = remaining.next();
|
||||
while (!next.done && visited.has(next.value.frameId)) next = remaining.next();
|
||||
if (next.done) break;
|
||||
pending.push(next.value);
|
||||
}
|
||||
if (cursor % 256 === 0) {
|
||||
const pending = checkpoint();
|
||||
if (pending) await pending;
|
||||
}
|
||||
const frame = pending[cursor];
|
||||
if (visited.has(frame.frameId)) continue;
|
||||
visited.add(frame.frameId);
|
||||
for (const child of children.get(frame.frameId) ?? []) pending.push(child);
|
||||
const doc = sources.get(frame.frameId);
|
||||
if (!doc) continue;
|
||||
const source = { target, frameId: frame.frameId };
|
||||
const coordinates = snapshotCoordinates(
|
||||
doc,
|
||||
layoutUnitsPerCssPixel,
|
||||
frame.frameId === rootFrameId
|
||||
? { x: metrics.cssLayoutViewport?.pageX, y: metrics.cssLayoutViewport?.pageY }
|
||||
: undefined,
|
||||
);
|
||||
let state: FrameProjectionState | null =
|
||||
frame.frameId === rootFrameId
|
||||
? viewportAvailable && coordinates
|
||||
? {
|
||||
status: "available",
|
||||
projection: {
|
||||
source,
|
||||
geometry: { sourceClips: [], edges: [], topViewport: viewport },
|
||||
},
|
||||
}
|
||||
: { status: "unavailable", source, reason: "snapshot-coordinates-unavailable" }
|
||||
: (projections.get(frame.frameId) ?? null);
|
||||
if (!state && frame.frameId !== rootFrameId)
|
||||
issues.push({
|
||||
target,
|
||||
frameId: frame.frameId,
|
||||
stage: "ownership",
|
||||
reason: "frame-ownership-unresolved",
|
||||
});
|
||||
if (!coordinates && (!state || state.status === "available"))
|
||||
state = { status: "unavailable", source, reason: "snapshot-coordinates-unavailable" };
|
||||
projections.set(frame.frameId, state);
|
||||
const projection = state?.status === "available" ? state.projection : null;
|
||||
if (!projection || (target.sessionId && !targetProjection))
|
||||
issues.push({
|
||||
target,
|
||||
frameId: frame.frameId,
|
||||
stage: "geometry",
|
||||
reason: "geometry-unavailable",
|
||||
...(state && state.status !== "available" ? { projectionIssue: state } : {}),
|
||||
});
|
||||
|
||||
// Only direct siblings share this pool. Keep normalization in its existing
|
||||
// breadth-first order, independent of owner read completion order.
|
||||
const siblings = (children.get(frame.frameId) ?? []).filter((child) =>
|
||||
sources.has(child.frameId),
|
||||
);
|
||||
const edge = projection?.geometry.edges[0];
|
||||
const clips = edge
|
||||
? [edge.destinationQuad, ...(edge.destinationClips ?? [])]
|
||||
: (projection?.geometry.sourceClips ?? []);
|
||||
let next = 0;
|
||||
let failure: unknown;
|
||||
await Promise.all(
|
||||
Array.from({ length: Math.min(4, siblings.length) }, async () => {
|
||||
while (next < siblings.length && !signal?.aborted && !failure) {
|
||||
const child = siblings[next++];
|
||||
const childSource = { target, frameId: child.frameId };
|
||||
if (!projection) {
|
||||
projections.set(
|
||||
child.frameId,
|
||||
state && state.status !== "available"
|
||||
? {
|
||||
status: "blocked",
|
||||
source: childSource,
|
||||
cause: state.status === "blocked" ? state.cause : state,
|
||||
}
|
||||
: null,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
if (child.ownerBackendNodeId === undefined) {
|
||||
projections.set(child.frameId, null);
|
||||
continue;
|
||||
}
|
||||
if (!snapshotCoordinates(sources.get(child.frameId)!, layoutUnitsPerCssPixel)) {
|
||||
projections.set(child.frameId, {
|
||||
status: "unavailable",
|
||||
source: childSource,
|
||||
reason: "snapshot-coordinates-unavailable",
|
||||
});
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
projections.set(
|
||||
child.frameId,
|
||||
await geometry.snapshotProjection(
|
||||
childSource,
|
||||
child.ownerBackendNodeId,
|
||||
clips,
|
||||
viewport,
|
||||
),
|
||||
);
|
||||
} catch (error) {
|
||||
failure ??= error;
|
||||
}
|
||||
}
|
||||
}),
|
||||
);
|
||||
// Join active reads and their cleanup before returning cancellation.
|
||||
if (failure) throw failure;
|
||||
throwCaptureAborted(signal);
|
||||
const normalized = await normalizeDocument(
|
||||
doc,
|
||||
strings,
|
||||
{
|
||||
frameId: frame.frameId,
|
||||
ownerFrameBackendNodeId: frame.ownerBackendNodeId ?? null,
|
||||
projection: state,
|
||||
target,
|
||||
...(target.sessionId ? { targetProjection } : {}),
|
||||
coordinates,
|
||||
},
|
||||
signal,
|
||||
);
|
||||
result.push({ ...normalized, frame });
|
||||
}
|
||||
return result;
|
||||
}
|
||||
@@ -2,7 +2,10 @@ import type { Rect, RenderedRef, VomResult } from "@browser-skill/vom";
|
||||
import type { CdpTarget } from "@/browser-driver/frame-graph";
|
||||
import { readRecordingDocumentIdentity } from "@/shared/recording-document-identity";
|
||||
import type { CapturedSurfaceProbe } from "./capture";
|
||||
import type { CapturedFrameDocument, FrameAxNode } from "./frame-capture";
|
||||
import type {
|
||||
FrameDocument as CapturedFrameDocument,
|
||||
FrameOwnedAxNode as FrameAxNode,
|
||||
} from "./frame-document";
|
||||
|
||||
/** Frame identity needed to resolve an `@eN` that lives in an iframe. */
|
||||
export interface CaptureVomFrame {
|
||||
|
||||
@@ -61,8 +61,9 @@ export function buildSemanticGraph(input: BuildSemanticGraphInput): SemanticGrap
|
||||
dom,
|
||||
sourceOrder: index,
|
||||
excluded:
|
||||
document.frameId === rootFrameId &&
|
||||
input.excludedBackendNodeIds?.has(dom.backendNodeId) === true,
|
||||
document.excludedBackendNodeIds?.has(dom.backendNodeId) === true ||
|
||||
(document.frameId === rootFrameId &&
|
||||
input.excludedBackendNodeIds?.has(dom.backendNodeId) === true),
|
||||
};
|
||||
nodes.set(id, node);
|
||||
frameNodeIds.push(id);
|
||||
@@ -90,9 +91,10 @@ export function buildSemanticGraph(input: BuildSemanticGraphInput): SemanticGrap
|
||||
ax,
|
||||
sourceOrder: document.domNodes.length + index,
|
||||
excluded:
|
||||
document.frameId === rootFrameId &&
|
||||
typeof ax.backendDOMNodeId === "number" &&
|
||||
input.excludedBackendNodeIds?.has(ax.backendDOMNodeId) === true,
|
||||
(document.excludedBackendNodeIds?.has(ax.backendDOMNodeId) === true ||
|
||||
(document.frameId === rootFrameId &&
|
||||
input.excludedBackendNodeIds?.has(ax.backendDOMNodeId) === true)),
|
||||
});
|
||||
frameNodeIds.push(id);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,291 @@
|
||||
import { type CdpFrame, type CdpTarget, cdpTargetKey } from "@/browser-driver/frame-graph";
|
||||
import { createCaptureCheckpoint } from "./capture-abort";
|
||||
import type { CapturedNode } from "./capture-types";
|
||||
import type { DecodedDocument, DecodedNode } from "./facts";
|
||||
export const REQUESTED_STYLES = [
|
||||
"position",
|
||||
"pointer-events",
|
||||
"cursor",
|
||||
"visibility",
|
||||
"opacity",
|
||||
] as const;
|
||||
const STYLE_COL = Object.fromEntries(
|
||||
REQUESTED_STYLES.map((name, index) => [name, index]),
|
||||
) as Record<(typeof REQUESTED_STYLES)[number], number>;
|
||||
/** Sparse array format Chrome uses for infrequently-set per-node fields. */
|
||||
interface SparseArray {
|
||||
index: number[];
|
||||
value: number[];
|
||||
}
|
||||
|
||||
interface RareBooleanData {
|
||||
index: number[];
|
||||
}
|
||||
|
||||
export interface SnapshotDocument {
|
||||
frameId?: string | number;
|
||||
documentURL?: number;
|
||||
scrollOffsetX?: number;
|
||||
scrollOffsetY?: number;
|
||||
nodes?: {
|
||||
parentIndex?: number[];
|
||||
nodeName?: number[];
|
||||
nodeType?: number[];
|
||||
backendNodeId?: number[];
|
||||
attributes?: number[][];
|
||||
/**
|
||||
* Per-node text value (index into strings), set for `#text` / CDATA nodes.
|
||||
* Element nodes carry -1. Same length as `backendNodeId`.
|
||||
*/
|
||||
nodeValue?: number[];
|
||||
/** Maps node array index → index into `documents[]` for frame content. */
|
||||
contentDocumentIndex?: SparseArray;
|
||||
inputValue?: SparseArray;
|
||||
textValue?: SparseArray;
|
||||
inputChecked?: RareBooleanData;
|
||||
optionSelected?: RareBooleanData;
|
||||
};
|
||||
layout?: {
|
||||
nodeIndex?: number[];
|
||||
styles?: number[][];
|
||||
bounds?: number[][];
|
||||
paintOrders?: number[];
|
||||
};
|
||||
}
|
||||
|
||||
export function snapshotFrameId(document: SnapshotDocument, strings: string[]): string | undefined {
|
||||
if (typeof document.frameId === "string") return document.frameId || undefined;
|
||||
if (typeof document.frameId === "number") return str(strings, document.frameId) || undefined;
|
||||
return undefined;
|
||||
}
|
||||
|
||||
export interface SnapshotReply {
|
||||
strings?: string[];
|
||||
documents?: SnapshotDocument[];
|
||||
}
|
||||
|
||||
function isSensitiveFormControl(node: Pick<CapturedNode, "tag" | "attrs">): boolean {
|
||||
return node.tag === "input" && (node.attrs.type ?? "").toLowerCase() === "password";
|
||||
}
|
||||
|
||||
function snapshotFormState(
|
||||
value: string | undefined,
|
||||
defaultValue: string,
|
||||
hasDefaultValue: boolean,
|
||||
sensitive: boolean,
|
||||
): CapturedNode["formState"] {
|
||||
if (sensitive) {
|
||||
if (value === undefined && !hasDefaultValue) return undefined;
|
||||
return (value ?? defaultValue) === "" ? "empty" : "filled";
|
||||
}
|
||||
if (value === undefined) return undefined;
|
||||
if (value === "") return "empty";
|
||||
return value === defaultValue ? "default" : "filled";
|
||||
}
|
||||
|
||||
function str(strings: string[], idx: number | undefined): string {
|
||||
if (idx === undefined || idx < 0) return "";
|
||||
return strings[idx] ?? "";
|
||||
}
|
||||
|
||||
export function sparseIndexMap(sparse: SparseArray | undefined): Map<number, number> {
|
||||
const out = new Map<number, number>();
|
||||
if (!sparse?.index || !sparse.value) return out;
|
||||
for (let i = 0; i < sparse.index.length; i++) {
|
||||
const nodeIndex = sparse.index[i];
|
||||
const docIndex = sparse.value[i];
|
||||
if (nodeIndex !== undefined && docIndex !== undefined) out.set(nodeIndex, docIndex);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
export async function decodeDocument(
|
||||
doc: SnapshotDocument,
|
||||
strings: string[],
|
||||
signal?: AbortSignal,
|
||||
): Promise<DecodedDocument> {
|
||||
const checkpoint = createCaptureCheckpoint(signal);
|
||||
const dn = doc.nodes;
|
||||
const dl = doc.layout;
|
||||
if (!dn?.backendNodeId) {
|
||||
return { nodes: [] };
|
||||
}
|
||||
|
||||
const count = dn.backendNodeId.length;
|
||||
const layoutByNode = new Map<number, number>();
|
||||
for (let i = 0; i < (dl?.nodeIndex?.length ?? 0); i++) {
|
||||
if (i % 256 === 0) {
|
||||
const pending = checkpoint();
|
||||
if (pending) await pending;
|
||||
}
|
||||
layoutByNode.set(dl!.nodeIndex![i], i);
|
||||
}
|
||||
|
||||
const inputValues = sparseIndexMap(dn.inputValue);
|
||||
const textValues = sparseIndexMap(dn.textValue);
|
||||
const checkedInputs = new Set(dn.inputChecked?.index ?? []);
|
||||
const selectedOptions = new Set(dn.optionSelected?.index ?? []);
|
||||
|
||||
// Collect visible text from #text child nodes so element CapturedNodes
|
||||
// carry a textContent value usable as a button/link label fallback.
|
||||
// nodeValue is a parallel array: string index for text nodes, -1 otherwise.
|
||||
const nodeTextContent = new Map<number, string[]>();
|
||||
if (dn.nodeValue) {
|
||||
for (let n = 0; n < count; n++) {
|
||||
if (n % 256 === 0) {
|
||||
const pending = checkpoint();
|
||||
if (pending) await pending;
|
||||
}
|
||||
const nvIdx = dn.nodeValue[n] ?? -1;
|
||||
if (nvIdx < 0) continue;
|
||||
const text = str(strings, nvIdx).trim();
|
||||
if (!text) continue;
|
||||
const parentIdx = dn.parentIndex?.[n] ?? -1;
|
||||
if (parentIdx >= 0) {
|
||||
const existing = nodeTextContent.get(parentIdx);
|
||||
if (existing) existing.push(text);
|
||||
else nodeTextContent.set(parentIdx, [text]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Decode fields without assigning a coordinate projection or semantic policy.
|
||||
const nodes: DecodedNode[] = [];
|
||||
for (let n = 0; n < count; n++) {
|
||||
if (n % 256 === 0) {
|
||||
const pending = checkpoint();
|
||||
if (pending) await pending;
|
||||
}
|
||||
const backendNodeId = dn.backendNodeId[n];
|
||||
const parentIdx = dn.parentIndex?.[n] ?? -1;
|
||||
const parentBackendNodeId = parentIdx >= 0 ? (dn.backendNodeId[parentIdx] ?? null) : null;
|
||||
const tag = str(strings, dn.nodeName?.[n]).toLowerCase();
|
||||
|
||||
const attrs: Record<string, string> = {};
|
||||
const pairs = dn.attributes?.[n] ?? [];
|
||||
for (let a = 0; a + 1 < pairs.length; a += 2) {
|
||||
attrs[str(strings, pairs[a]).toLowerCase()] = str(strings, pairs[a + 1]);
|
||||
}
|
||||
|
||||
const li = layoutByNode.get(n);
|
||||
const styleRow = li === undefined ? [] : (dl?.styles?.[li] ?? []);
|
||||
const styles: Record<string, string> = {};
|
||||
for (const name of REQUESTED_STYLES) styles[name] = str(strings, styleRow[STYLE_COL[name]]);
|
||||
const layout =
|
||||
li === undefined
|
||||
? undefined
|
||||
: {
|
||||
boundsSpace: "snapshot-document-layout" as const,
|
||||
bounds: dl?.bounds?.[li],
|
||||
styles,
|
||||
};
|
||||
|
||||
const textContent = nodeTextContent.get(n)?.join(" ");
|
||||
|
||||
const rawFormValueIndex = tag === "textarea" ? textValues.get(n) : inputValues.get(n);
|
||||
const rawFormValue =
|
||||
rawFormValueIndex !== undefined ? str(strings, rawFormValueIndex) : undefined;
|
||||
const sensitive = isSensitiveFormControl({ tag, attrs });
|
||||
const formDefaultValue = attrs.value ?? "";
|
||||
const formValue = sensitive ? undefined : rawFormValue;
|
||||
const formState = snapshotFormState(
|
||||
rawFormValue,
|
||||
formDefaultValue,
|
||||
Object.prototype.hasOwnProperty.call(attrs, "value"),
|
||||
sensitive,
|
||||
);
|
||||
if (sensitive) delete attrs.value;
|
||||
|
||||
nodes.push({
|
||||
backendNodeId,
|
||||
nodeType: dn.nodeType?.[n],
|
||||
parentBackendNodeId,
|
||||
|
||||
tag,
|
||||
attrs,
|
||||
layout,
|
||||
paintOrder: li === undefined ? 0 : (dl?.paintOrders?.[li] ?? 0),
|
||||
position: styles.position || "static",
|
||||
pointerEvents: styles["pointer-events"] || "auto",
|
||||
cursor: styles.cursor || "auto",
|
||||
|
||||
textContent,
|
||||
...(formValue !== undefined ? { formValue } : {}),
|
||||
...(tag === "input" || tag === "textarea"
|
||||
? {
|
||||
formPlaceholder: attrs.placeholder ?? "",
|
||||
...(!sensitive ? { formDefaultValue } : {}),
|
||||
...(formState ? { formState } : {}),
|
||||
}
|
||||
: {}),
|
||||
...(checkedInputs.has(n) ? { formValue: "true", formState: "filled" } : {}),
|
||||
...(selectedOptions.has(n) ? { formValue: attrs.value ?? textContent ?? "" } : {}),
|
||||
});
|
||||
}
|
||||
return { nodes };
|
||||
}
|
||||
|
||||
/** Snapshot membership and same-target edges are current response evidence.
|
||||
* The earlier graph only supplies compatible metadata and cross-target edges. */
|
||||
export function describeSnapshotFrames(
|
||||
snapshot: SnapshotReply,
|
||||
target: CdpTarget,
|
||||
hintById: ReadonlyMap<string, CdpFrame>,
|
||||
pageUrl?: string,
|
||||
): { frames: CdpFrame[]; ids: Set<string>; rootFrameId?: string } {
|
||||
const raw = snapshot.documents ?? [];
|
||||
const strings = snapshot.strings ?? [];
|
||||
const ids = new Set<string>();
|
||||
const invalid = new Set<string>();
|
||||
const frames = raw.map((doc) => {
|
||||
const frameId = snapshotFrameId(doc, strings);
|
||||
if (!frameId) return undefined;
|
||||
if (ids.has(frameId)) invalid.add(frameId);
|
||||
ids.add(frameId);
|
||||
const hint = hintById.get(frameId);
|
||||
const compatible =
|
||||
hint && cdpTargetKey(hint.target) === cdpTargetKey(target) ? hint : undefined;
|
||||
const url =
|
||||
(doc.documentURL === undefined ? undefined : strings[doc.documentURL]) || compatible?.url;
|
||||
return { frameId, target, ...(url ? { url } : {}) } as CdpFrame;
|
||||
});
|
||||
const root = frames[0];
|
||||
if (root) {
|
||||
const hint = hintById.get(root.frameId);
|
||||
const parent = hint?.parentFrameId ? hintById.get(hint.parentFrameId) : undefined;
|
||||
if (
|
||||
target.sessionId &&
|
||||
hint &&
|
||||
cdpTargetKey(hint.target) === cdpTargetKey(target) &&
|
||||
parent &&
|
||||
cdpTargetKey(parent.target) !== cdpTargetKey(target)
|
||||
) {
|
||||
root.parentFrameId = parent.frameId;
|
||||
root.ownerBackendNodeId = hint.ownerBackendNodeId;
|
||||
}
|
||||
if (!target.sessionId && !root.url && pageUrl) root.url = pageUrl;
|
||||
}
|
||||
const owned = new Set<string>();
|
||||
for (let i = 0; i < raw.length; i++) {
|
||||
const edges = raw[i].nodes?.contentDocumentIndex;
|
||||
const parent = frames[i];
|
||||
if (!parent || !edges) continue;
|
||||
for (let e = 0; e < edges.index.length; e++) {
|
||||
const child = frames[edges.value[e]];
|
||||
const owner = raw[i].nodes?.backendNodeId?.[edges.index[e]];
|
||||
if (!child || owner === undefined) continue;
|
||||
if (child === root || child === parent || owned.has(child.frameId)) {
|
||||
invalid.add(child.frameId);
|
||||
continue;
|
||||
}
|
||||
owned.add(child.frameId);
|
||||
child.parentFrameId = parent.frameId;
|
||||
child.ownerBackendNodeId = owner;
|
||||
}
|
||||
}
|
||||
return {
|
||||
frames: frames.filter((frame): frame is CdpFrame => !!frame && !invalid.has(frame.frameId)),
|
||||
ids,
|
||||
rootFrameId: root?.frameId,
|
||||
};
|
||||
}
|
||||
@@ -26,16 +26,25 @@ export type RpcErrorReason =
|
||||
| "ref_not_found"
|
||||
| "selector_not_found"
|
||||
| "target_not_fillable"
|
||||
| "fill_value_invalid"
|
||||
| "fill_target_changed"
|
||||
| "fill_focus_lost"
|
||||
| "fill_value_mismatch"
|
||||
| "fill_failed"
|
||||
| "target_not_select"
|
||||
| "option_not_found"
|
||||
| "single_select_value_count"
|
||||
| "tab_not_active"
|
||||
| "restricted_tab_url"
|
||||
| "cdp_extension_access_denied"
|
||||
| "borrow_conflict"
|
||||
| "screenshot_capture_failed"
|
||||
| "file_input_probe_failed"
|
||||
| "file_input_not_activated"
|
||||
| "set_file_input_failed"
|
||||
| "upload_mechanism_unsupported"
|
||||
| "file_drop_target_unavailable"
|
||||
| "file_drop_failed"
|
||||
| "download_capture_failed"
|
||||
| "transfer_outcome_unknown"
|
||||
| "transfer_timeout"
|
||||
@@ -523,12 +532,15 @@ export interface UploadFile {
|
||||
staged_path?: string;
|
||||
}
|
||||
|
||||
export type UploadMode = "input" | "drop";
|
||||
|
||||
export interface UploadParams {
|
||||
session_id: string;
|
||||
ref?: string;
|
||||
selector?: string;
|
||||
tab_id?: number;
|
||||
files: UploadFile[];
|
||||
mode?: UploadMode;
|
||||
timeout_ms?: number;
|
||||
}
|
||||
|
||||
|
||||
+37
-146
@@ -116,156 +116,15 @@ navigate navigate-back navigate-forward reload wait-for-navigation wai
|
||||
observe snapshot get-html screenshot console network
|
||||
click hover fill select press evaluate
|
||||
tab list|create|close|select|borrow|return window resize emulate
|
||||
request-help record start|stop
|
||||
upload download request-help record start|stop
|
||||
```
|
||||
|
||||
Required flags that are easy to get wrong:
|
||||
|
||||
### Tabs (require `--session <id>`)
|
||||
|
||||
| Command | Summary |
|
||||
|---------|---------|
|
||||
| `bsk tab list` | List tabs (`--scope user\|agent\|all`, default `all`) |
|
||||
| `bsk tab create` | New tab in Agent Window (`--url`, `--no-active`, `--index`) |
|
||||
| `bsk tab close <tab-id>` | Close an agent tab |
|
||||
| `bsk tab select <tab-id>` | Focus an agent tab |
|
||||
| `bsk tab borrow <tab-id>` | Move a user tab into the Agent Window |
|
||||
| `bsk tab return <tab-id>` | Return a borrowed tab to its original window |
|
||||
|
||||
### Observation (require `--session` unless noted)
|
||||
|
||||
| Command | Summary |
|
||||
|---------|---------|
|
||||
| `bsk snapshot` | First-choice static page understanding: accessibility tree with `@eN` element refs |
|
||||
| `bsk observe` | Semantic VOM observation with bounded perception probes for conditional surfaces |
|
||||
| `bsk get-html` | Raw HTML dump after snapshot is insufficient (high token cost) |
|
||||
| `bsk screenshot` | PNG capture after snapshot is insufficient: full visible tab, or `--ref @eN` to crop to one element (`--out` path optional) |
|
||||
|
||||
### Console & network debugging (read-only; require `--session`)
|
||||
|
||||
| Command | Summary |
|
||||
|---------|---------|
|
||||
| `bsk console` | Buffered page console messages, JS exceptions, and browser log entries (`--include-stack` for stack traces) |
|
||||
| `bsk network` | Buffered network responses (status, method, URL, MIME/resource type) and failures (`net::ERR_*` reason) |
|
||||
|
||||
Both capture from the moment the tab is attached and read a bounded per-tab buffer: `--since <seq>` pages from a cursor (`next_since` in the result), `--limit` (default 50, max 200), `--max-text-chars` (default 1000, max 4096), `--tab-id` to target a non-active tab. Both are strictly read-only — they never intercept or modify traffic, and request/response headers, bodies, and timings are not captured.
|
||||
|
||||
### Navigation
|
||||
|
||||
| Command | Summary |
|
||||
|---------|---------|
|
||||
| `bsk navigate <url>` | Go to URL in agent tab (`--wait-until`, `--timeout`) |
|
||||
| `bsk navigate-back` | History back one step |
|
||||
| `bsk navigate-forward` | History forward one step |
|
||||
| `bsk reload` | Reload current tab (`--hard` bypass cache) |
|
||||
|
||||
(`bsk navigate back` / `bsk navigate forward` are equivalent subcommands.)
|
||||
|
||||
### Interaction
|
||||
|
||||
| Command | Summary |
|
||||
|---------|---------|
|
||||
| `bsk click <ref-or-selector>` | Click element (`--button`, `--click-count`, `--modifiers`) |
|
||||
| `bsk hover <ref-or-selector>` | Move the mouse to an element and wait for hover UI to settle (`--settle`, `--modifiers`) |
|
||||
| `bsk fill <ref-or-selector> --value <text>` | Clear and type into input |
|
||||
| `bsk select <ref-or-selector> --value <v>` | Set `<select>` option(s) by `value` (repeat `--value` for multi-select) |
|
||||
| `bsk press <key>` | Key/combo (`Enter`, `Ctrl+A`, …; optional `--ref` to focus first) |
|
||||
|
||||
### File transfer (require `--session`)
|
||||
|
||||
| Command | Summary |
|
||||
|---------|---------|
|
||||
| `bsk upload <ref-or-selector> --file <path>` | Click one upload trigger and attach an agent-readable local file (`--file` is repeatable) |
|
||||
| `bsk download <ref-or-selector> --out <path>` | Click one download trigger and copy the single completed file to an exact local path (`--overwrite` is opt-in) |
|
||||
|
||||
The agent/harness decides whether a file transfer is appropriate and which local path belongs to the task. Treat upload as disclosure of that file to the current website, and download as accepting website-controlled bytes onto the local filesystem. Use only paths that are necessary for the user's bounded goal.
|
||||
|
||||
BrowserSkill enforces the mechanical boundary: files are staged under a session-scoped opaque transfer, only daemon-minted capabilities reach the extension, upload/download still obey Agent Window tab checks, and transfers are chunk/size bounded. Upload intercepts the native chooser for one transaction, locates the input activated in the resolved target's document, and assigns only the staged file paths. Download uniquely correlates one exact-target browser intent with one Chrome download in either event order, routes it through a daemon-minted relative directory, and lets the daemon validate and import it. Upload staging remains available for a later form submission and is removed when the session ends. Downloads cannot overwrite an existing destination unless `--overwrite` is explicit. BrowserSkill does not inspect file content or decide whether its meaning is sensitive.
|
||||
|
||||
Do not use `request-help` merely because a native file chooser or browser download is involved; try these commands first. For transfer failures, use the structured error instead of retrying blindly:
|
||||
|
||||
- `reason=file_input_not_activated` means the requested click did not activate exactly one `<input type="file">`; the page may use a non-input picker such as `window.showOpenFilePicker()`. Do not retry blindly. If human help is available, call `request-help` and tell the user the exact original local path to choose. The staged daemon path is internal and must not be shown to the user.
|
||||
- `reason=file_input_probe_failed` means BrowserSkill could not safely establish the browser-side upload transaction. Do not repeat the same action; use `request-help` when available.
|
||||
- `reason=set_file_input_failed` means BrowserSkill found the activated file input but Chrome rejected the staged path or assignment. Check the extension's file-URL access permission; otherwise use `request-help`.
|
||||
- `reason=download_capture_failed` means BrowserSkill could not attribute exactly one completed download to the requested target. Do not retry blindly or accept an unrelated browser download; use `request-help` when available.
|
||||
- `effect_state=none` means BrowserSkill confirmed that no file-transfer effect was committed. Follow the accompanying reason; a corrected target or explicit human fallback may be attempted.
|
||||
- `effect_state=unknown` means the browser may already have attached or created the file. Do not repeat the transfer. Observe the page if that can establish the result; otherwise stop and report the uncertainty.
|
||||
- `effect_state=committed` means the browser-side effect occurred even if later completion or cleanup failed. Do not repeat it; continue only after verifying the resulting page/download state.
|
||||
|
||||
If `request-help` returns `outcome="disabled"`, do not retry it. Stop gracefully and report the transfer mechanism that requires human intervention.
|
||||
|
||||
### Scripting & timing
|
||||
|
||||
| Command | Summary |
|
||||
|---------|---------|
|
||||
| `bsk evaluate <expression>` | Run JS in agent tab (see red lines). JS throw → stderr, **exit 0** (RPC success); use `--json` and check `.ok` to detect JS errors |
|
||||
| `bsk wait-for-navigation` | Block until load/DOM idle/etc. (`--wait-until`, `--timeout`) |
|
||||
| `bsk wait-ms <duration>` | Sleep (`500ms`, `2s`, `1m`; **no** `--session`) |
|
||||
|
||||
### Ask the human for help — `bsk request-help`
|
||||
|
||||
When a step needs a human (captcha, login, OTP) or you want the user to
|
||||
confirm an important action, pause and ask:
|
||||
|
||||
bsk request-help --session <id> --prompt "Solve the captcha, then click Done only after the site accepts it" \
|
||||
--title "Captcha required" --target @e7 --target "#submit" --timeout 5m
|
||||
|
||||
- `--prompt` (required): what the user should do.
|
||||
- `--title` (optional): custom title for the overlay panel. When omitted,
|
||||
the extension shows its default localized title.
|
||||
- `--target` (repeatable): a snapshot ref (`@e7`) or CSS selector
|
||||
(`#submit`) to scroll to and flash-highlight. **Strongly recommended** —
|
||||
whenever the prompt refers to a concrete element (a button to click, a
|
||||
field to fill, a checkbox to toggle), pass its `@eN` ref / selector so the
|
||||
user is guided straight to the right spot instead of hunting for it. For
|
||||
interaction scenarios, always include the relevant target(s); reserve a
|
||||
prompt with no `--target` for cases where there is genuinely no specific
|
||||
element to point at (e.g. "wait for the page to finish loading").
|
||||
- `--timeout` (default `5m`): how long to wait.
|
||||
- `--completion-criteria` (optional): JSON success detector. Use it only
|
||||
when there is a concrete post-help success signal, e.g.
|
||||
`{"any":[{"url_contains":"/dashboard"},{"selector_exists":"[data-testid='account-menu']"}],"stable_for_ms":1000}`.
|
||||
|
||||
The target tab is brought to the foreground; the page stays interactive
|
||||
while the agent control mask is hidden. The call blocks until the user
|
||||
explicitly acts, the timeout expires, cancellation arrives, or explicit
|
||||
completion criteria match. Page reloads, SPA route changes, and captcha
|
||||
refreshes do not return control by themselves. The result `outcome` is one of:
|
||||
|
||||
- `continued` — the user finished and clicked Done / return control (treat as confirm).
|
||||
- `cancelled` — the user clicked Cancel (treat as reject/abort).
|
||||
- `timed_out` — nobody acted within the timeout.
|
||||
- `completed` — the explicit `--completion-criteria` matched while the user had control.
|
||||
- `navigated` — deprecated legacy outcome. Do not rely on navigation as a completion signal.
|
||||
|
||||
`note` carries any text the user typed back. `resolved_targets` reports
|
||||
which refs/selectors matched a live element.
|
||||
|
||||
`request-help` does not refresh the page model after the user returns
|
||||
control. After a `continued` or `completed` result, issue a separate
|
||||
observation tool call (usually `bsk snapshot --session <id>`) before using
|
||||
new refs or reasoning about the post-help page state.
|
||||
#### Disabling request-help (unattended mode)
|
||||
Set `BSK_REQUEST_HELP=off` on unattended servers: `bsk request-help` then
|
||||
returns immediately with `outcome="disabled"` (no overlay, no waiting,
|
||||
exit 0). Any other value keeps it enabled. If you get `disabled`, do not
|
||||
retry — complete the task autonomously or stop gracefully.
|
||||
|
||||
### Recording — `bsk record`
|
||||
|
||||
Capture the user's own actions in the Agent Window to a **trace bundle**, for later LLM-driven automation. New CLI builds request **trace v3** (page observations + action chain); older extensions may still return **trace v2** (actions only), which the CLI exports as a single `trace.json` without `states/`.
|
||||
|
||||
```bash
|
||||
bsk record start --browser <instance-id-or-label> \
|
||||
[--url https://…] [--purpose "publish a wiki doc"] \
|
||||
[--max-page-tokens 3000] [--redact-values] \
|
||||
[--output trace]
|
||||
# `--url` is optional; default https://example.com/ when omitted (must be http(s)).
|
||||
# Blocks until the user clicks Finish in the recording panel, then writes:
|
||||
# trace/trace.json — action chain (+ state index when v3)
|
||||
# trace/states/ — v3 only: one `sN.txt` observe snapshot per settled page state
|
||||
|
||||
bsk record stop [--output trace] # terminal fallback if the browser panel is unavailable
|
||||
```text
|
||||
bsk fill <ref> --value <text> bsk select <ref> --value <option-value>
|
||||
bsk screenshot --out <path> bsk emulate --device <preset-id>
|
||||
bsk upload <ref> --file <path> bsk download <ref> --out <path>
|
||||
```
|
||||
|
||||
`select` matches an option's `value` attribute, not its visible label. Device preset ids are
|
||||
@@ -280,11 +139,43 @@ lowercase and hyphenated, such as `iphone-14`.
|
||||
- `record` captures a user's actions for later replay. Read `bsk record start --help` before use,
|
||||
and never record banking, SSO, password-manager, or other sensitive pages.
|
||||
|
||||
## File transfer
|
||||
|
||||
`upload` and `download` stage files through the daemon; the agent never touches browser-internal
|
||||
paths. Treat upload as disclosure to the website, download as accepting website-controlled bytes.
|
||||
|
||||
Upload has two independent mechanisms — choose explicitly, never rely on automatic fallback:
|
||||
|
||||
- **Default (input mode):** for upload buttons, file-input labels, or "upload from computer"
|
||||
actions. The command clicks the target and intercepts the native file chooser.
|
||||
- **`--mode drop`:** for reliably identified attachment-receiving areas — an explicit drop zone,
|
||||
chat composer, email editor, or form attachment area. Do not target page whitespace, generic
|
||||
containers, or areas whose attachment ownership is ambiguous.
|
||||
|
||||
Decision sequence when uploading:
|
||||
|
||||
1. Try input mode (the default).
|
||||
2. If it returns `reason=file_input_not_activated` with `effect_state=none`, re-observe. When a
|
||||
reliable attachment target exists, try `--mode drop` once against that target.
|
||||
3. Otherwise fall back to `request-help`.
|
||||
4. **Never** switch mechanisms or repeat when `effect_state` is `unknown` or `committed` — the
|
||||
browser may already have applied the file.
|
||||
|
||||
A successful drop means Chrome dispatched the native file-drop event; it does not prove the site
|
||||
accepted the attachment. Observe the page once after the command.
|
||||
|
||||
Download default-refuses to overwrite; pass `--overwrite` when replacing an existing file is
|
||||
intended. Read `bsk upload --help` and `bsk download --help` for all flags and error details.
|
||||
|
||||
## Recover without wandering
|
||||
|
||||
- Stale ref: observe again and retry the intended action once.
|
||||
- Unknown tab or session: list current tabs/sessions; never guess identifiers.
|
||||
- Timeout: inspect current page state before deciding whether one longer purposeful wait is useful.
|
||||
- Fill result unconfirmed (`fill_value_mismatch`): observe the field first; the page may have
|
||||
formatted the value. Continue if the visible result satisfies the user's intent. Otherwise correct
|
||||
the remaining difference; do not blindly repeat fill or immediately request human help. For other
|
||||
fill errors, follow the returned hint and inspect current state before retrying.
|
||||
- Unsupported command: continue with available capabilities; suggest updating only when the missing
|
||||
command is necessary.
|
||||
- Unrecoverable failure: report the blocker and stop the session in a finally-style path.
|
||||
|
||||
@@ -73,6 +73,14 @@ where
|
||||
P: Serialize + Send + 'static,
|
||||
R: DeserializeOwned + Send + 'static,
|
||||
{
|
||||
let parent_cancel = stdin_cancellation();
|
||||
if parent_cancel.as_ref().is_some_and(|rx| *rx.borrow()) {
|
||||
return Err(CliError::from_rpc(RpcError {
|
||||
code: ErrorCode::Cancelled,
|
||||
message: "parent closed the cancellation pipe".into(),
|
||||
data: Some(serde_json::json!({"reason": "cancelled_before_dispatch"})),
|
||||
}));
|
||||
}
|
||||
let rpc_id: RpcId = format!("{}-{}", rpc_id_prefix, random_short_id());
|
||||
let mut client = IpcClient::connect(sock.clone()).await?;
|
||||
let rpc_id_for_call = rpc_id.clone();
|
||||
@@ -88,16 +96,16 @@ where
|
||||
let outcome = tokio::select! {
|
||||
biased;
|
||||
res = &mut main_fut => res?,
|
||||
sig = wait_for_sigint() => {
|
||||
sig.context("install SIGINT handler").map_err(CliError::Local)?;
|
||||
debug!(rpc_id = %rpc_id_for_cancel, "SIGINT: forwarding cancel to daemon");
|
||||
sig = wait_for_cancel(parent_cancel) => {
|
||||
sig.context("wait for cancellation").map_err(CliError::Local)?;
|
||||
debug!(rpc_id = %rpc_id_for_cancel, "forwarding cancel to daemon");
|
||||
let _ = send_cancel(&sock, &rpc_id_for_cancel).await;
|
||||
match tokio::time::timeout(CANCEL_RESPONSE_GRACE, &mut main_fut).await {
|
||||
Ok(res) => res?,
|
||||
Err(_) => {
|
||||
debug!(
|
||||
rpc_id = %rpc_id_for_cancel,
|
||||
"SIGINT: cancel grace elapsed; synthesising cancelled error"
|
||||
"cancel grace elapsed; synthesising cancelled error"
|
||||
);
|
||||
return Err(CliError::from_rpc(RpcError {
|
||||
code: ErrorCode::Cancelled,
|
||||
@@ -112,6 +120,46 @@ where
|
||||
outcome.map_err(CliError::from_rpc)
|
||||
}
|
||||
|
||||
/// Release caller-owned staging even after cancellation. This deliberately
|
||||
/// exposes only transfer.release, not a general bypass for business RPCs.
|
||||
/// The entire batch, including connection setup, shares one bounded budget.
|
||||
pub fn release_transfers<'a>(
|
||||
sock: &Path,
|
||||
transfer_ids: impl IntoIterator<Item = &'a str>,
|
||||
) -> Result<(), CliError> {
|
||||
use bsk_protocol::tools::{TransferIdParams, TransferReleaseResult};
|
||||
const CLEANUP_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
let ids: Vec<_> = transfer_ids.into_iter().collect();
|
||||
if ids.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
let rt = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.context("build runtime for transfer cleanup")?;
|
||||
rt.block_on(async {
|
||||
tokio::time::timeout(CLEANUP_TIMEOUT, async {
|
||||
let mut client = IpcClient::connect(sock).await?;
|
||||
for id in ids {
|
||||
client
|
||||
.call::<_, TransferReleaseResult>(
|
||||
"transfer-release",
|
||||
Method::TransferRelease,
|
||||
Some(TransferIdParams {
|
||||
transfer_id: id.to_string(),
|
||||
}),
|
||||
CLEANUP_TIMEOUT,
|
||||
)
|
||||
.await?
|
||||
.map_err(CliError::from_rpc)?;
|
||||
}
|
||||
Ok::<_, CliError>(())
|
||||
})
|
||||
.await
|
||||
.context("transfer cleanup exceeded its total time budget")?
|
||||
})
|
||||
}
|
||||
|
||||
/// Send a `cancel { rpc_id }` frame over a fresh connection so it
|
||||
/// lands on the daemon while the original call is still parked.
|
||||
///
|
||||
@@ -146,8 +194,57 @@ fn random_short_id() -> String {
|
||||
bytes.iter().map(|b| format!("{b:02x}")).collect()
|
||||
}
|
||||
|
||||
/// Wait for either Ctrl-C or process SIGTERM. Returning here means
|
||||
/// the user has asked us to stop.
|
||||
/// Opt-in control pipe for parent processes that cannot send Ctrl-C (Node on
|
||||
/// Windows). One reader survives the short-lived runtimes used by multi-RPC
|
||||
/// commands such as upload. Ordinary CLI stdin and terminal Ctrl-C are unchanged.
|
||||
fn stdin_cancellation() -> Option<tokio::sync::watch::Receiver<bool>> {
|
||||
if std::env::var("BSK_CANCEL_ON_STDIN_CLOSE").as_deref() != Ok("1") {
|
||||
return None;
|
||||
}
|
||||
static CANCEL: std::sync::OnceLock<tokio::sync::watch::Receiver<bool>> =
|
||||
std::sync::OnceLock::new();
|
||||
Some(
|
||||
CANCEL
|
||||
.get_or_init(|| {
|
||||
let (tx, rx) = tokio::sync::watch::channel(false);
|
||||
std::thread::spawn(move || {
|
||||
use std::io::Read;
|
||||
let mut stdin = std::io::stdin().lock();
|
||||
let mut buf = [0; 64];
|
||||
loop {
|
||||
match stdin.read(&mut buf) {
|
||||
Ok(0) => break,
|
||||
Ok(_) => {}
|
||||
Err(err) if err.kind() == std::io::ErrorKind::Interrupted => continue,
|
||||
Err(_) => break,
|
||||
}
|
||||
}
|
||||
let _ = tx.send(true);
|
||||
});
|
||||
rx
|
||||
})
|
||||
.clone(),
|
||||
)
|
||||
}
|
||||
|
||||
async fn wait_for_cancel(
|
||||
parent_cancel: Option<tokio::sync::watch::Receiver<bool>>,
|
||||
) -> anyhow::Result<()> {
|
||||
let parent_closed = async move {
|
||||
match parent_cancel {
|
||||
Some(mut rx) => {
|
||||
let _ = rx.wait_for(|closed| *closed).await;
|
||||
}
|
||||
None => std::future::pending::<()>().await,
|
||||
}
|
||||
};
|
||||
tokio::select! {
|
||||
result = wait_for_sigint() => result,
|
||||
_ = parent_closed => Ok(()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Wait for terminal Ctrl-C.
|
||||
async fn wait_for_sigint() -> anyhow::Result<()> {
|
||||
tokio::signal::ctrl_c().await.context("listen for SIGINT")?;
|
||||
Ok(())
|
||||
|
||||
@@ -9,8 +9,7 @@ use anyhow::Context;
|
||||
use base64::Engine;
|
||||
use bsk_protocol::Method;
|
||||
use bsk_protocol::tools::{
|
||||
DownloadParams, DownloadResult, TransferChunkParams, TransferChunkResult, TransferIdParams,
|
||||
TransferReleaseResult,
|
||||
DownloadParams, DownloadResult, TransferChunkParams, TransferChunkResult,
|
||||
};
|
||||
use clap::Args;
|
||||
use uuid::Uuid;
|
||||
@@ -70,13 +69,7 @@ pub fn dispatch(args: DownloadArgs, format: Format) -> Result<(), CliError> {
|
||||
CliError::Local(anyhow::anyhow!("daemon returned no download transfer id"))
|
||||
})?;
|
||||
let write_result = write_transfer(&info.sock_path, &transfer_id, &args.out, args.overwrite);
|
||||
let _: Result<TransferReleaseResult, CliError> = crate::cli::business_rpc::call(
|
||||
info.sock_path,
|
||||
"transfer-release",
|
||||
Method::TransferRelease,
|
||||
Some(TransferIdParams { transfer_id }),
|
||||
Duration::from_secs(5),
|
||||
);
|
||||
let _ = crate::cli::business_rpc::release_transfers(&info.sock_path, [transfer_id.as_str()]);
|
||||
write_result?;
|
||||
match format {
|
||||
Format::Json => {
|
||||
|
||||
@@ -182,7 +182,7 @@ pub enum Command {
|
||||
/// Set `<select>` option values by `value` attribute.
|
||||
Select(SelectArgs),
|
||||
|
||||
/// Upload one or more local files through a page file chooser.
|
||||
/// Upload files through a page file input or explicit drop target.
|
||||
Upload(UploadArgs),
|
||||
|
||||
/// Capture one browser download and write it to a local path.
|
||||
|
||||
@@ -43,15 +43,24 @@ pub mod reason {
|
||||
pub const REF_NOT_FOUND: &str = "ref_not_found";
|
||||
pub const SELECTOR_NOT_FOUND: &str = "selector_not_found";
|
||||
pub const TARGET_NOT_FILLABLE: &str = "target_not_fillable";
|
||||
pub const FILL_VALUE_INVALID: &str = "fill_value_invalid";
|
||||
pub const FILL_TARGET_CHANGED: &str = "fill_target_changed";
|
||||
pub const FILL_FOCUS_LOST: &str = "fill_focus_lost";
|
||||
pub const FILL_VALUE_MISMATCH: &str = "fill_value_mismatch";
|
||||
pub const FILL_FAILED: &str = "fill_failed";
|
||||
pub const TARGET_NOT_SELECT: &str = "target_not_select";
|
||||
pub const OPTION_NOT_FOUND: &str = "option_not_found";
|
||||
pub const SINGLE_SELECT_VALUE_COUNT: &str = "single_select_value_count";
|
||||
pub const TAB_NOT_ACTIVE: &str = "tab_not_active";
|
||||
pub const CDP_EXTENSION_ACCESS_DENIED: &str = "cdp_extension_access_denied";
|
||||
pub const BORROW_CONFLICT: &str = "borrow_conflict";
|
||||
pub const SCREENSHOT_CAPTURE_FAILED: &str = "screenshot_capture_failed";
|
||||
pub const FILE_INPUT_PROBE_FAILED: &str = "file_input_probe_failed";
|
||||
pub const FILE_INPUT_NOT_ACTIVATED: &str = "file_input_not_activated";
|
||||
pub const SET_FILE_INPUT_FAILED: &str = "set_file_input_failed";
|
||||
pub const UPLOAD_MECHANISM_UNSUPPORTED: &str = "upload_mechanism_unsupported";
|
||||
pub const FILE_DROP_TARGET_UNAVAILABLE: &str = "file_drop_target_unavailable";
|
||||
pub const FILE_DROP_FAILED: &str = "file_drop_failed";
|
||||
pub const DOWNLOAD_CAPTURE_FAILED: &str = "download_capture_failed";
|
||||
pub const TRANSFER_OUTCOME_UNKNOWN: &str = "transfer_outcome_unknown";
|
||||
pub const TRANSFER_TIMEOUT: &str = "transfer_timeout";
|
||||
@@ -203,6 +212,13 @@ pub fn info_for_error(code: ErrorCode, data: Option<&serde_json::Value>) -> Rend
|
||||
return base;
|
||||
};
|
||||
match (code, reason) {
|
||||
(ErrorCode::CdpFailed, reason::CDP_EXTENSION_ACCESS_DENIED) => RenderInfo {
|
||||
summary: "Chrome blocked CDP access to another extension's content in this tab",
|
||||
hint: Some(
|
||||
"a web page can contain a restricted extension frame; disable the conflicting extension and reload, or use `bsk navigate <url>` to leave this page; reconnecting BrowserSkill alone does not remove the restriction",
|
||||
),
|
||||
exit_code: base.exit_code,
|
||||
},
|
||||
(_, reason::TRANSFER_OUTCOME_UNKNOWN) => RenderInfo {
|
||||
summary: "the file transfer outcome could not be confirmed",
|
||||
hint: Some(
|
||||
@@ -244,7 +260,42 @@ pub fn info_for_error(code: ErrorCode, data: Option<&serde_json::Value>) -> Rend
|
||||
(ErrorCode::InvalidParams, reason::TARGET_NOT_FILLABLE) => RenderInfo {
|
||||
summary: "target element is not fillable",
|
||||
hint: Some(
|
||||
"choose an input, textarea, or contenteditable element from the latest snapshot",
|
||||
"observe the page and choose an enabled, editable text input, textarea, or contenteditable; use the appropriate interaction for other control types",
|
||||
),
|
||||
exit_code: base.exit_code,
|
||||
},
|
||||
(ErrorCode::InvalidParams, reason::FILL_VALUE_INVALID) => RenderInfo {
|
||||
summary: "the requested text does not fit the field constraints",
|
||||
hint: Some(
|
||||
"check the field's input type and maxlength; use a compatible value without silently truncating or changing the user's intended data",
|
||||
),
|
||||
exit_code: base.exit_code,
|
||||
},
|
||||
(ErrorCode::CdpFailed, reason::FILL_TARGET_CHANGED) => RenderInfo {
|
||||
summary: "the fill target changed during the action",
|
||||
hint: Some(
|
||||
"observe the current page and field value; if the intended result is missing, use a fresh editable target before retrying",
|
||||
),
|
||||
exit_code: base.exit_code,
|
||||
},
|
||||
(ErrorCode::CdpFailed, reason::FILL_FOCUS_LOST) => RenderInfo {
|
||||
summary: "the page moved focus away from the fill target",
|
||||
hint: Some(
|
||||
"observe the page for a dialog or rerender and resolve it before retrying; fill manages field focus and does not require bringing the browser to the foreground",
|
||||
),
|
||||
exit_code: base.exit_code,
|
||||
},
|
||||
(ErrorCode::CdpFailed, reason::FILL_VALUE_MISMATCH) => RenderInfo {
|
||||
summary: "the fill result could not be confirmed",
|
||||
hint: Some(
|
||||
"observe the field before retrying; the page may have formatted the value. Continue if the visible result satisfies the user's intent; otherwise correct the remaining difference. Do not blindly repeat fill or immediately request human help",
|
||||
),
|
||||
exit_code: base.exit_code,
|
||||
},
|
||||
(ErrorCode::CdpFailed, reason::FILL_FAILED) => RenderInfo {
|
||||
summary: "fill encountered a browser or page-script error",
|
||||
hint: Some(
|
||||
"observe the page before retrying because the field may already have changed; retry only if the intended result is missing and the target is still available",
|
||||
),
|
||||
exit_code: base.exit_code,
|
||||
},
|
||||
@@ -303,7 +354,7 @@ pub fn info_for_error(code: ErrorCode, data: Option<&serde_json::Value>) -> Rend
|
||||
(ErrorCode::Unsupported, reason::FILE_INPUT_NOT_ACTIVATED) => RenderInfo {
|
||||
summary: "the upload trigger did not activate a file input",
|
||||
hint: Some(
|
||||
"the page may use a non-input picker such as showOpenFilePicker(); do not retry blindly — use `bsk request-help` with the exact original local path, or stop if human help is disabled",
|
||||
"the page may use a non-input picker such as showOpenFilePicker(); when effect_state is none, re-observe and use `--mode drop` once only for a reliably identified attachment-receiving area — otherwise use `bsk request-help` with the exact original local path",
|
||||
),
|
||||
exit_code: base.exit_code,
|
||||
},
|
||||
@@ -323,6 +374,30 @@ pub fn info_for_error(code: ErrorCode, data: Option<&serde_json::Value>) -> Rend
|
||||
),
|
||||
exit_code: base.exit_code,
|
||||
},
|
||||
(ErrorCode::Unsupported, reason::UPLOAD_MECHANISM_UNSUPPORTED) => RenderInfo {
|
||||
summary: "the browser does not support native file-drop upload",
|
||||
hint: Some(
|
||||
"do not retry the same drop; use a standard file-input target or `bsk request-help`",
|
||||
),
|
||||
exit_code: base.exit_code,
|
||||
},
|
||||
(
|
||||
ErrorCode::PermissionDenied | ErrorCode::Timeout | ErrorCode::CdpFailed,
|
||||
reason::FILE_DROP_TARGET_UNAVAILABLE,
|
||||
) => RenderInfo {
|
||||
summary: "the file-drop target could not be safely resolved",
|
||||
hint: Some(
|
||||
"rerun observe and select the attachment-receiving editor or drop zone itself; do not guess another target — no file was delivered when effect_state is none",
|
||||
),
|
||||
exit_code: base.exit_code,
|
||||
},
|
||||
(ErrorCode::Timeout | ErrorCode::CdpFailed, reason::FILE_DROP_FAILED) => RenderInfo {
|
||||
summary: "the browser could not complete the native file drop",
|
||||
hint: Some(
|
||||
"do not retry when effect_state is unknown; observe the page first, otherwise use `bsk request-help`",
|
||||
),
|
||||
exit_code: base.exit_code,
|
||||
},
|
||||
(ErrorCode::CdpFailed, reason::DOWNLOAD_CAPTURE_FAILED) => RenderInfo {
|
||||
summary: "the browser download could not be attributed or completed",
|
||||
hint: Some(
|
||||
@@ -415,6 +490,56 @@ mod tests {
|
||||
ErrorCode::NoBrowserConnected,
|
||||
];
|
||||
|
||||
#[test]
|
||||
fn fill_reasons_provide_recovery_without_changing_exit_codes() {
|
||||
for (code, reason, hint_fragment) in [
|
||||
(
|
||||
ErrorCode::InvalidParams,
|
||||
reason::TARGET_NOT_FILLABLE,
|
||||
"enabled, editable",
|
||||
),
|
||||
(
|
||||
ErrorCode::InvalidParams,
|
||||
reason::FILL_VALUE_INVALID,
|
||||
"maxlength",
|
||||
),
|
||||
(
|
||||
ErrorCode::CdpFailed,
|
||||
reason::FILL_TARGET_CHANGED,
|
||||
"fresh editable target",
|
||||
),
|
||||
(
|
||||
ErrorCode::CdpFailed,
|
||||
reason::FILL_FOCUS_LOST,
|
||||
"does not require bringing the browser",
|
||||
),
|
||||
(
|
||||
ErrorCode::CdpFailed,
|
||||
reason::FILL_VALUE_MISMATCH,
|
||||
"Continue if the visible result satisfies",
|
||||
),
|
||||
(
|
||||
ErrorCode::CdpFailed,
|
||||
reason::FILL_FAILED,
|
||||
"may already have changed",
|
||||
),
|
||||
] {
|
||||
let data = serde_json::json!({ "reason": reason });
|
||||
let info = info_for_error(code, Some(&data));
|
||||
assert!(info.hint.unwrap().contains(hint_fragment), "{reason}");
|
||||
assert_eq!(info.exit_code, info_for(code).exit_code);
|
||||
}
|
||||
let mismatch = serde_json::json!({ "reason": reason::FILL_VALUE_MISMATCH });
|
||||
let info = info_for_error(ErrorCode::CdpFailed, Some(&mismatch));
|
||||
assert_eq!(info.summary, "the fill result could not be confirmed");
|
||||
assert!(info.hint.unwrap().contains("Do not blindly repeat fill"));
|
||||
let unknown = serde_json::json!({ "reason": "future_fill_reason" });
|
||||
assert_eq!(
|
||||
info_for_error(ErrorCode::CdpFailed, Some(&unknown)),
|
||||
info_for(ErrorCode::CdpFailed)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn element_not_visible_overrides_permission_denied_copy() {
|
||||
let data = serde_json::json!({ "reason": reason::ELEMENT_NOT_VISIBLE });
|
||||
@@ -427,6 +552,18 @@ mod tests {
|
||||
assert!(!info.summary.contains("sandbox"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extension_access_denied_explains_the_frame_restriction_and_navigation_recovery() {
|
||||
let data = serde_json::json!({ "reason": reason::CDP_EXTENSION_ACCESS_DENIED });
|
||||
let info = info_for_error(ErrorCode::CdpFailed, Some(&data));
|
||||
assert!(info.summary.contains("another extension"));
|
||||
let hint = info.hint.unwrap();
|
||||
assert!(hint.contains("restricted extension frame"));
|
||||
assert!(hint.contains("bsk navigate <url>"));
|
||||
assert!(hint.contains("disable the conflicting extension"));
|
||||
assert_eq!(info.exit_code, 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn session_busy_overrides_timeout_copy() {
|
||||
let data = serde_json::json!({ "reason": reason::SESSION_BUSY });
|
||||
@@ -477,7 +614,10 @@ mod tests {
|
||||
info.summary,
|
||||
"the upload trigger did not activate a file input"
|
||||
);
|
||||
assert!(info.hint.unwrap().contains("exact original local path"));
|
||||
let hint = info.hint.unwrap();
|
||||
assert!(hint.contains("--mode drop"));
|
||||
assert!(hint.contains("reliably identified attachment-receiving area"));
|
||||
assert!(hint.contains("exact original local path"));
|
||||
|
||||
let control = serde_json::json!({ "reason": reason::FILE_INPUT_PROBE_FAILED });
|
||||
let info = info_for_error(ErrorCode::Timeout, Some(&control));
|
||||
@@ -487,6 +627,29 @@ mod tests {
|
||||
);
|
||||
assert!(info.hint.unwrap().contains("do not retry"));
|
||||
|
||||
let unsupported_drop =
|
||||
serde_json::json!({ "reason": reason::UPLOAD_MECHANISM_UNSUPPORTED });
|
||||
let info = info_for_error(ErrorCode::Unsupported, Some(&unsupported_drop));
|
||||
assert_eq!(
|
||||
info.summary,
|
||||
"the browser does not support native file-drop upload"
|
||||
);
|
||||
assert!(info.hint.unwrap().contains("standard file-input"));
|
||||
|
||||
let unavailable_target =
|
||||
serde_json::json!({ "reason": reason::FILE_DROP_TARGET_UNAVAILABLE });
|
||||
let info = info_for_error(ErrorCode::PermissionDenied, Some(&unavailable_target));
|
||||
assert_eq!(
|
||||
info.summary,
|
||||
"the file-drop target could not be safely resolved"
|
||||
);
|
||||
assert!(info.hint.unwrap().contains("effect_state is none"));
|
||||
|
||||
let failed_drop = serde_json::json!({ "reason": reason::FILE_DROP_FAILED });
|
||||
let info = info_for_error(ErrorCode::Timeout, Some(&failed_drop));
|
||||
assert!(info.summary.contains("native file drop"));
|
||||
assert!(info.hint.unwrap().contains("effect_state is unknown"));
|
||||
|
||||
let download = serde_json::json!({ "reason": reason::DOWNLOAD_CAPTURE_FAILED });
|
||||
let info = info_for_error(ErrorCode::CdpFailed, Some(&download));
|
||||
assert!(info.summary.contains("download could not be attributed"));
|
||||
|
||||
@@ -6,9 +6,7 @@ use std::path::{Path, PathBuf};
|
||||
use std::time::{Duration, SystemTime};
|
||||
|
||||
#[cfg(windows)]
|
||||
use std::os::windows::process::CommandExt;
|
||||
#[cfg(windows)]
|
||||
use std::process::{Command, Stdio};
|
||||
mod windows;
|
||||
|
||||
use anyhow::{Context, Result, bail};
|
||||
use clap::Args;
|
||||
@@ -363,7 +361,8 @@ fn install_candidate_with_client(
|
||||
crate::daemon::start::run_stop().context("stop bsk daemon before update")?;
|
||||
}
|
||||
|
||||
let action = replace_binary_for_update(&target, &binary, daemon_was_running)?;
|
||||
let restart_args = daemon_was_running.then(StartArgs::default);
|
||||
let action = replace_binary_for_update(&target, &binary, restart_args.as_ref())?;
|
||||
|
||||
if daemon_was_running && matches!(action, InstallAction::Replaced) {
|
||||
crate::daemon::start::run_start(StartArgs::default())
|
||||
@@ -394,8 +393,8 @@ pub(crate) fn download_candidate_binary(
|
||||
|
||||
/// Daemon-side install: download, verify, and replace the executable at
|
||||
/// `target` (on Windows: stage the replacement next to it). Unlike the
|
||||
/// CLI path this never stops or starts the daemon — the daemon drives
|
||||
/// its own restart once the binary is replaced.
|
||||
/// CLI path this never stops the daemon. On Windows the helper starts
|
||||
/// its replacement after exit; elsewhere the daemon drives its own restart.
|
||||
///
|
||||
/// `target` must be captured *before* any replacement happens: on Linux
|
||||
/// `std::env::current_exe` starts returning a ` (deleted)`-suffixed
|
||||
@@ -403,10 +402,11 @@ pub(crate) fn download_candidate_binary(
|
||||
pub(crate) fn self_install_candidate(
|
||||
candidate: &UpdateCandidate,
|
||||
target: &Path,
|
||||
restart_args: &StartArgs,
|
||||
) -> Result<InstallAction> {
|
||||
let client = update_http_client(ARCHIVE_FETCH_TIMEOUT)?;
|
||||
let binary = download_candidate_binary(candidate, &client)?;
|
||||
replace_binary_at_path(target, &binary)
|
||||
replace_binary_for_update(target, &binary, Some(restart_args))
|
||||
}
|
||||
|
||||
/// Outcome of one daemon auto-update step (see [`auto_update_step`]).
|
||||
@@ -423,8 +423,8 @@ pub(crate) enum AutoUpdateOutcome {
|
||||
/// The new binary replaced the old one; the daemon should now
|
||||
/// restart into it.
|
||||
Replaced { latest: String },
|
||||
/// Windows staged the replacement next to the running binary; a
|
||||
/// daemon/terminal restart finishes it.
|
||||
/// Windows started a helper that will replace the executable and
|
||||
/// restart the daemon once the current process exits.
|
||||
Staged { latest: String },
|
||||
}
|
||||
|
||||
@@ -668,6 +668,7 @@ fn render_report(format: Format, report: &UpdateReport) -> Result<()> {
|
||||
}
|
||||
if matches!(report.install_action, Some("staged")) {
|
||||
println!("the detached update helper will apply the replacement after exit");
|
||||
println!("if it fails, see the *.update-*.log file next to the executable");
|
||||
}
|
||||
}
|
||||
Format::Json => {
|
||||
@@ -688,22 +689,22 @@ pub fn extract_bsk_binary(archive_bytes: &[u8], kind: ArchiveKind) -> Result<Vec
|
||||
}
|
||||
|
||||
pub fn replace_binary_at_path(target: &Path, binary: &[u8]) -> Result<InstallAction> {
|
||||
replace_binary_for_update(target, binary, false)
|
||||
replace_binary_for_update(target, binary, None)
|
||||
}
|
||||
|
||||
fn replace_binary_for_update(
|
||||
target: &Path,
|
||||
binary: &[u8],
|
||||
restart_daemon: bool,
|
||||
restart_args: Option<&StartArgs>,
|
||||
) -> Result<InstallAction> {
|
||||
#[cfg(windows)]
|
||||
{
|
||||
stage_windows_replacement(target, binary, restart_daemon)
|
||||
stage_windows_replacement(target, binary, restart_args)
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
{
|
||||
let _ = restart_daemon;
|
||||
let _ = restart_args;
|
||||
replace_binary_atomically(target, binary)
|
||||
}
|
||||
}
|
||||
@@ -749,33 +750,69 @@ fn replace_binary_atomically(target: &Path, binary: &[u8]) -> Result<InstallActi
|
||||
fn stage_windows_replacement(
|
||||
target: &Path,
|
||||
binary: &[u8],
|
||||
restart_daemon: bool,
|
||||
restart_args: Option<&StartArgs>,
|
||||
) -> Result<InstallAction> {
|
||||
let paths = staged_replacement_paths(target, std::process::id())?;
|
||||
std::fs::write(&paths.binary_path, binary)
|
||||
.with_context(|| format!("write {}", paths.binary_path.display()))?;
|
||||
std::fs::write(
|
||||
&paths.script_path,
|
||||
windows_replacement_script(target, &paths.binary_path, restart_daemon),
|
||||
)
|
||||
.with_context(|| format!("write {}", paths.script_path.display()))?;
|
||||
// The helper owns replacement/restart after this process exits. Do not
|
||||
// report Staged until it has actually begun executing the script.
|
||||
let _child = spawn_windows_replacement(target, binary, restart_args, 120)?;
|
||||
Ok(InstallAction::Staged)
|
||||
}
|
||||
|
||||
const CREATE_NO_WINDOW: u32 = 0x0800_0000;
|
||||
let command = format!("\"{}\"", paths.script_path.display());
|
||||
if let Err(err) = Command::new("cmd.exe")
|
||||
.args(["/D", "/S", "/C"])
|
||||
.arg(command)
|
||||
.stdin(Stdio::null())
|
||||
.stdout(Stdio::null())
|
||||
.stderr(Stdio::null())
|
||||
.creation_flags(CREATE_NO_WINDOW)
|
||||
.spawn()
|
||||
{
|
||||
#[cfg(windows)]
|
||||
fn spawn_windows_replacement(
|
||||
target: &Path,
|
||||
binary: &[u8],
|
||||
restart_args: Option<&StartArgs>,
|
||||
attempts: u32,
|
||||
) -> Result<windows::Helper> {
|
||||
let paths = staged_replacement_paths(target, std::process::id())?;
|
||||
let ready_path = paths.script_path.with_extension("ready");
|
||||
let log_path = paths.script_path.with_extension("log");
|
||||
let result = (|| {
|
||||
// A previous failed attempt by this process must not acknowledge a
|
||||
// new helper. All paths are private to this target and process id.
|
||||
match std::fs::remove_file(&ready_path) {
|
||||
Ok(()) => {}
|
||||
Err(err) if err.kind() == std::io::ErrorKind::NotFound => {}
|
||||
Err(err) => return Err(err.into()),
|
||||
}
|
||||
std::fs::write(&paths.binary_path, binary)
|
||||
.with_context(|| format!("write {}", paths.binary_path.display()))?;
|
||||
std::fs::write(
|
||||
&paths.script_path,
|
||||
windows_replacement_script(restart_args, attempts),
|
||||
)
|
||||
.with_context(|| format!("write {}", paths.script_path.display()))?;
|
||||
let mut child = windows::spawn(
|
||||
&paths.script_path,
|
||||
&paths.binary_path,
|
||||
target,
|
||||
&ready_path,
|
||||
&log_path,
|
||||
)
|
||||
.context("launch detached Windows update helper")?;
|
||||
let deadline = std::time::Instant::now() + Duration::from_secs(5);
|
||||
loop {
|
||||
if std::fs::remove_file(&ready_path).is_ok() {
|
||||
return Ok(child);
|
||||
}
|
||||
if let Some(status) = child.try_wait()? {
|
||||
bail!("Windows update helper exited before becoming ready: {status}");
|
||||
}
|
||||
if std::time::Instant::now() >= deadline {
|
||||
let _ = child.kill();
|
||||
let _ = child.wait();
|
||||
bail!("Windows update helper did not become ready within 5 seconds");
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(20));
|
||||
}
|
||||
})();
|
||||
if result.is_err() {
|
||||
let _ = std::fs::remove_file(&paths.binary_path);
|
||||
let _ = std::fs::remove_file(&paths.script_path);
|
||||
return Err(err).context("launch detached Windows update helper");
|
||||
let _ = std::fs::remove_file(&ready_path);
|
||||
}
|
||||
Ok(InstallAction::Staged)
|
||||
result.with_context(|| format!("Windows update helper failed; see {}", log_path.display()))
|
||||
}
|
||||
|
||||
pub fn staged_replacement_paths(target: &Path, pid: u32) -> Result<StagedReplacementPaths> {
|
||||
@@ -792,25 +829,43 @@ pub fn staged_replacement_paths(target: &Path, pid: u32) -> Result<StagedReplace
|
||||
}
|
||||
|
||||
#[cfg(any(windows, test))]
|
||||
fn windows_replacement_script(target: &Path, staged_binary: &Path, restart_daemon: bool) -> String {
|
||||
let target = target.display().to_string().replace('%', "%%");
|
||||
let staged_binary = staged_binary.display().to_string().replace('%', "%%");
|
||||
let restart = if restart_daemon {
|
||||
format!("start \"\" /B \"{target}\" daemon start >nul 2>nul\r\n")
|
||||
} else {
|
||||
String::new()
|
||||
};
|
||||
fn windows_replacement_script(restart_args: Option<&StartArgs>, attempts: u32) -> String {
|
||||
// Only fixed switches and numeric values go into the ASCII script. Paths
|
||||
// stay in Unicode environment variables, including %, ! and shell symbols.
|
||||
let restart = restart_args.map_or_else(String::new, |args| {
|
||||
format!(
|
||||
"\"%BSK_UPDATE_TARGET%\" daemon start --port {} --session-idle {}ms --daemon-idle {}ms\r\nif errorlevel 1 goto failed_restart\r\n",
|
||||
args.resolved_port(),
|
||||
args.resolved_session_idle().as_millis(),
|
||||
args.resolved_daemon_idle().as_millis(),
|
||||
)
|
||||
});
|
||||
format!(
|
||||
"@echo off\r\n\
|
||||
setlocal\r\n\
|
||||
setlocal DisableDelayedExpansion\r\n\
|
||||
> \"%BSK_UPDATE_READY%\" echo ready\r\n\
|
||||
if errorlevel 1 exit /b 1\r\n\
|
||||
set /a attempts=0\r\n\
|
||||
:retry\r\n\
|
||||
move /Y \"{staged_binary}\" \"{target}\" >nul 2>nul\r\n\
|
||||
if errorlevel 1 (\r\n\
|
||||
timeout /t 1 /nobreak >nul\r\n\
|
||||
goto retry\r\n\
|
||||
)\r\n\
|
||||
move /Y \"%BSK_UPDATE_SOURCE%\" \"%BSK_UPDATE_TARGET%\" >nul\r\n\
|
||||
if not errorlevel 1 goto replaced\r\n\
|
||||
set /a attempts+=1\r\n\
|
||||
if %attempts% geq {attempts} goto failed_replace\r\n\
|
||||
if not exist \"%BSK_UPDATE_SOURCE%\" goto failed_replace\r\n\
|
||||
ping -n 2 127.0.0.1 >nul\r\n\
|
||||
goto retry\r\n\
|
||||
:failed_replace\r\n\
|
||||
echo Failed to replace the executable after %attempts% attempts.\r\n\
|
||||
goto failed\r\n\
|
||||
:replaced\r\n\
|
||||
{restart}\
|
||||
del /F /Q \"%~f0\" >nul 2>nul\r\n"
|
||||
del /F /Q \"%BSK_UPDATE_LOG%\" >nul 2>nul\r\n\
|
||||
(del /F /Q \"%BSK_UPDATE_SCRIPT%\" >nul 2>nul & exit 0)\r\n\
|
||||
:failed_restart\r\n\
|
||||
echo Replaced the executable but failed to restart the daemon.\r\n\
|
||||
:failed\r\n\
|
||||
del /F /Q \"%BSK_UPDATE_SOURCE%\" >nul 2>nul\r\n\
|
||||
(del /F /Q \"%BSK_UPDATE_SCRIPT%\" >nul 2>nul & exit 1)\r\n"
|
||||
)
|
||||
}
|
||||
|
||||
@@ -964,6 +1019,7 @@ mod tests {
|
||||
assert_eq!(extracted, b"windows binary");
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
#[test]
|
||||
fn replaces_binary_at_path() {
|
||||
let tmp = tempfile::TempDir::new().unwrap();
|
||||
@@ -998,29 +1054,139 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn windows_replacement_script_retries_restarts_and_cleans_itself() {
|
||||
let script = windows_replacement_script(
|
||||
Path::new(r"C:\Program Files\bsk.exe"),
|
||||
Path::new(r"C:\Program Files\bsk.exe.update-42"),
|
||||
true,
|
||||
fn windows_replacement_script_preserves_restart_config() {
|
||||
let args = StartArgs {
|
||||
port: Some(54321),
|
||||
session_idle: Some(Duration::from_millis(1234)),
|
||||
daemon_idle: Some(Duration::from_secs(75)),
|
||||
..Default::default()
|
||||
};
|
||||
let script = windows_replacement_script(Some(&args), 120);
|
||||
assert!(script.is_ascii());
|
||||
assert!(
|
||||
script
|
||||
.contains("daemon start --port 54321 --session-idle 1234ms --daemon-idle 75000ms")
|
||||
);
|
||||
|
||||
assert!(script.contains(":retry"));
|
||||
assert!(script.contains("move /Y"));
|
||||
assert!(script.contains(r#"start "" /B "C:\Program Files\bsk.exe" daemon start"#));
|
||||
assert!(script.contains(r#"del /F /Q "%~f0""#));
|
||||
assert!(!windows_replacement_script(None, 120).contains("daemon start"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn windows_replacement_script_omits_restart_when_daemon_was_not_running() {
|
||||
let script = windows_replacement_script(
|
||||
Path::new(r"C:\bsk.exe"),
|
||||
Path::new(r"C:\bsk.exe.update-42"),
|
||||
false,
|
||||
);
|
||||
#[cfg(windows)]
|
||||
fn wait_for_helper(mut child: windows::Helper) -> std::process::ExitStatus {
|
||||
let deadline = std::time::Instant::now() + Duration::from_secs(10);
|
||||
loop {
|
||||
if let Some(status) = child.try_wait().unwrap() {
|
||||
return status;
|
||||
}
|
||||
if std::time::Instant::now() >= deadline {
|
||||
let _ = child.kill();
|
||||
let _ = child.wait();
|
||||
panic!("Windows update helper did not exit");
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(20));
|
||||
}
|
||||
}
|
||||
|
||||
assert!(!script.contains("daemon start"));
|
||||
assert!(script.contains(r#"del /F /Q "%~f0""#));
|
||||
#[cfg(windows)]
|
||||
#[test]
|
||||
fn windows_helper_replaces_in_unicode_spaces_and_shell_symbol_paths() {
|
||||
let tmp = tempfile::TempDir::new().unwrap();
|
||||
for name in [
|
||||
"ascii",
|
||||
"with space",
|
||||
"中文目录",
|
||||
"literal %PATH% ! & (folder)",
|
||||
] {
|
||||
let dir = tmp.path().join(name);
|
||||
std::fs::create_dir(&dir).unwrap();
|
||||
let target = dir.join("bsk.exe");
|
||||
std::fs::write(&target, b"old binary").unwrap();
|
||||
let child = spawn_windows_replacement(&target, b"new binary", None, 5).unwrap();
|
||||
let paths = staged_replacement_paths(&target, std::process::id()).unwrap();
|
||||
let status = wait_for_helper(child);
|
||||
assert!(
|
||||
status.success(),
|
||||
"{name}: {:?}",
|
||||
std::fs::read_to_string(paths.script_path.with_extension("log"))
|
||||
);
|
||||
assert_eq!(std::fs::read(&target).unwrap(), b"new binary", "{name}");
|
||||
// Successful updates leave no helper, staged binary, ready file or log.
|
||||
assert_eq!(std::fs::read_dir(&dir).unwrap().count(), 1, "{name}");
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
#[test]
|
||||
fn windows_helper_waits_for_unlock_then_replaces() {
|
||||
use std::os::windows::fs::OpenOptionsExt;
|
||||
let tmp = tempfile::TempDir::new().unwrap();
|
||||
let target = tmp.path().join("bsk.exe");
|
||||
std::fs::write(&target, b"old binary").unwrap();
|
||||
// Deny delete sharing, like a running Windows executable.
|
||||
let lock = std::fs::OpenOptions::new()
|
||||
.read(true)
|
||||
.share_mode(1)
|
||||
.open(&target)
|
||||
.unwrap();
|
||||
let mut child = spawn_windows_replacement(&target, b"new binary", None, 5).unwrap();
|
||||
std::thread::sleep(Duration::from_millis(150));
|
||||
assert!(child.try_wait().unwrap().is_none());
|
||||
assert_eq!(std::fs::read(&target).unwrap(), b"old binary");
|
||||
drop(lock);
|
||||
assert!(wait_for_helper(child).success());
|
||||
assert_eq!(std::fs::read(&target).unwrap(), b"new binary");
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
#[test]
|
||||
fn windows_helper_bounds_retries_and_preserves_old_binary_on_failure() {
|
||||
use std::os::windows::fs::OpenOptionsExt;
|
||||
let tmp = tempfile::TempDir::new().unwrap();
|
||||
let target = tmp.path().join("bsk.exe");
|
||||
std::fs::write(&target, b"old binary").unwrap();
|
||||
let _lock = std::fs::OpenOptions::new()
|
||||
.read(true)
|
||||
.share_mode(1)
|
||||
.open(&target)
|
||||
.unwrap();
|
||||
let child = spawn_windows_replacement(&target, b"new binary", None, 2).unwrap();
|
||||
assert!(!wait_for_helper(child).success());
|
||||
let paths = staged_replacement_paths(&target, std::process::id()).unwrap();
|
||||
assert_eq!(std::fs::read(&target).unwrap(), b"old binary");
|
||||
assert!(!paths.binary_path.exists());
|
||||
assert!(!paths.script_path.exists());
|
||||
let log_bytes = std::fs::read(paths.script_path.with_extension("log")).unwrap();
|
||||
let log = String::from_utf8_lossy(&log_bytes);
|
||||
assert!(
|
||||
log.contains("Failed to replace the executable after 2 attempts."),
|
||||
"{log}"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
#[test]
|
||||
fn windows_helper_reports_restart_failure_without_losing_replacement() {
|
||||
let tmp = tempfile::TempDir::new().unwrap();
|
||||
let target = tmp.path().join("bsk.exe");
|
||||
std::fs::write(&target, b"old binary").unwrap();
|
||||
// Not a valid PE executable: replacement succeeds but restart fails.
|
||||
let child = spawn_windows_replacement(
|
||||
&target,
|
||||
b"invalid executable",
|
||||
Some(&StartArgs::default()),
|
||||
2,
|
||||
)
|
||||
.unwrap();
|
||||
assert!(!wait_for_helper(child).success());
|
||||
let paths = staged_replacement_paths(&target, std::process::id()).unwrap();
|
||||
let bytes = std::fs::read(paths.script_path.with_extension("log")).unwrap();
|
||||
let log = String::from_utf8_lossy(&bytes);
|
||||
assert!(
|
||||
log.contains("Replaced the executable but failed to restart the daemon."),
|
||||
"{log}"
|
||||
);
|
||||
assert_eq!(std::fs::read(&target).unwrap(), b"invalid executable");
|
||||
assert!(!paths.script_path.exists());
|
||||
assert!(!paths.binary_path.exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1279,10 +1445,9 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn auto_update_step_staged_outcome_requests_no_restart() {
|
||||
// The Windows shape: the replacement is staged next to the
|
||||
// running binary, so the outcome must not ask for the immediate
|
||||
// self-restart that `Replaced` triggers.
|
||||
fn auto_update_step_reports_staged_handoff() {
|
||||
// The helper, rather than this process, starts the new executable.
|
||||
// The daemon must exit to let that staged handoff finish.
|
||||
let candidate = test_candidate();
|
||||
let outcome =
|
||||
auto_update_step(Some(&candidate), true, 0, |_| Ok(InstallAction::Staged)).unwrap();
|
||||
|
||||
@@ -0,0 +1,203 @@
|
||||
//! Launch the update helper with only its stdio handles inherited.
|
||||
//!
|
||||
//! std::process::Command on Windows also inherits other inheritable handles.
|
||||
//! A daemon's listening socket must not survive in the helper: it would keep
|
||||
//! the port occupied after the daemon exits and prevent its replacement starting.
|
||||
|
||||
use std::ffi::OsStr;
|
||||
use std::fs::File;
|
||||
use std::io;
|
||||
use std::os::windows::ffi::OsStrExt;
|
||||
use std::os::windows::io::{AsRawHandle, FromRawHandle, OwnedHandle};
|
||||
use std::os::windows::process::ExitStatusExt;
|
||||
use std::path::Path;
|
||||
use std::process::ExitStatus;
|
||||
|
||||
use windows_sys::Win32::Foundation::{
|
||||
HANDLE, HANDLE_FLAG_INHERIT, SetHandleInformation, WAIT_OBJECT_0, WAIT_TIMEOUT,
|
||||
};
|
||||
use windows_sys::Win32::System::Threading::{
|
||||
CREATE_NEW_PROCESS_GROUP, CREATE_NO_WINDOW, CREATE_UNICODE_ENVIRONMENT, CreateProcessW,
|
||||
DeleteProcThreadAttributeList, EXTENDED_STARTUPINFO_PRESENT, GetExitCodeProcess,
|
||||
InitializeProcThreadAttributeList, PROC_THREAD_ATTRIBUTE_HANDLE_LIST, PROCESS_INFORMATION,
|
||||
STARTF_USESTDHANDLES, STARTUPINFOEXW, TerminateProcess, UpdateProcThreadAttribute,
|
||||
WaitForSingleObject,
|
||||
};
|
||||
|
||||
pub(super) struct Helper(OwnedHandle);
|
||||
|
||||
impl Helper {
|
||||
pub(super) fn try_wait(&mut self) -> io::Result<Option<ExitStatus>> {
|
||||
// SAFETY: the owned process handle remains valid for both calls.
|
||||
match unsafe { WaitForSingleObject(self.0.as_raw_handle(), 0) } {
|
||||
WAIT_OBJECT_0 => {
|
||||
let mut code = 0;
|
||||
if unsafe { GetExitCodeProcess(self.0.as_raw_handle(), &mut code) } == 0 {
|
||||
return Err(io::Error::last_os_error());
|
||||
}
|
||||
Ok(Some(ExitStatus::from_raw(code)))
|
||||
}
|
||||
WAIT_TIMEOUT => Ok(None),
|
||||
_ => Err(io::Error::last_os_error()),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn kill(&mut self) -> io::Result<()> {
|
||||
// SAFETY: this handle belongs to the helper we created.
|
||||
if unsafe { TerminateProcess(self.0.as_raw_handle(), 1) } == 0 {
|
||||
return Err(io::Error::last_os_error());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn wait(&mut self) -> io::Result<()> {
|
||||
// Only used after kill(); do not leave a live helper on startup failure.
|
||||
if unsafe { WaitForSingleObject(self.0.as_raw_handle(), 5000) } != WAIT_OBJECT_0 {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::TimedOut,
|
||||
"update helper did not exit",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
// The opaque attribute list needs pointer-aligned storage and explicit teardown.
|
||||
struct AttributeList(Vec<usize>);
|
||||
|
||||
impl Drop for AttributeList {
|
||||
fn drop(&mut self) {
|
||||
// SAFETY: this wrapper is constructed only after successful initialization.
|
||||
unsafe { DeleteProcThreadAttributeList(self.0.as_mut_ptr().cast()) };
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn spawn(
|
||||
script: &Path,
|
||||
source: &Path,
|
||||
target: &Path,
|
||||
ready: &Path,
|
||||
log_path: &Path,
|
||||
) -> io::Result<Helper> {
|
||||
let root = std::env::var_os("SystemRoot")
|
||||
.ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, "SystemRoot is not set"))?;
|
||||
let application = wide(Path::new(&root).join("System32/cmd.exe").as_os_str());
|
||||
// /S /C strips one outer pair of quotes. Environment expansion keeps paths
|
||||
// Unicode and avoids CRT escaping, batch-file encoding, and CALL expansion.
|
||||
let mut command = wide(OsStr::new(
|
||||
"cmd.exe /D /V:OFF /S /C \"\"%BSK_UPDATE_SCRIPT%\"\"",
|
||||
));
|
||||
let overrides = [
|
||||
("BSK_UPDATE_SCRIPT", script.as_os_str()),
|
||||
("BSK_UPDATE_SOURCE", source.as_os_str()),
|
||||
("BSK_UPDATE_TARGET", target.as_os_str()),
|
||||
("BSK_UPDATE_READY", ready.as_os_str()),
|
||||
("BSK_UPDATE_LOG", log_path.as_os_str()),
|
||||
];
|
||||
let mut env: Vec<_> = std::env::vars_os()
|
||||
.filter(|(key, _)| {
|
||||
let key = key.to_string_lossy();
|
||||
!overrides
|
||||
.iter()
|
||||
.any(|(name, _)| key.eq_ignore_ascii_case(name))
|
||||
&& !key.eq_ignore_ascii_case(crate::daemon::start::DAEMONIZED_ENV)
|
||||
&& !key.eq_ignore_ascii_case(crate::daemon::start::DAEMON_REPLACEMENT_WAIT_ENV)
|
||||
})
|
||||
.collect();
|
||||
env.extend(
|
||||
overrides
|
||||
.into_iter()
|
||||
.map(|(key, value)| (key.into(), value.to_owned())),
|
||||
);
|
||||
env.sort_by_key(|(key, _)| key.to_string_lossy().to_uppercase());
|
||||
let mut environment = Vec::new();
|
||||
for (key, value) in env {
|
||||
environment.extend(key.encode_wide());
|
||||
environment.push(b'=' as u16);
|
||||
environment.extend(value.encode_wide());
|
||||
environment.push(0);
|
||||
}
|
||||
environment.push(0);
|
||||
|
||||
let input = File::open("NUL")?;
|
||||
let log = File::create(log_path)?;
|
||||
let handles: [HANDLE; 2] = [input.as_raw_handle(), log.as_raw_handle()];
|
||||
for &handle in &handles {
|
||||
// SAFETY: these are dedicated helper stdio handles, held until spawn
|
||||
// finishes. The explicit handle list excludes every other parent handle.
|
||||
if unsafe { SetHandleInformation(handle, HANDLE_FLAG_INHERIT, HANDLE_FLAG_INHERIT) } == 0 {
|
||||
return Err(io::Error::last_os_error());
|
||||
}
|
||||
}
|
||||
|
||||
let mut size = 0;
|
||||
// SAFETY: the first call queries the allocation size, as required by Win32.
|
||||
unsafe { InitializeProcThreadAttributeList(std::ptr::null_mut(), 1, 0, &mut size) };
|
||||
if size == 0 {
|
||||
return Err(io::Error::last_os_error());
|
||||
}
|
||||
let mut storage = vec![0usize; size.div_ceil(std::mem::size_of::<usize>())];
|
||||
// SAFETY: storage has the requested size and pointer alignment.
|
||||
if unsafe { InitializeProcThreadAttributeList(storage.as_mut_ptr().cast(), 1, 0, &mut size) }
|
||||
== 0
|
||||
{
|
||||
return Err(io::Error::last_os_error());
|
||||
}
|
||||
let mut attributes = AttributeList(storage);
|
||||
// SAFETY: the handle array and attribute storage outlive CreateProcessW.
|
||||
if unsafe {
|
||||
UpdateProcThreadAttribute(
|
||||
attributes.0.as_mut_ptr().cast(),
|
||||
0,
|
||||
PROC_THREAD_ATTRIBUTE_HANDLE_LIST as usize,
|
||||
handles.as_ptr().cast(),
|
||||
std::mem::size_of_val(&handles),
|
||||
std::ptr::null_mut(),
|
||||
std::ptr::null(),
|
||||
)
|
||||
} == 0
|
||||
{
|
||||
return Err(io::Error::last_os_error());
|
||||
}
|
||||
// SAFETY: these Win32 structs permit zero initialization; required fields
|
||||
// are populated below before the process is created.
|
||||
let mut startup: STARTUPINFOEXW = unsafe { std::mem::zeroed() };
|
||||
startup.StartupInfo.cb = std::mem::size_of::<STARTUPINFOEXW>() as u32;
|
||||
startup.StartupInfo.dwFlags = STARTF_USESTDHANDLES;
|
||||
startup.StartupInfo.hStdInput = handles[0];
|
||||
startup.StartupInfo.hStdOutput = handles[1];
|
||||
startup.StartupInfo.hStdError = handles[1];
|
||||
startup.lpAttributeList = attributes.0.as_mut_ptr().cast();
|
||||
let mut process: PROCESS_INFORMATION = unsafe { std::mem::zeroed() };
|
||||
// SAFETY: strings are NUL-terminated UTF-16, the environment is double-NUL
|
||||
// terminated, and all buffers/handles remain alive for the duration of spawn.
|
||||
if unsafe {
|
||||
CreateProcessW(
|
||||
application.as_ptr(),
|
||||
command.as_mut_ptr(),
|
||||
std::ptr::null(),
|
||||
std::ptr::null(),
|
||||
1,
|
||||
CREATE_NO_WINDOW
|
||||
| CREATE_NEW_PROCESS_GROUP
|
||||
| CREATE_UNICODE_ENVIRONMENT
|
||||
| EXTENDED_STARTUPINFO_PRESENT,
|
||||
environment.as_ptr().cast(),
|
||||
std::ptr::null(),
|
||||
&startup.StartupInfo,
|
||||
&mut process,
|
||||
)
|
||||
} == 0
|
||||
{
|
||||
return Err(io::Error::last_os_error());
|
||||
}
|
||||
// SAFETY: successful CreateProcessW transfers these two handles to us.
|
||||
let _thread = unsafe { OwnedHandle::from_raw_handle(process.hThread) };
|
||||
Ok(Helper(unsafe {
|
||||
OwnedHandle::from_raw_handle(process.hProcess)
|
||||
}))
|
||||
}
|
||||
|
||||
fn wide(value: &OsStr) -> Vec<u16> {
|
||||
value.encode_wide().chain(Some(0)).collect()
|
||||
}
|
||||
@@ -10,19 +10,34 @@ use base64::Engine;
|
||||
use bsk_protocol::Method;
|
||||
use bsk_protocol::tools::{
|
||||
TransferBeginParams, TransferBeginResult, TransferChunkParams, TransferChunkResult,
|
||||
TransferIdParams, TransferReadyResult, TransferReleaseResult, UploadFile, UploadParams,
|
||||
UploadResult,
|
||||
TransferIdParams, TransferReadyResult, UploadFile, UploadMode, UploadParams, UploadResult,
|
||||
};
|
||||
use clap::Args;
|
||||
use clap::{Args, ValueEnum};
|
||||
|
||||
use crate::cli::ensure_daemon::ensure_daemon;
|
||||
use crate::cli::error::{CliError, Format};
|
||||
use crate::cli::interaction::split_target;
|
||||
use crate::cli::navigate::parse_timeout_ms;
|
||||
|
||||
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, ValueEnum)]
|
||||
pub enum UploadModeArg {
|
||||
#[default]
|
||||
Input,
|
||||
Drop,
|
||||
}
|
||||
|
||||
impl From<UploadModeArg> for UploadMode {
|
||||
fn from(value: UploadModeArg) -> Self {
|
||||
match value {
|
||||
UploadModeArg::Input => Self::Input,
|
||||
UploadModeArg::Drop => Self::Drop,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Args)]
|
||||
pub struct UploadArgs {
|
||||
/// Snapshot ref (`@e3`) or CSS selector for the file input / chooser trigger.
|
||||
/// Snapshot ref (`@e3`) or CSS selector for the chooser trigger / drop target.
|
||||
pub target: Option<String>,
|
||||
#[arg(long = "ref")]
|
||||
pub ref_: Option<String>,
|
||||
@@ -31,6 +46,9 @@ pub struct UploadArgs {
|
||||
/// Local file to upload. Repeat for a multiple-file input.
|
||||
#[arg(long = "file", required = true)]
|
||||
pub files: Vec<PathBuf>,
|
||||
/// Browser mechanism used to deliver the staged files.
|
||||
#[arg(long, value_enum, default_value_t)]
|
||||
pub mode: UploadModeArg,
|
||||
#[arg(long)]
|
||||
pub session: String,
|
||||
#[arg(long = "tab-id")]
|
||||
@@ -50,14 +68,12 @@ pub fn dispatch(args: UploadArgs, format: Format) -> Result<(), CliError> {
|
||||
let (ref_, selector) = split_target(args.target, args.ref_, args.selector)?;
|
||||
let mut staged = Vec::new();
|
||||
for path in &args.files {
|
||||
match stage_file(&info.sock_path, &args.session, path) {
|
||||
Ok(file) => staged.push(file),
|
||||
Err(err) => {
|
||||
for (id, _) in &staged {
|
||||
let _ = release(&info.sock_path, id);
|
||||
}
|
||||
return Err(err);
|
||||
}
|
||||
if let Err(err) = stage_file(&info.sock_path, &args.session, path, &mut staged) {
|
||||
let _ = crate::cli::business_rpc::release_transfers(
|
||||
&info.sock_path,
|
||||
staged.iter().map(|(id, _)| id.as_str()),
|
||||
);
|
||||
return Err(err);
|
||||
}
|
||||
}
|
||||
let params = UploadParams {
|
||||
@@ -73,6 +89,7 @@ pub fn dispatch(args: UploadArgs, format: Format) -> Result<(), CliError> {
|
||||
staged_path: None,
|
||||
})
|
||||
.collect(),
|
||||
mode: args.mode.into(),
|
||||
timeout_ms: Some(args.timeout),
|
||||
};
|
||||
let result = crate::cli::business_rpc::call::<_, UploadResult>(
|
||||
@@ -82,6 +99,17 @@ pub fn dispatch(args: UploadArgs, format: Format) -> Result<(), CliError> {
|
||||
Some(params),
|
||||
ipc_timeout(args.timeout),
|
||||
);
|
||||
// The helper can reject cancellation before handing any request to the
|
||||
// daemon. In that case staging still belongs to this CLI.
|
||||
if result.as_ref().is_err_and(|err| {
|
||||
err.data()
|
||||
.is_some_and(|data| data["reason"] == "cancelled_before_dispatch")
|
||||
}) {
|
||||
let _ = crate::cli::business_rpc::release_transfers(
|
||||
&info.sock_path,
|
||||
staged.iter().map(|(id, _)| id.as_str()),
|
||||
);
|
||||
}
|
||||
// Once tool.upload is dispatched, staging ownership belongs to the
|
||||
// session. A transport timeout cannot prove that Chrome did not attach
|
||||
// the file, so releasing here could invalidate a late successful attach.
|
||||
@@ -98,7 +126,12 @@ pub fn dispatch(args: UploadArgs, format: Format) -> Result<(), CliError> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn stage_file(sock: &Path, session: &str, path: &PathBuf) -> Result<(String, String), CliError> {
|
||||
fn stage_file(
|
||||
sock: &Path,
|
||||
session: &str,
|
||||
path: &PathBuf,
|
||||
staged: &mut Vec<(String, String)>,
|
||||
) -> Result<(), CliError> {
|
||||
let mut file = File::open(path)
|
||||
.with_context(|| format!("open upload file {}", path.display()))
|
||||
.map_err(CliError::Local)?;
|
||||
@@ -129,55 +162,39 @@ fn stage_file(sock: &Path, session: &str, path: &PathBuf) -> Result<(String, Str
|
||||
}),
|
||||
Duration::from_secs(10),
|
||||
)?;
|
||||
let staged = (|| {
|
||||
let mut offset = 0u64;
|
||||
let mut buf = vec![0u8; begin.chunk_size as usize];
|
||||
loop {
|
||||
let n = file.read(&mut buf).map_err(|e| CliError::Local(e.into()))?;
|
||||
if n == 0 {
|
||||
break;
|
||||
}
|
||||
let reply: TransferChunkResult = crate::cli::business_rpc::call(
|
||||
sock.to_path_buf(),
|
||||
"transfer-chunk",
|
||||
Method::TransferChunk,
|
||||
Some(TransferChunkParams {
|
||||
transfer_id: begin.transfer_id.clone(),
|
||||
offset,
|
||||
data_base64: base64::engine::general_purpose::STANDARD.encode(&buf[..n]),
|
||||
}),
|
||||
Duration::from_secs(30),
|
||||
)?;
|
||||
offset = reply.next_offset;
|
||||
// Record ownership immediately so one error path releases both completed
|
||||
// files and the current partially staged file.
|
||||
staged.push((begin.transfer_id.clone(), name));
|
||||
let mut offset = 0u64;
|
||||
let mut buf = vec![0u8; begin.chunk_size as usize];
|
||||
loop {
|
||||
let n = file.read(&mut buf).map_err(|e| CliError::Local(e.into()))?;
|
||||
if n == 0 {
|
||||
break;
|
||||
}
|
||||
let _: TransferReadyResult = crate::cli::business_rpc::call(
|
||||
let reply: TransferChunkResult = crate::cli::business_rpc::call(
|
||||
sock.to_path_buf(),
|
||||
"transfer-finish",
|
||||
Method::TransferFinish,
|
||||
Some(TransferIdParams {
|
||||
"transfer-chunk",
|
||||
Method::TransferChunk,
|
||||
Some(TransferChunkParams {
|
||||
transfer_id: begin.transfer_id.clone(),
|
||||
offset,
|
||||
data_base64: base64::engine::general_purpose::STANDARD.encode(&buf[..n]),
|
||||
}),
|
||||
Duration::from_secs(10),
|
||||
Duration::from_secs(30),
|
||||
)?;
|
||||
Ok::<_, CliError>(())
|
||||
})();
|
||||
if let Err(err) = staged {
|
||||
let _ = release(sock, &begin.transfer_id);
|
||||
return Err(err);
|
||||
offset = reply.next_offset;
|
||||
}
|
||||
Ok((begin.transfer_id, name))
|
||||
}
|
||||
|
||||
fn release(sock: &Path, id: &str) -> Result<TransferReleaseResult, CliError> {
|
||||
crate::cli::business_rpc::call(
|
||||
let _: TransferReadyResult = crate::cli::business_rpc::call(
|
||||
sock.to_path_buf(),
|
||||
"transfer-release",
|
||||
Method::TransferRelease,
|
||||
"transfer-finish",
|
||||
Method::TransferFinish,
|
||||
Some(TransferIdParams {
|
||||
transfer_id: id.to_string(),
|
||||
transfer_id: begin.transfer_id.clone(),
|
||||
}),
|
||||
Duration::from_secs(5),
|
||||
)
|
||||
Duration::from_secs(10),
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn ipc_timeout(timeout_ms: u32) -> Duration {
|
||||
|
||||
@@ -1399,6 +1399,28 @@ mod windows {
|
||||
connected = pipe.connect() => {
|
||||
match connected {
|
||||
Ok(()) => {
|
||||
// Keep the connected instance alive until its replacement
|
||||
// exists. Otherwise a fast handler can close the last
|
||||
// instance and make new clients fail with NotFound.
|
||||
loop {
|
||||
match ServerOptions::new()
|
||||
.access_inbound(true)
|
||||
.access_outbound(true)
|
||||
.create(&listener.pipe_name)
|
||||
{
|
||||
Ok(next) => {
|
||||
listener.first = Some(next);
|
||||
break;
|
||||
}
|
||||
Err(err) => {
|
||||
warn!(?err, "create next named-pipe instance failed");
|
||||
tokio::select! {
|
||||
_ = &mut shutdown => return,
|
||||
_ = tokio::time::sleep(std::time::Duration::from_millis(50)) => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
on_open();
|
||||
let handler = handler.clone();
|
||||
let on_act = on_activity.clone();
|
||||
@@ -1467,6 +1489,118 @@ mod windows {
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::sync::Mutex;
|
||||
use std::time::Duration;
|
||||
use tokio::net::windows::named_pipe::ClientOptions;
|
||||
use tokio::sync::oneshot;
|
||||
|
||||
fn isolated_listener() -> NamedPipeListener {
|
||||
let pipe_name = format!(r"\\.\pipe\bsk-test-{}", uuid::Uuid::new_v4());
|
||||
let first = ServerOptions::new()
|
||||
.first_pipe_instance(true)
|
||||
.create(&pipe_name)
|
||||
.unwrap();
|
||||
NamedPipeListener {
|
||||
pipe_name,
|
||||
first: Some(first),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn next_instance_exists_before_connection_is_handed_off() {
|
||||
let listener = isolated_listener();
|
||||
let name = listener.pipe_name.clone();
|
||||
let probe_name = name.clone();
|
||||
let (probe_tx, probe_rx) = oneshot::channel();
|
||||
let probe_tx = Mutex::new(Some(probe_tx));
|
||||
let (stop_tx, stop_rx) = oneshot::channel();
|
||||
let server = tokio::spawn(serve(
|
||||
listener,
|
||||
crate::daemon::ipc::default_ping_handler(),
|
||||
move || {
|
||||
if let Some(tx) = probe_tx.lock().unwrap().take() {
|
||||
// This callback runs before the connection task can finish.
|
||||
// A second client must already have an instance to open.
|
||||
let result = ClientOptions::new().open(&probe_name).map(drop);
|
||||
let _ = tx.send(result);
|
||||
}
|
||||
},
|
||||
|| {},
|
||||
|| {},
|
||||
async {
|
||||
let _ = stop_rx.await;
|
||||
},
|
||||
));
|
||||
let client = ClientOptions::new().open(&name).unwrap();
|
||||
let result = tokio::time::timeout(Duration::from_secs(5), probe_rx).await;
|
||||
drop(client);
|
||||
let _ = stop_tx.send(());
|
||||
server.await.unwrap();
|
||||
result
|
||||
.expect("accept callback ran")
|
||||
.unwrap()
|
||||
.expect("next pipe instance is ready");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn rapid_disconnects_and_concurrent_rpc_connections() {
|
||||
let listener = isolated_listener();
|
||||
let name = listener.pipe_name.clone();
|
||||
let (stop_tx, stop_rx) = oneshot::channel();
|
||||
let server = tokio::spawn(serve(
|
||||
listener,
|
||||
crate::daemon::ipc::default_ping_handler(),
|
||||
|| {},
|
||||
|| {},
|
||||
|| {},
|
||||
async {
|
||||
let _ = stop_rx.await;
|
||||
},
|
||||
));
|
||||
let exercise = async {
|
||||
// Include clients that close without sending any request.
|
||||
for _ in 0..64 {
|
||||
drop(
|
||||
crate::ipc_client::Client::connect_path(name.clone().into())
|
||||
.await
|
||||
.unwrap(),
|
||||
);
|
||||
}
|
||||
let mut clients = tokio::task::JoinSet::new();
|
||||
for _ in 0..8 {
|
||||
let name = name.clone();
|
||||
clients.spawn(async move {
|
||||
for _ in 0..32 {
|
||||
let mut client =
|
||||
crate::ipc_client::Client::connect_path(name.clone().into())
|
||||
.await
|
||||
.unwrap();
|
||||
let reply: bsk_protocol::PingResult = client
|
||||
.call(
|
||||
bsk_protocol::Method::SystemPing,
|
||||
&serde_json::json!({}),
|
||||
Duration::from_secs(5),
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert!(reply.pong);
|
||||
}
|
||||
});
|
||||
}
|
||||
while let Some(result) = clients.join_next().await {
|
||||
result.unwrap();
|
||||
}
|
||||
};
|
||||
let result = tokio::time::timeout(Duration::from_secs(30), exercise).await;
|
||||
let _ = stop_tx.send(());
|
||||
server.await.unwrap();
|
||||
result.expect("connection exercise completed");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
|
||||
@@ -90,17 +90,21 @@ pub fn pid_alive(pid: u32) -> bool {
|
||||
|
||||
#[cfg(windows)]
|
||||
{
|
||||
use windows_sys::Win32::Foundation::CloseHandle;
|
||||
use windows_sys::Win32::Foundation::{CloseHandle, WAIT_TIMEOUT};
|
||||
use windows_sys::Win32::System::Threading::{
|
||||
OpenProcess, PROCESS_QUERY_LIMITED_INFORMATION,
|
||||
OpenProcess, PROCESS_SYNCHRONIZE, WaitForSingleObject,
|
||||
};
|
||||
unsafe {
|
||||
let handle = OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, 0, pid);
|
||||
let handle = OpenProcess(PROCESS_SYNCHRONIZE, 0, pid);
|
||||
if handle.is_null() {
|
||||
return false;
|
||||
}
|
||||
// An exited process can still be opened while another handle
|
||||
// keeps its kernel object alive. Poll its termination signal;
|
||||
// unlike checking STILL_ACTIVE, this also handles exit code 259.
|
||||
let alive = WaitForSingleObject(handle, 0) == WAIT_TIMEOUT;
|
||||
CloseHandle(handle);
|
||||
true
|
||||
alive
|
||||
}
|
||||
}
|
||||
|
||||
@@ -110,3 +114,39 @@ pub fn pid_alive(pid: u32) -> bool {
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(test, windows))]
|
||||
mod tests {
|
||||
use super::pid_alive;
|
||||
use std::os::windows::process::CommandExt;
|
||||
use std::process::Command;
|
||||
|
||||
#[test]
|
||||
fn current_process_is_alive() {
|
||||
assert!(pid_alive(std::process::id()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_process_is_not_alive() {
|
||||
assert!(!pid_alive(0));
|
||||
assert!(!pid_alive(u32::MAX));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn exited_process_with_retained_handle_is_not_alive() {
|
||||
for exit_code in [0, 259] {
|
||||
let mut child = Command::new("cmd.exe")
|
||||
.args(["/D", "/C", &format!("exit {exit_code}")])
|
||||
.creation_flags(0x0800_0000) // CREATE_NO_WINDOW
|
||||
.spawn()
|
||||
.expect("spawn short-lived process");
|
||||
assert_eq!(child.wait().unwrap().code(), Some(exit_code));
|
||||
// Keep Child (and its process handle) alive during the probe.
|
||||
assert!(
|
||||
!pid_alive(child.id()),
|
||||
"exited process with code {exit_code}"
|
||||
);
|
||||
drop(child);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -232,8 +232,8 @@ pub fn run_foreground(cfg: DaemonConfig) -> Result<()> {
|
||||
let session_idle_task = spawn_session_idle_reaper(Arc::clone(&state));
|
||||
let browser_liveness_task = spawn_browser_liveness_reaper(Arc::clone(&state));
|
||||
// Fired by the update check task after a successful auto-update:
|
||||
// the replacement daemon has already been spawned, so this
|
||||
// process should shut down and let it take over.
|
||||
// the replacement daemon (or Windows update helper) is ready, so
|
||||
// this process should shut down and let it take over.
|
||||
let restart_notify = Arc::new(tokio::sync::Notify::new());
|
||||
let update_check_task =
|
||||
spawn_update_check_task(Arc::clone(&state), Arc::clone(&restart_notify));
|
||||
@@ -539,8 +539,8 @@ pub(crate) fn spawn_session_idle_reaper(state: Arc<DaemonState>) -> tokio::task:
|
||||
/// task spawns the replacement daemon (see
|
||||
/// [`DAEMON_REPLACEMENT_WAIT_ENV`]) and fires `restart` so this process
|
||||
/// shuts down and the new version takes over; on Windows the
|
||||
/// replacement can only be staged, so it just logs that a restart is
|
||||
/// needed.
|
||||
/// helper waits for this process to exit, replaces the binary, and starts
|
||||
/// the new daemon with the same configuration.
|
||||
pub(crate) fn spawn_update_check_task(
|
||||
state: Arc<DaemonState>,
|
||||
restart: Arc<tokio::sync::Notify>,
|
||||
@@ -605,7 +605,11 @@ pub(crate) fn spawn_update_check_task(
|
||||
|candidate| {
|
||||
let target =
|
||||
exe_path.as_deref().context("current executable unknown")?;
|
||||
update::self_install_candidate(candidate, target)
|
||||
update::self_install_candidate(
|
||||
candidate,
|
||||
target,
|
||||
&restart_start_args(&state.config),
|
||||
)
|
||||
},
|
||||
)
|
||||
})
|
||||
@@ -635,10 +639,11 @@ pub(crate) fn spawn_update_check_task(
|
||||
sessions,
|
||||
"auto-update postponed: agent session(s) active; will retry on the next tick"
|
||||
),
|
||||
update::AutoUpdateOutcome::Staged { latest } => warn!(
|
||||
%latest,
|
||||
"auto-update staged the new binary but cannot replace the running daemon in place; restart the daemon (or terminal) to finish the upgrade"
|
||||
),
|
||||
update::AutoUpdateOutcome::Staged { latest } => {
|
||||
info!(%latest, "update helper ready; exiting so it can replace and restart the daemon");
|
||||
restart.notify_one();
|
||||
return;
|
||||
}
|
||||
update::AutoUpdateOutcome::Replaced { latest } => {
|
||||
info!(
|
||||
current = env!("CARGO_PKG_VERSION"),
|
||||
|
||||
@@ -6,6 +6,7 @@ use bsk::cli::daemon::{DaemonCmd, parse_duration};
|
||||
use bsk::cli::navigate::NavigateCmd;
|
||||
use bsk::cli::record::{RecordCmd, RecordSub};
|
||||
use bsk::cli::session::{SessionCmd, SessionSub};
|
||||
use bsk::cli::upload::UploadModeArg;
|
||||
use bsk::{Cli, Command};
|
||||
use clap::Parser;
|
||||
|
||||
@@ -13,6 +14,54 @@ fn parse(args: &[&str]) -> Cli {
|
||||
Cli::try_parse_from(args).expect("clap parse should succeed")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_upload_modes() {
|
||||
let cli = parse(&[
|
||||
"bsk",
|
||||
"upload",
|
||||
"@e1",
|
||||
"--file",
|
||||
"image.png",
|
||||
"--session",
|
||||
"s1",
|
||||
]);
|
||||
let Command::Upload(default_args) = cli.command else {
|
||||
panic!("expected upload subcommand");
|
||||
};
|
||||
assert_eq!(default_args.mode, UploadModeArg::Input);
|
||||
|
||||
let cli = parse(&[
|
||||
"bsk",
|
||||
"upload",
|
||||
"@e1",
|
||||
"--file",
|
||||
"image.png",
|
||||
"--session",
|
||||
"s1",
|
||||
"--mode",
|
||||
"drop",
|
||||
]);
|
||||
let Command::Upload(drop_args) = cli.command else {
|
||||
panic!("expected upload subcommand");
|
||||
};
|
||||
assert_eq!(drop_args.mode, UploadModeArg::Drop);
|
||||
|
||||
assert!(
|
||||
Cli::try_parse_from([
|
||||
"bsk",
|
||||
"upload",
|
||||
"@e1",
|
||||
"--file",
|
||||
"image.png",
|
||||
"--session",
|
||||
"s1",
|
||||
"--mode",
|
||||
"auto",
|
||||
])
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_daemon_start_with_defaults() {
|
||||
let cli = parse(&["bsk", "daemon", "start"]);
|
||||
|
||||
@@ -0,0 +1,342 @@
|
||||
//! Real Windows CLI cancellation, including caller-owned transfer cleanup.
|
||||
#![cfg(windows)]
|
||||
|
||||
use std::path::Path;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use bsk::daemon::{file_transfer::TransferRegistry, info::DaemonInfo};
|
||||
use bsk_protocol::tools::{TransferBeginParams, TransferIdParams};
|
||||
use bsk_protocol::{ErrorCode, Frame, Method, ResponseBody, ResponseFrame, RpcError};
|
||||
use serde_json::json;
|
||||
use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader};
|
||||
use tokio::net::windows::named_pipe::ServerOptions;
|
||||
use tokio::process::Command;
|
||||
use tokio::sync::{Notify, watch};
|
||||
|
||||
fn response<T: serde::Serialize>(result: Result<T, RpcError>) -> ResponseBody {
|
||||
match result {
|
||||
Ok(value) => ResponseBody::Ok(serde_json::to_value(value).unwrap()),
|
||||
Err(err) => ResponseBody::Err(err),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stdin_close_cancels_business_but_releases_resources_with_a_bounded_budget() {
|
||||
// Resolve the production registry's root before starting any runtime threads.
|
||||
let registry_home = tempfile::tempdir().unwrap();
|
||||
let original_home = std::env::var_os("BSK_HOME");
|
||||
unsafe {
|
||||
std::env::set_var("BSK_HOME", registry_home.path());
|
||||
}
|
||||
let transfers = Arc::new(TransferRegistry::new().unwrap());
|
||||
unsafe {
|
||||
match original_home {
|
||||
Some(value) => std::env::set_var("BSK_HOME", value),
|
||||
None => std::env::remove_var("BSK_HOME"),
|
||||
}
|
||||
}
|
||||
transfers.initialize().unwrap();
|
||||
tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.unwrap()
|
||||
.block_on(exercise(
|
||||
transfers,
|
||||
®istry_home.path().join("run/transfers"),
|
||||
));
|
||||
}
|
||||
|
||||
async fn exercise(transfers: Arc<TransferRegistry>, staging_root: &Path) {
|
||||
for mode in [
|
||||
"wait",
|
||||
"upload",
|
||||
"upload-multi",
|
||||
"upload-before-dispatch",
|
||||
"download",
|
||||
"upload-multi-stall",
|
||||
"ordinary",
|
||||
] {
|
||||
let home = tempfile::tempdir().unwrap();
|
||||
let pipe_name = format!(r"\\.\pipe\bsk-parent-cancel-{}", uuid::Uuid::new_v4());
|
||||
let info = DaemonInfo::now(std::process::id(), (&pipe_name).into(), 0, "0.2.0");
|
||||
std::fs::write(
|
||||
home.path().join("daemon.json"),
|
||||
serde_json::to_vec(&info).unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
let mut pipe = ServerOptions::new()
|
||||
.first_pipe_instance(true)
|
||||
.create(&pipe_name)
|
||||
.unwrap();
|
||||
let registered = Arc::new(Notify::new());
|
||||
let pending = Arc::new(Mutex::new(None::<String>));
|
||||
let ids = Arc::new(Mutex::new(Vec::<String>::new()));
|
||||
let released = Arc::new(Mutex::new(Vec::<String>::new()));
|
||||
let received = Arc::new(Mutex::new(Vec::<Method>::new()));
|
||||
let download_id = if mode == "download" {
|
||||
let staging = transfers.begin_download("fixture").unwrap();
|
||||
let browser_dir = home.path().join("BrowserSkill").join(&staging.transfer_id);
|
||||
std::fs::create_dir_all(&browser_dir).unwrap();
|
||||
let file = browser_dir.join("download.txt");
|
||||
std::fs::write(&file, b"download fixture").unwrap();
|
||||
transfers
|
||||
.import_download(&staging.transfer_id, &file)
|
||||
.unwrap();
|
||||
ids.lock().unwrap().push(staging.transfer_id.clone());
|
||||
Some(staging.transfer_id)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let (cancel, cancelled) = watch::channel(false);
|
||||
let ready = Arc::clone(®istered);
|
||||
let name = pipe_name.clone();
|
||||
let registry = Arc::clone(&transfers);
|
||||
let server_ids = Arc::clone(&ids);
|
||||
let server_released = Arc::clone(&released);
|
||||
let server_received = Arc::clone(&received);
|
||||
let server = tokio::spawn(async move {
|
||||
let mut connections = tokio::task::JoinSet::new();
|
||||
loop {
|
||||
pipe.connect().await.unwrap();
|
||||
let next = ServerOptions::new().create(&name).unwrap();
|
||||
let connection = std::mem::replace(&mut pipe, next);
|
||||
let ready = Arc::clone(&ready);
|
||||
let pending = Arc::clone(&pending);
|
||||
let cancel = cancel.clone();
|
||||
let mut cancelled = cancelled.clone();
|
||||
let name = name.clone();
|
||||
let registry = Arc::clone(®istry);
|
||||
let ids = Arc::clone(&server_ids);
|
||||
let released = Arc::clone(&server_released);
|
||||
let received = Arc::clone(&server_received);
|
||||
let download_id = download_id.clone();
|
||||
connections.spawn(async move {
|
||||
let (read, mut write) = tokio::io::split(connection);
|
||||
let mut reader = BufReader::new(read);
|
||||
let mut line = String::new();
|
||||
while reader.read_line(&mut line).await.unwrap_or(0) > 0 {
|
||||
let Frame::Request(req) = serde_json::from_str::<Frame>(&line).unwrap()
|
||||
else {
|
||||
panic!("expected request");
|
||||
};
|
||||
line.clear();
|
||||
received.lock().unwrap().push(req.method.clone());
|
||||
let params = req.params.clone().unwrap_or_default();
|
||||
let body = match req.method {
|
||||
Method::SystemStatus => ResponseBody::Ok(json!({
|
||||
"daemon_version": "0.2.0", "protocol_version": "1.1",
|
||||
"pid": std::process::id(), "uptime_secs": 0,
|
||||
"ws_port": 0, "sock_path": name,
|
||||
"browsers": [], "sessions": [], "version_skew_browsers": []
|
||||
})),
|
||||
Method::TransferBegin => {
|
||||
let begin = registry
|
||||
.begin_upload(serde_json::from_value(params).unwrap())
|
||||
.unwrap();
|
||||
ids.lock().unwrap().push(begin.transfer_id.clone());
|
||||
response(Ok(begin))
|
||||
}
|
||||
Method::ToolDownload => ResponseBody::Ok(json!({
|
||||
"tab_id": 1, "suggested_filename": "download.txt",
|
||||
"byte_size": 16, "transfer_id": download_id,
|
||||
})),
|
||||
Method::ToolWaitMs if mode == "ordinary" => {
|
||||
ResponseBody::Ok(json!({"waited_ms": 1}))
|
||||
}
|
||||
Method::TransferChunk
|
||||
if mode == "upload-before-dispatch"
|
||||
|| (mode.contains("multi")
|
||||
&& ids.lock().unwrap().len() == 1) =>
|
||||
{
|
||||
response(
|
||||
registry.write_chunk(serde_json::from_value(params).unwrap()),
|
||||
)
|
||||
}
|
||||
Method::TransferFinish if mode != "upload-before-dispatch" => response(
|
||||
registry.finish_upload(serde_json::from_value(params).unwrap()),
|
||||
),
|
||||
Method::ToolWaitMs
|
||||
| Method::TransferChunk
|
||||
| Method::TransferRead
|
||||
| Method::TransferFinish => {
|
||||
*pending.lock().unwrap() = Some(req.id.clone());
|
||||
ready.notify_one();
|
||||
cancelled.wait_for(|value| *value).await.unwrap();
|
||||
if mode == "upload-before-dispatch" {
|
||||
// Finish won the race with cancellation, but tool.upload must
|
||||
// not be dispatched and the CLI still owns the staging.
|
||||
response(
|
||||
registry
|
||||
.finish_upload(serde_json::from_value(params).unwrap()),
|
||||
)
|
||||
} else {
|
||||
ResponseBody::Err(RpcError {
|
||||
code: ErrorCode::Cancelled,
|
||||
message: "parent cancellation reached daemon".into(),
|
||||
data: None,
|
||||
})
|
||||
}
|
||||
}
|
||||
Method::Cancel => {
|
||||
let target = params["rpc_id"].as_str().unwrap();
|
||||
assert_eq!(pending.lock().unwrap().as_deref(), Some(target));
|
||||
cancel.send(true).unwrap();
|
||||
ResponseBody::Ok(json!({"cancelled": true}))
|
||||
}
|
||||
Method::TransferRelease => {
|
||||
let params: TransferIdParams =
|
||||
serde_json::from_value(params).unwrap();
|
||||
released.lock().unwrap().push(params.transfer_id.clone());
|
||||
if mode.contains("stall") {
|
||||
std::future::pending::<()>().await;
|
||||
}
|
||||
response(Ok(registry.release(params)))
|
||||
}
|
||||
other => panic!("unexpected request after cancellation: {other:?}"),
|
||||
};
|
||||
let response = Frame::Response(ResponseFrame { id: req.id, body });
|
||||
let mut bytes = serde_json::to_vec(&response).unwrap();
|
||||
bytes.push(b'\n');
|
||||
if write.write_all(&bytes).await.is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
let mut cmd = Command::new(env!("CARGO_BIN_EXE_bsk"));
|
||||
cmd.env("BSK_HOME", home.path())
|
||||
.env("BSK_AUTO_UPDATE", "off")
|
||||
.env_remove("BSK_CANCEL_ON_STDIN_CLOSE")
|
||||
.stdout(std::process::Stdio::piped())
|
||||
.stderr(std::process::Stdio::piped())
|
||||
.creation_flags(0x0800_0000)
|
||||
.kill_on_drop(true);
|
||||
if mode.starts_with("upload") {
|
||||
let source = home.path().join("upload.txt");
|
||||
// Declare 300 MiB without allocating or transferring that much data.
|
||||
std::fs::File::create(&source)
|
||||
.unwrap()
|
||||
.set_len(if mode == "upload-before-dispatch" {
|
||||
16
|
||||
} else {
|
||||
300 * 1024 * 1024
|
||||
})
|
||||
.unwrap();
|
||||
cmd.args(["upload", "--session", "fixture", "--selector", "input"]);
|
||||
if mode.contains("multi") {
|
||||
let first = home.path().join("first.txt");
|
||||
std::fs::write(&first, b"first file").unwrap();
|
||||
cmd.arg("--file").arg(first);
|
||||
}
|
||||
cmd.arg("--file").arg(source);
|
||||
} else if mode == "download" {
|
||||
let out = home.path().join("out.txt");
|
||||
std::fs::write(&out, b"existing output").unwrap();
|
||||
cmd.args([
|
||||
"download",
|
||||
"--session",
|
||||
"fixture",
|
||||
"--selector",
|
||||
"a",
|
||||
"--overwrite",
|
||||
"--out",
|
||||
])
|
||||
.arg(out);
|
||||
} else {
|
||||
cmd.args(["wait-ms", if mode == "ordinary" { "1ms" } else { "60s" }]);
|
||||
}
|
||||
cmd.arg("--json");
|
||||
if mode == "ordinary" {
|
||||
cmd.stdin(std::process::Stdio::null());
|
||||
} else {
|
||||
cmd.env("BSK_CANCEL_ON_STDIN_CLOSE", "1")
|
||||
.stdin(std::process::Stdio::piped());
|
||||
}
|
||||
let mut child = cmd.spawn().unwrap();
|
||||
if mode != "ordinary" {
|
||||
tokio::time::timeout(Duration::from_secs(10), registered.notified())
|
||||
.await
|
||||
.expect("CLI registered business RPC");
|
||||
child
|
||||
.stdin
|
||||
.as_mut()
|
||||
.unwrap()
|
||||
.write_all(b"still connected")
|
||||
.await
|
||||
.unwrap();
|
||||
drop(child.stdin.take());
|
||||
}
|
||||
let start = Instant::now();
|
||||
let output = tokio::time::timeout(Duration::from_secs(8), child.wait_with_output())
|
||||
.await
|
||||
.expect("CLI must settle promptly, including cleanup")
|
||||
.unwrap();
|
||||
server.abort();
|
||||
let _ = server.await;
|
||||
assert_eq!(
|
||||
output.status.code(),
|
||||
Some(if mode == "ordinary" { 0 } else { 2 }),
|
||||
"{mode}: stdout={} stderr={}",
|
||||
String::from_utf8_lossy(&output.stdout),
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
if mode != "ordinary" {
|
||||
let body: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap();
|
||||
assert_eq!(body["code"], "cancelled");
|
||||
if mode != "upload-before-dispatch" {
|
||||
assert_eq!(body["message"], "parent cancellation reached daemon");
|
||||
}
|
||||
let received = received.lock().unwrap();
|
||||
let cancel = received.iter().position(|m| *m == Method::Cancel).unwrap();
|
||||
assert!(
|
||||
received[cancel + 1..]
|
||||
.iter()
|
||||
.all(|m| *m == Method::TransferRelease),
|
||||
"only cleanup may follow cancellation: {received:?}"
|
||||
);
|
||||
}
|
||||
let ids = ids.lock().unwrap();
|
||||
if mode.contains("stall") {
|
||||
assert_eq!(released.lock().unwrap().len(), 1);
|
||||
assert!(start.elapsed() >= Duration::from_secs(4));
|
||||
// The CLI cannot force an unresponsive daemon to clean up.
|
||||
transfers.release_session("fixture");
|
||||
} else {
|
||||
assert_eq!(
|
||||
*released.lock().unwrap(),
|
||||
*ids,
|
||||
"{mode}: every allocation must be released"
|
||||
);
|
||||
for id in ids.iter() {
|
||||
assert!(!staging_root.join(id).exists(), "staging leaked: {id}");
|
||||
}
|
||||
if mode.starts_with("upload") {
|
||||
let retry = transfers
|
||||
.begin_upload(TransferBeginParams {
|
||||
session_id: "fixture".into(),
|
||||
name: "retry.txt".into(),
|
||||
byte_size: 300 * 1024 * 1024,
|
||||
})
|
||||
.expect("same session must have room for another 300 MiB upload");
|
||||
transfers.release(TransferIdParams {
|
||||
transfer_id: retry.transfer_id,
|
||||
});
|
||||
}
|
||||
}
|
||||
if mode == "download" {
|
||||
assert_eq!(
|
||||
std::fs::read(home.path().join("out.txt")).unwrap(),
|
||||
b"existing output"
|
||||
);
|
||||
assert!(
|
||||
!std::fs::read_dir(home.path())
|
||||
.unwrap()
|
||||
.flatten()
|
||||
.any(|entry| entry.file_name().to_string_lossy().ends_with(".part"))
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,343 @@
|
||||
//! Exercise self-update with a real executable and a local release server.
|
||||
#![cfg(windows)]
|
||||
|
||||
use std::fs;
|
||||
use std::io::{Cursor, Read, Write};
|
||||
use std::net::{TcpListener, TcpStream};
|
||||
use std::os::windows::process::CommandExt;
|
||||
use std::path::PathBuf;
|
||||
use std::process::{Child, Command, Stdio};
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
||||
use std::thread;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use bsk::daemon::info::DaemonInfo;
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
const MARKER: &[u8] = b"windows-update-regression-fixture";
|
||||
|
||||
struct ReleaseServer {
|
||||
url: String,
|
||||
requests: Arc<AtomicUsize>,
|
||||
stop: Arc<AtomicBool>,
|
||||
worker: Option<thread::JoinHandle<()>>,
|
||||
}
|
||||
|
||||
impl ReleaseServer {
|
||||
fn new(binary: &[u8]) -> Self {
|
||||
let mut archive = zip::ZipWriter::new(Cursor::new(Vec::new()));
|
||||
archive
|
||||
.start_file(
|
||||
"bsk.exe",
|
||||
zip::write::SimpleFileOptions::default()
|
||||
.compression_method(zip::CompressionMethod::Stored),
|
||||
)
|
||||
.unwrap();
|
||||
archive.write_all(binary).unwrap();
|
||||
let archive = archive.finish().unwrap().into_inner();
|
||||
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
listener.set_nonblocking(true).unwrap();
|
||||
let url = format!("http://{}", listener.local_addr().unwrap());
|
||||
let manifest = serde_json::to_vec(&serde_json::json!({
|
||||
"version": "999.0.0",
|
||||
"assets": {"windows-x64": {
|
||||
"url": format!("{url}/bsk.zip"),
|
||||
"sha256": Sha256::digest(&archive).iter().map(|byte| format!("{byte:02x}")).collect::<String>(),
|
||||
}},
|
||||
}))
|
||||
.unwrap();
|
||||
let stop = Arc::new(AtomicBool::new(false));
|
||||
let worker_stop = Arc::clone(&stop);
|
||||
let requests = Arc::new(AtomicUsize::new(0));
|
||||
let worker_requests = Arc::clone(&requests);
|
||||
let worker = thread::spawn(move || {
|
||||
while !worker_stop.load(Ordering::SeqCst) {
|
||||
let (mut stream, _) = match listener.accept() {
|
||||
Ok(connection) => connection,
|
||||
Err(err) if err.kind() == std::io::ErrorKind::WouldBlock => {
|
||||
thread::sleep(Duration::from_millis(10));
|
||||
continue;
|
||||
}
|
||||
Err(err) => panic!("accept release request: {err}"),
|
||||
};
|
||||
// Windows accept() inherits the listener's nonblocking mode.
|
||||
// Keep accept polling for shutdown, but read/write each request
|
||||
// in blocking mode with the bounded timeouts below.
|
||||
stream.set_nonblocking(false).unwrap();
|
||||
stream
|
||||
.set_read_timeout(Some(Duration::from_secs(5)))
|
||||
.unwrap();
|
||||
stream
|
||||
.set_write_timeout(Some(Duration::from_secs(5)))
|
||||
.unwrap();
|
||||
let mut request = Vec::new();
|
||||
let mut chunk = [0; 1024];
|
||||
while !request.windows(4).any(|window| window == b"\r\n\r\n") {
|
||||
let len = stream.read(&mut chunk).unwrap();
|
||||
if len == 0 {
|
||||
break;
|
||||
}
|
||||
request.extend_from_slice(&chunk[..len]);
|
||||
}
|
||||
let body = if request.starts_with(b"GET /bsk.zip ") {
|
||||
worker_requests.fetch_add(1, Ordering::SeqCst);
|
||||
&archive
|
||||
} else {
|
||||
&manifest
|
||||
};
|
||||
write!(
|
||||
stream,
|
||||
"HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
|
||||
body.len()
|
||||
)
|
||||
.unwrap();
|
||||
stream.write_all(body).unwrap();
|
||||
}
|
||||
});
|
||||
Self {
|
||||
url: format!("{url}/version.json"),
|
||||
requests,
|
||||
stop,
|
||||
worker: Some(worker),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for ReleaseServer {
|
||||
fn drop(&mut self) {
|
||||
self.stop.store(true, Ordering::SeqCst);
|
||||
let _ = self.worker.take().unwrap().join();
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn release_server_waits_for_delayed_and_fragmented_request_headers() {
|
||||
let server = ReleaseServer::new(b"test binary");
|
||||
let address = server
|
||||
.url
|
||||
.strip_prefix("http://")
|
||||
.unwrap()
|
||||
.strip_suffix("/version.json")
|
||||
.unwrap();
|
||||
let mut stream = TcpStream::connect(address).unwrap();
|
||||
stream
|
||||
.set_read_timeout(Some(Duration::from_secs(5)))
|
||||
.unwrap();
|
||||
stream
|
||||
.set_write_timeout(Some(Duration::from_secs(5)))
|
||||
.unwrap();
|
||||
// Give accept() time to run before any bytes arrive, then split the
|
||||
// headers across writes. Both reads must wait rather than return WouldBlock.
|
||||
thread::sleep(Duration::from_millis(100));
|
||||
stream.write_all(b"GET /version.json HTTP/1.1\r\n").unwrap();
|
||||
thread::sleep(Duration::from_millis(100));
|
||||
stream.write_all(b"Host: localhost\r\n\r\n").unwrap();
|
||||
let mut response = String::new();
|
||||
stream.read_to_string(&mut response).unwrap();
|
||||
assert!(response.starts_with("HTTP/1.1 200 OK\r\n"), "{response}");
|
||||
let (_, body) = response.split_once("\r\n\r\n").unwrap();
|
||||
let manifest: serde_json::Value = serde_json::from_str(body).unwrap();
|
||||
assert_eq!(manifest["version"], "999.0.0");
|
||||
}
|
||||
|
||||
struct Fixture {
|
||||
_tmp: tempfile::TempDir,
|
||||
exe: PathBuf,
|
||||
home: PathBuf,
|
||||
binary: Vec<u8>,
|
||||
server: ReleaseServer,
|
||||
daemon: Option<Child>,
|
||||
}
|
||||
|
||||
impl Fixture {
|
||||
fn new() -> Self {
|
||||
let tmp = tempfile::TempDir::new().unwrap();
|
||||
let dir = tmp.path().join("中文 space %PATH% ! & (update)");
|
||||
fs::create_dir(&dir).unwrap();
|
||||
let exe = dir.join("bsk.exe");
|
||||
let home = tmp.path().join("home");
|
||||
fs::create_dir(&home).unwrap();
|
||||
fs::copy(env!("CARGO_BIN_EXE_bsk"), &exe).unwrap();
|
||||
// A PE overlay distinguishes the replacement without requiring a
|
||||
// second build or changing the executable's behavior/version.
|
||||
let mut binary = fs::read(&exe).unwrap();
|
||||
binary.extend_from_slice(MARKER);
|
||||
let server = ReleaseServer::new(&binary);
|
||||
Self {
|
||||
_tmp: tmp,
|
||||
exe,
|
||||
home,
|
||||
binary,
|
||||
server,
|
||||
daemon: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn command(&self) -> Command {
|
||||
let mut cmd = Command::new(&self.exe);
|
||||
cmd.env("BSK_HOME", &self.home)
|
||||
.env("BSK_UPDATE_MANIFEST_URL", &self.server.url)
|
||||
.env("BSK_AUTO_UPDATE", "off")
|
||||
.env("RUST_LOG", "info")
|
||||
.env("NO_PROXY", "127.0.0.1,localhost")
|
||||
.env_remove("BSK_DAEMONIZED")
|
||||
.env_remove("BSK_DAEMON_REPLACES_PID")
|
||||
.stdin(Stdio::null())
|
||||
.creation_flags(0x0800_0000);
|
||||
cmd
|
||||
}
|
||||
|
||||
fn wait_for(&self, description: &str, mut check: impl FnMut() -> bool) {
|
||||
let deadline = Instant::now() + Duration::from_secs(30);
|
||||
while Instant::now() < deadline {
|
||||
if check() {
|
||||
return;
|
||||
}
|
||||
thread::sleep(Duration::from_millis(50));
|
||||
}
|
||||
let diagnostics: Vec<_> = fs::read_dir(self.exe.parent().unwrap())
|
||||
.unwrap()
|
||||
.flatten()
|
||||
.filter(|entry| entry.path().extension().is_some_and(|ext| ext == "log"))
|
||||
.map(|entry| {
|
||||
(
|
||||
entry.path(),
|
||||
String::from_utf8_lossy(&fs::read(entry.path()).unwrap_or_default())
|
||||
.into_owned(),
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
let home_logs: Vec<_> = fs::read_dir(&self.home)
|
||||
.unwrap()
|
||||
.flatten()
|
||||
.filter(|entry| entry.file_name().to_string_lossy().contains("log"))
|
||||
.map(|entry| {
|
||||
(
|
||||
entry.path(),
|
||||
String::from_utf8_lossy(&fs::read(entry.path()).unwrap_or_default())
|
||||
.into_owned(),
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
panic!(
|
||||
"timed out waiting for {description}; helper logs: {diagnostics:?}; daemon logs: {home_logs:?}"
|
||||
);
|
||||
}
|
||||
|
||||
fn info(&self) -> Option<DaemonInfo> {
|
||||
serde_json::from_slice(&fs::read(self.home.join("daemon.json")).ok()?).ok()
|
||||
}
|
||||
|
||||
fn updated(&self) -> bool {
|
||||
// A locked executable may briefly reject reads during replacement.
|
||||
fs::read(&self.exe).is_ok_and(|binary| binary == self.binary)
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Fixture {
|
||||
fn drop(&mut self) {
|
||||
// Stop only the daemon belonging to this isolated BSK_HOME. Also reap
|
||||
// the foreground process on assertion failures to avoid leaking locks.
|
||||
let _ = self.command().args(["daemon", "stop"]).output();
|
||||
if let Some(child) = self.daemon.as_mut() {
|
||||
let _ = child.kill();
|
||||
let _ = child.wait();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn manual_update_replaces_the_running_executable_after_cli_exit() {
|
||||
let fixture = Fixture::new();
|
||||
let out = fixture
|
||||
.command()
|
||||
.args(["--json", "update", "--yes"])
|
||||
.output()
|
||||
.unwrap();
|
||||
assert!(
|
||||
out.status.success(),
|
||||
"{}",
|
||||
String::from_utf8_lossy(&out.stderr)
|
||||
);
|
||||
let report: serde_json::Value = serde_json::from_slice(&out.stdout).unwrap();
|
||||
assert_eq!(report["status"], "staged");
|
||||
fixture.wait_for("CLI replacement and helper cleanup", || {
|
||||
fixture.updated() && fs::read_dir(fixture.exe.parent().unwrap()).unwrap().count() == 1
|
||||
});
|
||||
assert!(
|
||||
fixture.info().is_none(),
|
||||
"an update must not start a previously absent daemon"
|
||||
);
|
||||
assert_eq!(fixture.server.requests.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn automatic_update_exits_old_daemon_and_restarts_on_the_same_port() {
|
||||
let mut fixture = Fixture::new();
|
||||
let port = unused_port();
|
||||
let log = fs::File::create(fixture.home.join("foreground.log")).unwrap();
|
||||
let child = fixture
|
||||
.command()
|
||||
.env("BSK_AUTO_UPDATE", "on")
|
||||
// Exercise the environment inherited by a normally detached daemon.
|
||||
.env("BSK_DAEMONIZED", "1")
|
||||
.args([
|
||||
"daemon",
|
||||
"start",
|
||||
"--port",
|
||||
&port.to_string(),
|
||||
"--session-idle",
|
||||
"1234ms",
|
||||
"--daemon-idle",
|
||||
"30s",
|
||||
])
|
||||
.stdout(Stdio::from(log.try_clone().unwrap()))
|
||||
.stderr(Stdio::from(log))
|
||||
.spawn()
|
||||
.unwrap();
|
||||
let old_pid = child.id();
|
||||
fixture.daemon = Some(child);
|
||||
fixture.wait_for("replacement daemon", || {
|
||||
fixture.updated()
|
||||
&& fixture
|
||||
.info()
|
||||
.is_some_and(|info| info.pid != old_pid && info.ws_port == port)
|
||||
});
|
||||
assert!(
|
||||
fixture
|
||||
.daemon
|
||||
.as_mut()
|
||||
.unwrap()
|
||||
.try_wait()
|
||||
.unwrap()
|
||||
.is_some(),
|
||||
"old daemon must exit"
|
||||
);
|
||||
fixture.wait_for("helper cleanup", || {
|
||||
fs::read_dir(fixture.exe.parent().unwrap()).unwrap().count() == 1
|
||||
});
|
||||
let status = fixture
|
||||
.command()
|
||||
.args(["--json", "status"])
|
||||
.output()
|
||||
.unwrap();
|
||||
assert!(
|
||||
status.status.success(),
|
||||
"{}",
|
||||
String::from_utf8_lossy(&status.stderr)
|
||||
);
|
||||
assert_eq!(
|
||||
fixture.server.requests.load(Ordering::SeqCst),
|
||||
1,
|
||||
"the replacement must not stage another update immediately"
|
||||
);
|
||||
}
|
||||
|
||||
fn unused_port() -> u16 {
|
||||
TcpListener::bind("127.0.0.1:0")
|
||||
.unwrap()
|
||||
.local_addr()
|
||||
.unwrap()
|
||||
.port()
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user