mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
Compare commits
137
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0822c53c65 | ||
|
|
caea98cd52 | ||
|
|
e8137b42cc | ||
|
|
cb5dc148d5 | ||
|
|
3c5f838c56 | ||
|
|
9337bd557b | ||
|
|
162a31ae92 | ||
|
|
2f84234a31 | ||
|
|
72c10aab75 | ||
|
|
1eeb55af02 | ||
|
|
4992a54864 | ||
|
|
8b195eb0df | ||
|
|
eae4db1729 | ||
|
|
e961395be9 | ||
|
|
a6d1d02acc | ||
|
|
5ebe609c40 | ||
|
|
321494d93f | ||
|
|
f2184e74c5 | ||
|
|
eeb633d68e | ||
|
|
6d181105f2 | ||
|
|
ebad0ed1b5 | ||
|
|
498f8fb4a7 | ||
|
|
cef59137e4 | ||
|
|
f70978bfcf | ||
|
|
1498cd425c | ||
|
|
e831d65db3 | ||
|
|
47ac947f18 | ||
|
|
69d64c4eb1 | ||
|
|
99c997d6d5 | ||
|
|
93853ebc65 | ||
|
|
248bb90a2d | ||
|
|
6e6c178434 | ||
|
|
386ef91c24 | ||
|
|
0dc11b30ea | ||
|
|
759eebd4b6 | ||
|
|
fd3ce202a1 | ||
|
|
618bd6a0df | ||
|
|
c455a09fc3 | ||
|
|
b6be356e40 | ||
|
|
945bf1523d | ||
|
|
a928a9fb76 | ||
|
|
0b2d7e95cc | ||
|
|
556bea1645 | ||
|
|
0302d5592b | ||
|
|
eb34ec0dbf | ||
|
|
43067b6bfe | ||
|
|
6cc0d38489 | ||
|
|
0b71963344 | ||
|
|
59e2c3224e | ||
|
|
d70df753f6 | ||
|
|
66ae555e40 | ||
|
|
fee7c0983c | ||
|
|
51473c01ff | ||
|
|
49744233f7 | ||
|
|
95571f8b70 | ||
|
|
8344e311fa | ||
|
|
63338cd98e | ||
|
|
bd193ceb5d | ||
|
|
5ed828736f | ||
|
|
ee620acf64 | ||
|
|
05a36075b8 | ||
|
|
1bd13d5c3c | ||
|
|
59d6533eff | ||
|
|
536d3a9844 | ||
|
|
a77353bce8 | ||
|
|
91621b2545 | ||
|
|
641d934337 | ||
|
|
a4fed136e3 | ||
|
|
3c77ef7b01 | ||
|
|
3b21f2f0a7 | ||
|
|
405433ab65 | ||
|
|
58c7f38a05 | ||
|
|
e0c88295c7 | ||
|
|
dc36e38cb9 | ||
|
|
d5de9b0154 | ||
|
|
995061514c | ||
|
|
406bb21b7e | ||
|
|
f06a514bd2 | ||
|
|
1e53ee63ae | ||
|
|
120dc441bb | ||
|
|
710b1219f4 | ||
|
|
177ecd3ebf | ||
|
|
0e6b9c4542 | ||
|
|
506adbe708 | ||
|
|
984d017b0f | ||
|
|
5df93143c5 | ||
|
|
7163f2f9b0 | ||
|
|
e9f0584af3 | ||
|
|
2997f495db | ||
|
|
f96bd72c3c | ||
|
|
329c68054b | ||
|
|
5cd43553ed | ||
|
|
9ad33801c7 | ||
|
|
7aaa5a2d01 | ||
|
|
210b21b637 | ||
|
|
48f64ec0df | ||
|
|
c48a53269e | ||
|
|
3bbd6ab19f | ||
|
|
1a70d2200e | ||
|
|
554861d6b1 | ||
|
|
0d7f976335 | ||
|
|
d1d3eb6a94 | ||
|
|
38fd7d3f4c | ||
|
|
b5062ac19b | ||
|
|
41cb576256 | ||
|
|
f060a7c074 | ||
|
|
60b62d5420 | ||
|
|
70b851a3e8 | ||
|
|
da3534d066 | ||
|
|
a004291848 | ||
|
|
b1b1ae1d48 | ||
|
|
8822bba9a5 | ||
|
|
c74c119a82 | ||
|
|
60fbf30e24 | ||
|
|
b801bc1f62 | ||
|
|
bf59753aa3 | ||
|
|
a86cbf724f | ||
|
|
76d447caa0 | ||
|
|
14e5bbe4fc | ||
|
|
bef722107f | ||
|
|
6800f8fdfe | ||
|
|
4b6cdde168 | ||
|
|
4aade4b94f | ||
|
|
616af1099f | ||
|
|
3656f0dd3c | ||
|
|
9742d83213 | ||
|
|
615e866608 | ||
|
|
c08fed5de3 | ||
|
|
887d1ed0b9 | ||
|
|
1a3d3b7de8 | ||
|
|
72834980d2 | ||
|
|
38b0debba9 | ||
|
|
4677149e5e | ||
|
|
8ea5477e11 | ||
|
|
17af1af65a | ||
|
|
a4b6a61d29 | ||
|
|
cad0b5cbc9 |
@@ -37,6 +37,35 @@ jobs:
|
||||
- name: Run Rust tests
|
||||
run: cargo test --workspace --locked
|
||||
|
||||
windows-daemon:
|
||||
name: Windows daemon, installer and cancellation regression tests
|
||||
runs-on: windows-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Run Windows named-pipe tests
|
||||
run: cargo test -p bsk --lib --locked daemon::ipc::windows::tests
|
||||
|
||||
- name: Run Windows process liveness tests
|
||||
run: cargo test -p bsk --lib --locked daemon::lockfile::tests
|
||||
|
||||
- name: Run Windows update tests
|
||||
run: cargo test -p bsk --lib --locked cli::update::tests
|
||||
|
||||
- name: Run Windows update and cancellation process tests
|
||||
run: cargo test -p bsk --test windows_update --test windows_parent_cancel --locked
|
||||
|
||||
- name: Run installer tests (Windows PowerShell 5.1)
|
||||
shell: powershell
|
||||
run: ./scripts/install-windows.test.ps1
|
||||
|
||||
- name: Run installer tests (PowerShell 7)
|
||||
shell: pwsh
|
||||
run: ./scripts/install-windows.test.ps1
|
||||
|
||||
frontend:
|
||||
name: Frontend lint, typecheck, tests, build
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
@@ -325,3 +325,180 @@ jobs:
|
||||
|
||||
echo "Chrome Web Store submission created successfully:"
|
||||
jq '{itemId, state, warningInfo}' "$PUBLISH_RESPONSE"
|
||||
|
||||
publish-edge-addons:
|
||||
needs: [resolve, build-extension]
|
||||
runs-on: ubuntu-latest
|
||||
environment: edge-addons
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Download build artifact
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: extension-zip
|
||||
path: dist
|
||||
|
||||
- name: Validate Microsoft Edge Add-ons configuration
|
||||
shell: bash
|
||||
env:
|
||||
API_KEY: ${{ secrets.EDGE_ADDONS_API_KEY }}
|
||||
CLIENT_ID: ${{ vars.EDGE_ADDONS_CLIENT_ID }}
|
||||
PRODUCT_ID: ${{ vars.EDGE_ADDONS_PRODUCT_ID }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "$API_KEY" ]; then
|
||||
echo "::error::Environment secret EDGE_ADDONS_API_KEY is not configured"
|
||||
exit 1
|
||||
fi
|
||||
if [ -z "$CLIENT_ID" ]; then
|
||||
echo "::error::Environment variable EDGE_ADDONS_CLIENT_ID is not configured"
|
||||
exit 1
|
||||
fi
|
||||
if [ -z "$PRODUCT_ID" ]; then
|
||||
echo "::error::Environment variable EDGE_ADDONS_PRODUCT_ID is not configured"
|
||||
exit 1
|
||||
fi
|
||||
if ! [[ "$PRODUCT_ID" =~ ^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$ ]]; then
|
||||
echo "::error::EDGE_ADDONS_PRODUCT_ID must be the Partner Center product GUID, not the public Edge Add-ons extension ID"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
ARCHIVE="$(find dist -maxdepth 1 -name '*.zip' -type f -print -quit)"
|
||||
if [ -z "$ARCHIVE" ]; then
|
||||
echo "::error::No extension zip found in the downloaded artifact"
|
||||
exit 1
|
||||
fi
|
||||
echo "ARCHIVE=$ARCHIVE" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Upload and publish Microsoft Edge Add-ons extension
|
||||
shell: bash
|
||||
env:
|
||||
API_KEY: ${{ secrets.EDGE_ADDONS_API_KEY }}
|
||||
CLIENT_ID: ${{ vars.EDGE_ADDONS_CLIENT_ID }}
|
||||
PRODUCT_ID: ${{ vars.EDGE_ADDONS_PRODUCT_ID }}
|
||||
RELEASE_VERSION: ${{ needs.resolve.outputs.version }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
API_ROOT="https://api.addons.microsoftedge.microsoft.com/v1"
|
||||
UPLOAD_HEADERS="$RUNNER_TEMP/edge-upload-headers.txt"
|
||||
UPLOAD_RESPONSE="$RUNNER_TEMP/edge-upload-response.json"
|
||||
STATUS_RESPONSE="$RUNNER_TEMP/edge-status-response.json"
|
||||
PUBLISH_HEADERS="$RUNNER_TEMP/edge-publish-headers.txt"
|
||||
PUBLISH_RESPONSE="$RUNNER_TEMP/edge-publish-response.json"
|
||||
|
||||
operation_id_from_headers() {
|
||||
local headers="$1"
|
||||
local location
|
||||
location="$(awk '
|
||||
tolower($1) == "location:" {
|
||||
sub(/\r$/, "")
|
||||
sub(/^[^:]*:[[:space:]]*/, "")
|
||||
print
|
||||
exit
|
||||
}
|
||||
' "$headers")"
|
||||
location="${location%%\?*}"
|
||||
printf '%s' "${location##*/}"
|
||||
}
|
||||
|
||||
poll_operation() {
|
||||
local operation_name="$1"
|
||||
local status_url="$2"
|
||||
local status="InProgress"
|
||||
local http_status
|
||||
|
||||
for attempt in $(seq 1 30); do
|
||||
sleep 5
|
||||
http_status="$(curl --silent --show-error \
|
||||
--output "$STATUS_RESPONSE" \
|
||||
--write-out '%{http_code}' \
|
||||
--header "Authorization: ApiKey $API_KEY" \
|
||||
--header "X-ClientID: $CLIENT_ID" \
|
||||
"$status_url")"
|
||||
if [ "$http_status" -lt 200 ] || [ "$http_status" -ge 300 ]; then
|
||||
echo "::error::Microsoft Edge Add-ons $operation_name status check failed with HTTP $http_status"
|
||||
jq . "$STATUS_RESPONSE" || cat "$STATUS_RESPONSE"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
status="$(jq -r '.status // empty' "$STATUS_RESPONSE")"
|
||||
echo "$operation_name status check $attempt/30: $status"
|
||||
case "$status" in
|
||||
Succeeded)
|
||||
return 0
|
||||
;;
|
||||
InProgress)
|
||||
;;
|
||||
Failed)
|
||||
echo "::error::Microsoft Edge Add-ons $operation_name failed"
|
||||
jq . "$STATUS_RESPONSE"
|
||||
exit 1
|
||||
;;
|
||||
*)
|
||||
echo "::error::Microsoft Edge Add-ons returned an unknown $operation_name status: $status"
|
||||
jq . "$STATUS_RESPONSE"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
echo "::error::Microsoft Edge Add-ons $operation_name did not complete after 30 checks"
|
||||
jq . "$STATUS_RESPONSE"
|
||||
exit 1
|
||||
}
|
||||
|
||||
HTTP_STATUS="$(curl --silent --show-error \
|
||||
--output "$UPLOAD_RESPONSE" \
|
||||
--dump-header "$UPLOAD_HEADERS" \
|
||||
--write-out '%{http_code}' \
|
||||
--request POST \
|
||||
--header "Authorization: ApiKey $API_KEY" \
|
||||
--header "X-ClientID: $CLIENT_ID" \
|
||||
--header "Content-Type: application/zip" \
|
||||
--data-binary "@$ARCHIVE" \
|
||||
"$API_ROOT/products/$PRODUCT_ID/submissions/draft/package")"
|
||||
if [ "$HTTP_STATUS" != "202" ]; then
|
||||
echo "::error::Microsoft Edge Add-ons upload failed with HTTP $HTTP_STATUS"
|
||||
jq . "$UPLOAD_RESPONSE" || cat "$UPLOAD_RESPONSE"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
UPLOAD_OPERATION_ID="$(operation_id_from_headers "$UPLOAD_HEADERS")"
|
||||
if [ -z "$UPLOAD_OPERATION_ID" ]; then
|
||||
echo "::error::Microsoft Edge Add-ons upload response has no Location operation ID"
|
||||
cat "$UPLOAD_HEADERS"
|
||||
exit 1
|
||||
fi
|
||||
poll_operation \
|
||||
"package upload" \
|
||||
"$API_ROOT/products/$PRODUCT_ID/submissions/draft/package/operations/$UPLOAD_OPERATION_ID"
|
||||
|
||||
HTTP_STATUS="$(curl --silent --show-error \
|
||||
--output "$PUBLISH_RESPONSE" \
|
||||
--dump-header "$PUBLISH_HEADERS" \
|
||||
--write-out '%{http_code}' \
|
||||
--request POST \
|
||||
--header "Authorization: ApiKey $API_KEY" \
|
||||
--header "X-ClientID: $CLIENT_ID" \
|
||||
--header "Content-Type: application/json" \
|
||||
--data "{\"notes\":\"Automated BrowserSkill extension v${RELEASE_VERSION} release from GitHub Actions.\"}" \
|
||||
"$API_ROOT/products/$PRODUCT_ID/submissions")"
|
||||
if [ "$HTTP_STATUS" != "202" ]; then
|
||||
echo "::error::Microsoft Edge Add-ons publish failed with HTTP $HTTP_STATUS"
|
||||
jq . "$PUBLISH_RESPONSE" || cat "$PUBLISH_RESPONSE"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
PUBLISH_OPERATION_ID="$(operation_id_from_headers "$PUBLISH_HEADERS")"
|
||||
if [ -z "$PUBLISH_OPERATION_ID" ]; then
|
||||
echo "::error::Microsoft Edge Add-ons publish response has no Location operation ID"
|
||||
cat "$PUBLISH_HEADERS"
|
||||
exit 1
|
||||
fi
|
||||
poll_operation \
|
||||
"submission" \
|
||||
"$API_ROOT/products/$PRODUCT_ID/submissions/operations/$PUBLISH_OPERATION_ID"
|
||||
|
||||
echo "Microsoft Edge Add-ons submission created successfully:"
|
||||
jq '{status, message, errorCode, errors}' "$STATUS_RESPONSE"
|
||||
|
||||
@@ -7,10 +7,18 @@ dist/
|
||||
.output/
|
||||
.wxt/
|
||||
.web-extension-id
|
||||
.pnpm-store/
|
||||
|
||||
# Git worktrees
|
||||
.worktrees/
|
||||
|
||||
# Logs
|
||||
*.log
|
||||
|
||||
# Local browser evaluation artifacts
|
||||
evals/browser/results/
|
||||
evals/browser/agents.local.json
|
||||
|
||||
# macOS
|
||||
.DS_Store
|
||||
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
# Changelog
|
||||
|
||||
All notable changes to BrowserSkill will be documented in this file.
|
||||
The format is based on [Keep a Changelog](https://keepachangelog.com/).
|
||||
|
||||
Starting from 0.2.0, CLI / Extension / DSH Plugin share the same version number.
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
## [0.2.0] - 2026-09-02
|
||||
|
||||
### Added
|
||||
- File transfer: upload and download support across CLI, Extension, and DSH Plugin
|
||||
- File transfer: drag-and-drop upload (`drop-to-upload`)
|
||||
- VOM semantic graph, name enrichment, and hover perception modules
|
||||
- VOM hover probing (opt-in via `observe` parameter)
|
||||
- DSH Plugin: browser tool parity with CLI commands
|
||||
- Edge Add-ons automated publishing in CI
|
||||
- Protocol upgrade reminder when CLI / Extension protocol versions differ
|
||||
- Browser evaluation harness (`evals/browser/`)
|
||||
- Unified release script (`scripts/release.mjs`)
|
||||
|
||||
### Changed
|
||||
- **Version scheme**: all three components now share the same semver
|
||||
- VOM rendering algorithm optimizations
|
||||
- Leaner SKILL.md agent instructions
|
||||
- DSH Plugin: simplified browser commands
|
||||
- Borrow confirmation UX — proactive focus and longer timeout
|
||||
- PiP window now has a close button
|
||||
|
||||
### Fixed
|
||||
- Screenshot media type detection (was hard-coded to `image/png`)
|
||||
- DSH Plugin session lifecycle stability
|
||||
- DSH Plugin Cordis package ID mismatch
|
||||
- Observation thumbnail media type sniffing
|
||||
- Upload/download race conditions and layout bypass issues
|
||||
- VOM repeated name and safety policy issues
|
||||
|
||||
---
|
||||
|
||||
*Previous releases used independent version numbers per component.*
|
||||
|
||||
## CLI 0.1.11 / Extension 0.1.7 / DSH Plugin 0.1.2 — 2026-08-26 ~ 2026-08-29
|
||||
|
||||
### Added
|
||||
- DSH Plugin sidebar integration, session lifecycle, and archive cleanup
|
||||
- Recorder iframe and OOPIF support
|
||||
|
||||
### Changed
|
||||
- VOM functional refactor
|
||||
|
||||
### Fixed
|
||||
- Recorder safety policy and bug fixes
|
||||
|
||||
## CLI 0.1.10 / Extension 0.1.6 — 2026-08-08
|
||||
|
||||
### Added
|
||||
- VOM observation recording and settled-state detection
|
||||
- Record overlay timer
|
||||
|
||||
### Fixed
|
||||
- Browser keepalive disconnect handling
|
||||
|
||||
## CLI 0.1.9 / Extension 0.1.5 — 2026-07-29
|
||||
|
||||
### Added
|
||||
- CLI auto-update mechanism
|
||||
- Trace v3 protocol and recorder
|
||||
|
||||
### Fixed
|
||||
- MV3 keepalive disconnect
|
||||
|
||||
## CLI 0.1.8 / Extension 0.1.4 — 2026-07-22
|
||||
|
||||
### Added
|
||||
- More browser interaction actions
|
||||
|
||||
### Fixed
|
||||
- Windows named-pipe hash-only path issue
|
||||
|
||||
## CLI 0.1.7 / Extension 0.1.3 — 2026-07-07
|
||||
|
||||
Initial public release pair.
|
||||
|
||||
## CLI 0.1.6 — 2026-06-30
|
||||
|
||||
### Fixed
|
||||
- Minor CLI fixes
|
||||
|
||||
## CLI 0.1.5 / Extension 0.1.2 — 2026-06-22
|
||||
|
||||
First tagged releases.
|
||||
Generated
+3
-2
@@ -134,9 +134,10 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "bsk"
|
||||
version = "0.1.10"
|
||||
version = "0.2.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"base64",
|
||||
"bsk-protocol",
|
||||
"clap",
|
||||
"console",
|
||||
@@ -168,7 +169,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "bsk-protocol"
|
||||
version = "0.1.10"
|
||||
version = "0.2.0"
|
||||
dependencies = [
|
||||
"schemars",
|
||||
"semver",
|
||||
|
||||
+3
-2
@@ -7,7 +7,7 @@ members = [
|
||||
default-members = ["crates/bsk-cli"]
|
||||
|
||||
[workspace.package]
|
||||
version = "0.1.10"
|
||||
version = "0.2.0"
|
||||
edition = "2024"
|
||||
rust-version = "1.85"
|
||||
license = "MIT"
|
||||
@@ -36,13 +36,14 @@ flate2 = "1.1.9"
|
||||
tar = "0.4.46"
|
||||
zip = { version = "7.2.0", default-features = false, features = ["deflate"] }
|
||||
sha2 = "0.11.0"
|
||||
base64 = "0.22"
|
||||
tempfile = "3"
|
||||
nix = { version = "0.29", features = ["signal", "process"] }
|
||||
rand = "0.8"
|
||||
uuid = { version = "1", features = ["v4", "serde"] }
|
||||
|
||||
# Workspace crates (version required for `cargo publish` of dependents)
|
||||
bsk-protocol = { version = "0.1.9", path = "crates/bsk-protocol" }
|
||||
bsk-protocol = { version = "0.2.0", path = "crates/bsk-protocol" }
|
||||
|
||||
[profile.release]
|
||||
strip = "symbols"
|
||||
|
||||
@@ -208,6 +208,7 @@ The repository is a Cargo + pnpm workspace:
|
||||
- `apps/extension` — browser extension
|
||||
- `packages/ui` and `packages/i18n` — shared extension UI support
|
||||
- `packages/dsh-plugin-browserskill` — DeepSeek Harness plugin (`@wxg-prc-cpg/browser-skill-dsh-plugin`)
|
||||
- [`evals/browser`](evals/browser/README.md) — deterministic local pages and agent-neutral browser capability evaluation
|
||||
|
||||
## License
|
||||
|
||||
|
||||
@@ -178,6 +178,7 @@ Agent 不直接与浏览器通信。它通过 `bsk` CLI 下发浏览器任务;
|
||||
- `apps/extension` — 浏览器扩展
|
||||
- `packages/ui` 和 `packages/i18n` — 扩展 UI 共享支持
|
||||
- `packages/dsh-plugin-browserskill` — DeepSeek Harness 插件(`@wxg-prc-cpg/browser-skill-dsh-plugin`)
|
||||
- [`evals/browser`](evals/browser/README.zh-CN.md) — 确定性本地页面与 Agent 无关的浏览器能力测试台
|
||||
|
||||
## 许可证
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# BrowserSkill — Privacy Policy
|
||||
|
||||
**Last updated:** May 25, 2026
|
||||
**Last updated:** August 20, 2026
|
||||
|
||||
This Privacy Policy describes how the **BrowserSkill** browser extension (the "Extension") handles information when you install and use it. BrowserSkill is published as part of the open-source [BrowserSkill](https://github.com/Tencent/BrowserSkill) project. The source code is publicly auditable.
|
||||
|
||||
@@ -14,7 +14,7 @@ BrowserSkill is a local automation bridge that lets AI coding agents (such as Cu
|
||||
|
||||
## 2. Single Purpose
|
||||
|
||||
The Extension's single purpose is to expose browser automation primitives (navigation, DOM observation, screenshots, clicks, form filling, tab management) to a locally running BrowserSkill daemon over a WebSocket connection on `127.0.0.1`, so that an AI agent invoked by the user can interact with web pages on the user's behalf.
|
||||
The Extension's single purpose is to expose browser automation primitives (navigation, DOM observation, screenshots, clicks, form filling, task-scoped file transfer, and tab management) to a locally running BrowserSkill daemon over a WebSocket connection on `127.0.0.1`, so that an AI agent invoked by the user can interact with web pages on the user's behalf.
|
||||
|
||||
## 3. Data the Extension Accesses
|
||||
|
||||
@@ -26,6 +26,7 @@ Depending on the commands the user (via their AI agent) sends to the local daemo
|
||||
| **User input simulated by the agent** | Mouse clicks, keystrokes, and form values that the AI agent dispatches through the Chrome DevTools Protocol (CDP). | Required to perform automation actions the user has asked the agent to do. |
|
||||
| **Tab and window metadata** | Tab IDs, URLs, titles, window IDs of the Agent Window and any tabs the user explicitly authorizes. | Required to target automation commands at the correct tab/window. |
|
||||
| **Local extension storage** | A randomly generated 8-character instance ID and an optional user-supplied label. | Used so the local daemon can recognize this browser instance across reconnects. No personal data is stored. |
|
||||
| **File transfers requested by the agent** | Local files explicitly supplied to `bsk upload`, and the file created by a single `bsk download` action. | Required to attach a task file to a web page or return a browser-generated download to the invoking local agent. |
|
||||
| **OS notifications** | Permission to display a system notification when the agent requests to "borrow" one of the user's existing tabs. | Required to obtain explicit, per-tab user consent before the agent touches any pre-existing tab. |
|
||||
|
||||
## 4. Data the Extension Does **Not** Collect
|
||||
@@ -34,7 +35,7 @@ BrowserSkill does **not**:
|
||||
|
||||
- Send any data to remote servers, the Extension's authors, or any third party.
|
||||
- Call any LLM, AI, or cloud API. The Extension contains no API keys, model identifiers, or remote endpoints.
|
||||
- Read or transmit cookies, browsing history, bookmarks, downloads, saved passwords, or autofill data.
|
||||
- Read or transmit cookies, browsing history, bookmarks, saved passwords, or autofill data. It observes only the download initiated by an active `bsk download` call, not download history generally.
|
||||
- Use webcam, microphone, geolocation, or any device sensor.
|
||||
- Include analytics, telemetry, crash reporting, advertising SDKs, or fingerprinting code.
|
||||
- Track users across websites or across sessions.
|
||||
@@ -50,6 +51,7 @@ The Extension requests the following Chrome permissions. Each is used solely for
|
||||
- **`alarms`** — Periodically wake the service worker to keep the local WebSocket connection alive.
|
||||
- **`idle`** — Detect when the device returns from idle/locked so the Extension can promptly re-establish the local WebSocket connection after the machine wakes. No idle data is stored or transmitted.
|
||||
- **`notifications`** — Show a system notification to obtain user approval before the agent borrows a user-owned tab.
|
||||
- **`downloads`** — Correlate and route the one browser download initiated by an active `bsk download` command. If that claimed transaction fails, BrowserSkill cancels an in-progress file or removes its completed temporary browser file. It is not used to enumerate download history or alter unclaimed downloads.
|
||||
- **`storage`** — Persist a random instance ID and optional label in `chrome.storage.local`.
|
||||
- **Host permission `<all_urls>`** — Inject a small status overlay (showing "Agent Active") on pages controlled by the agent, and enable automation across whatever sites the user directs the agent to. The Extension does **not** read or transmit page content from sites the agent is not actively driving.
|
||||
|
||||
@@ -61,6 +63,7 @@ All Extension activity stays on the user's local device. The only network traffi
|
||||
|
||||
- The instance ID and optional label persist in `chrome.storage.local` until the user uninstalls the Extension or clears extension storage.
|
||||
- Page content, screenshots, DOM snapshots, and other observed data are returned to the local daemon in response to commands and are **not retained by the Extension**. They live only as long as the agent's tool call.
|
||||
- Upload and download bytes are staged by the local daemon in a private, session-scoped directory. Download staging is removed after it is copied to the requested destination. Upload staging is retained until the session ends so a later form submission can still read the attached file. Remaining staging is removed when the session ends or disconnects, or when the daemon next starts after a crash.
|
||||
|
||||
## 8. User Control
|
||||
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<title>BrowserSkill file-transfer fixture</title>
|
||||
<style>
|
||||
body { font: 16px system-ui; max-width: 760px; margin: 48px auto; }
|
||||
.actions { display: flex; gap: 12px; align-items: center; margin: 24px 0; }
|
||||
button, a { padding: 10px 16px; border: 1px solid #777; border-radius: 8px; }
|
||||
a[aria-disabled="true"] { opacity: .45; pointer-events: none; }
|
||||
#drop { padding: 36px; border: 3px dashed #777; border-radius: 12px; text-align: center; }
|
||||
#drop.dragging { border-color: #2563eb; background: #eff6ff; }
|
||||
img { display: block; max-width: 480px; max-height: 360px; margin-top: 20px; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Upload then download the same image</h1>
|
||||
<p id="status">No image selected</p>
|
||||
<div class="actions">
|
||||
<button id="upload" type="button">Upload through file input</button>
|
||||
<input id="file" type="file" accept="image/png" hidden />
|
||||
<a id="download" aria-disabled="true">Download uploaded image</a>
|
||||
</div>
|
||||
<div id="drop" role="button" tabindex="0">Drop an image here</div>
|
||||
<img id="preview" alt="Uploaded image preview" hidden />
|
||||
<script>
|
||||
const input = document.querySelector('#file');
|
||||
const upload = document.querySelector('#upload');
|
||||
const drop = document.querySelector('#drop');
|
||||
const download = document.querySelector('#download');
|
||||
const preview = document.querySelector('#preview');
|
||||
const status = document.querySelector('#status');
|
||||
let objectUrl;
|
||||
|
||||
window.__bskTransferFixture = { events: [], result: null };
|
||||
|
||||
function accept(file, mechanism) {
|
||||
if (!file) return;
|
||||
if (objectUrl) URL.revokeObjectURL(objectUrl);
|
||||
objectUrl = URL.createObjectURL(file);
|
||||
preview.src = objectUrl;
|
||||
preview.hidden = false;
|
||||
download.href = objectUrl;
|
||||
download.download = file.name;
|
||||
download.removeAttribute('aria-disabled');
|
||||
status.textContent = `Ready via ${mechanism}: ${file.name} (${file.size} bytes)`;
|
||||
window.__bskTransferFixture.result = {
|
||||
mechanism,
|
||||
name: file.name,
|
||||
size: file.size,
|
||||
type: file.type,
|
||||
hasPathProperty: 'path' in file,
|
||||
};
|
||||
}
|
||||
|
||||
upload.addEventListener('click', () => input.click());
|
||||
input.addEventListener('change', () => accept(input.files[0], 'input'));
|
||||
|
||||
for (const type of ['dragenter', 'dragover']) {
|
||||
drop.addEventListener(type, event => {
|
||||
const types = [...event.dataTransfer.types];
|
||||
window.__bskTransferFixture.events.push({ type, types });
|
||||
if (!types.includes('Files')) return;
|
||||
event.preventDefault();
|
||||
drop.classList.add('dragging');
|
||||
});
|
||||
}
|
||||
drop.addEventListener('dragleave', () => drop.classList.remove('dragging'));
|
||||
drop.addEventListener('drop', event => {
|
||||
window.__bskTransferFixture.events.push({
|
||||
type: 'drop',
|
||||
types: [...event.dataTransfer.types],
|
||||
fileCount: event.dataTransfer.files.length,
|
||||
});
|
||||
event.preventDefault();
|
||||
drop.classList.remove('dragging');
|
||||
accept(event.dataTransfer.files[0], 'drop');
|
||||
});
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@browser-skill/extension",
|
||||
"version": "0.1.6",
|
||||
"version": "0.2.0",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
|
||||
@@ -254,6 +254,69 @@ describe("ChromiumCdp", () => {
|
||||
expect(cdp.isAttached(7)).toBe(false);
|
||||
});
|
||||
|
||||
it.each([
|
||||
false,
|
||||
true,
|
||||
])("waits for detach before reconnecting a returned tab (detach rejects: %s)", async (rejectDetach) => {
|
||||
const { api } = fakeApi();
|
||||
const attached = new Set<number>();
|
||||
let finishDetach!: () => void;
|
||||
vi.mocked(api.attach).mockImplementation(async ({ tabId }) => {
|
||||
if (attached.has(tabId!)) throw new Error("Another debugger is already attached");
|
||||
attached.add(tabId!);
|
||||
});
|
||||
vi.mocked(api.detach).mockImplementation(async ({ tabId }) => {
|
||||
attached.delete(tabId!);
|
||||
});
|
||||
vi.mocked(api.detach).mockImplementationOnce(
|
||||
({ tabId }) =>
|
||||
new Promise<void>((resolve, reject) => {
|
||||
finishDetach = () => {
|
||||
attached.delete(tabId!);
|
||||
if (rejectDetach) reject(new Error("tab already detached"));
|
||||
else resolve();
|
||||
};
|
||||
}),
|
||||
);
|
||||
const cdp = new ChromiumCdp(api);
|
||||
cdp.trackSessionTab("aa11", 7);
|
||||
await cdp.ensureAttached(7);
|
||||
const returning = cdp.releaseSessionTab("aa11", 7);
|
||||
await vi.waitFor(() => expect(api.detach).toHaveBeenCalledOnce());
|
||||
let duplicateDetachFinished = false;
|
||||
const duplicateDetach = cdp.detach(7).then(() => {
|
||||
duplicateDetachFinished = true;
|
||||
});
|
||||
|
||||
cdp.trackSessionTab("bb22", 7);
|
||||
// Register rejection handlers immediately; the pre-fix driver rejects
|
||||
// both commands while the browser still has the previous attachment.
|
||||
const pending = Promise.allSettled([
|
||||
cdp.send(7, "Runtime.evaluate", { expression: "document.title" }),
|
||||
cdp.send(7, "DOM.getDocument"),
|
||||
]);
|
||||
await cdp.send(8, "DOM.getDocument");
|
||||
const finishedEarly = duplicateDetachFinished;
|
||||
finishDetach();
|
||||
await Promise.all([returning, duplicateDetach]);
|
||||
|
||||
expect(await pending).toEqual([
|
||||
{ status: "fulfilled", value: { ok: true } },
|
||||
{ status: "fulfilled", value: { ok: true } },
|
||||
]);
|
||||
expect(finishedEarly).toBe(false);
|
||||
expect(vi.mocked(api.attach).mock.calls.filter(([target]) => target.tabId === 7)).toHaveLength(
|
||||
2,
|
||||
);
|
||||
expect(cdp.isAttached(7)).toBe(true);
|
||||
expect(cdp.isAttached(8)).toBe(true);
|
||||
await cdp.detachSession("bb22");
|
||||
expect(api.detach).toHaveBeenCalledTimes(2);
|
||||
expect(cdp.isAttached(7)).toBe(false);
|
||||
expect(attached.has(7)).toBe(false);
|
||||
expect(cdp.isAttached(8)).toBe(true);
|
||||
});
|
||||
|
||||
it("detachAll() iterates every cached tab", async () => {
|
||||
const { api } = fakeApi();
|
||||
const cdp = new ChromiumCdp(api);
|
||||
@@ -651,6 +714,155 @@ describe("ChromiumCdp", () => {
|
||||
expect(cdp.networkEntriesSince(33, 0, 50, 1000).entries[0].url).toBeUndefined();
|
||||
});
|
||||
|
||||
it("releases only the returning session's claim on a tab", async () => {
|
||||
const { api } = fakeApi();
|
||||
const cdp = new ChromiumCdp(api);
|
||||
await cdp.ensureAttached(1);
|
||||
await cdp.ensureAttached(2);
|
||||
cdp.trackSessionTab("aa11", 1);
|
||||
cdp.trackSessionTab("bb22", 1);
|
||||
cdp.trackSessionTab("aa11", 2);
|
||||
|
||||
await cdp.releaseSessionTab("unknown", 1);
|
||||
await cdp.releaseSessionTab("aa11", 1);
|
||||
expect(api.detach).not.toHaveBeenCalled();
|
||||
|
||||
await cdp.releaseSessionTab("bb22", 1);
|
||||
await cdp.releaseSessionTab("bb22", 1);
|
||||
expect(api.detach).toHaveBeenCalledExactlyOnceWith({ tabId: 1 });
|
||||
expect(cdp.isAttached(2)).toBe(true);
|
||||
|
||||
await cdp.detachSession("aa11");
|
||||
expect(api.detach).toHaveBeenCalledTimes(2);
|
||||
expect(api.detach).toHaveBeenLastCalledWith({ tabId: 2 });
|
||||
});
|
||||
|
||||
it.each([
|
||||
false,
|
||||
true,
|
||||
])("releases a tab with an attachment in flight (new owner: %s)", async (newOwner) => {
|
||||
const { api } = fakeApi();
|
||||
let finishAttach!: () => void;
|
||||
vi.mocked(api.attach).mockImplementationOnce(
|
||||
() =>
|
||||
new Promise<void>((resolve) => {
|
||||
finishAttach = resolve;
|
||||
}),
|
||||
);
|
||||
const cdp = new ChromiumCdp(api);
|
||||
cdp.trackSessionTab("aa11", 1);
|
||||
const attaching = cdp.ensureAttached(1);
|
||||
const releasing = cdp.releaseSessionTab("aa11", 1);
|
||||
if (newOwner) cdp.trackSessionTab("bb22", 1);
|
||||
finishAttach();
|
||||
await Promise.all([attaching, releasing]);
|
||||
|
||||
expect(cdp.isAttached(1)).toBe(newOwner);
|
||||
expect(api.detach).toHaveBeenCalledTimes(newOwner ? 0 : 1);
|
||||
});
|
||||
|
||||
it("stops accepting dialogs on a returned tab even when another session keeps CDP attached", async () => {
|
||||
const { api, onEvent } = fakeApi();
|
||||
let controlled = true;
|
||||
const cdp = new ChromiumCdp(api, { shouldAutoAcceptDialog: () => controlled });
|
||||
cdp.trackSessionTab("aa11", 1);
|
||||
cdp.trackSessionTab("bb22", 1);
|
||||
await cdp.ensureAttached(1);
|
||||
onEvent.fire({ tabId: 1 }, "Page.javascriptDialogOpening", {
|
||||
type: "confirm",
|
||||
message: "before return",
|
||||
});
|
||||
await vi.waitFor(() => expect(cdp.dialogsSince(1, 0)).toHaveLength(1));
|
||||
vi.mocked(api.sendCommand).mockClear();
|
||||
|
||||
controlled = false;
|
||||
await cdp.releaseSessionTab("aa11", 1);
|
||||
expect(cdp.isAttached(1)).toBe(true);
|
||||
onEvent.fire({ tabId: 1 }, "Page.javascriptDialogOpening", {
|
||||
type: "confirm",
|
||||
message: "after return",
|
||||
});
|
||||
await Promise.resolve();
|
||||
expect(api.sendCommand).not.toHaveBeenCalled();
|
||||
expect(cdp.dialogsSince(1, 0)).toHaveLength(1);
|
||||
|
||||
controlled = true;
|
||||
onEvent.fire({ tabId: 1 }, "Page.javascriptDialogOpening", {
|
||||
type: "alert",
|
||||
message: "borrowed again",
|
||||
});
|
||||
await vi.waitFor(() => expect(cdp.dialogsSince(1, 0)).toHaveLength(2));
|
||||
expect(api.sendCommand).toHaveBeenCalledExactlyOnceWith(
|
||||
{ tabId: 1 },
|
||||
"Page.handleJavaScriptDialog",
|
||||
{ accept: true },
|
||||
);
|
||||
});
|
||||
|
||||
it("ignores dialog events after detach, including a pending eligibility check", async () => {
|
||||
const { api, onEvent } = fakeApi();
|
||||
let resolveEligibility!: (value: boolean) => void;
|
||||
const shouldAutoAcceptDialog = vi.fn(
|
||||
() =>
|
||||
new Promise<boolean>((resolve) => {
|
||||
resolveEligibility = resolve;
|
||||
}),
|
||||
);
|
||||
const cdp = new ChromiumCdp(api, { shouldAutoAcceptDialog });
|
||||
await cdp.ensureAttached(1);
|
||||
onEvent.fire({ tabId: 1 }, "Page.javascriptDialogOpening", { type: "alert" });
|
||||
await cdp.detach(1);
|
||||
resolveEligibility(true);
|
||||
await Promise.resolve();
|
||||
onEvent.fire({ tabId: 1 }, "Page.javascriptDialogOpening", { type: "alert" });
|
||||
|
||||
expect(shouldAutoAcceptDialog).toHaveBeenCalledOnce();
|
||||
expect(api.sendCommand).not.toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
"Page.handleJavaScriptDialog",
|
||||
expect.anything(),
|
||||
);
|
||||
expect(cdp.dialogsSince(1, 0)).toEqual([]);
|
||||
});
|
||||
|
||||
it("does not accept dialogs when the tab's current scope cannot be determined", async () => {
|
||||
const { api, onEvent } = fakeApi();
|
||||
const shouldAutoAcceptDialog = vi.fn(async () => {
|
||||
throw new Error("tab is gone");
|
||||
});
|
||||
const cdp = new ChromiumCdp(api, { shouldAutoAcceptDialog });
|
||||
await cdp.ensureAttached(1);
|
||||
vi.mocked(api.sendCommand).mockClear();
|
||||
|
||||
onEvent.fire({ tabId: 1 }, "Page.javascriptDialogOpening", { type: "confirm" });
|
||||
await Promise.resolve();
|
||||
|
||||
expect(shouldAutoAcceptDialog).toHaveBeenCalledWith(1);
|
||||
expect(api.sendCommand).not.toHaveBeenCalled();
|
||||
expect(cdp.dialogsSince(1, 0)).toEqual([]);
|
||||
});
|
||||
|
||||
it("does not restore dialog state when an acceptance completes after detach", async () => {
|
||||
const { api, onEvent } = fakeApi();
|
||||
const cdp = new ChromiumCdp(api);
|
||||
await cdp.ensureAttached(1);
|
||||
let finishAccept!: () => void;
|
||||
vi.mocked(api.sendCommand).mockImplementationOnce(
|
||||
() =>
|
||||
new Promise<void>((resolve) => {
|
||||
finishAccept = resolve;
|
||||
}),
|
||||
);
|
||||
|
||||
onEvent.fire({ tabId: 1 }, "Page.javascriptDialogOpening", { type: "alert" });
|
||||
await cdp.detach(1);
|
||||
finishAccept();
|
||||
await Promise.resolve();
|
||||
|
||||
expect(cdp.dialogsSince(1, 0)).toEqual([]);
|
||||
expect(cdp.dialogCursor(1)).toBe(0);
|
||||
});
|
||||
|
||||
it("detachSession only detaches tabs no other session owns", async () => {
|
||||
const { api } = fakeApi();
|
||||
const cdp = new ChromiumCdp(api);
|
||||
|
||||
@@ -165,6 +165,7 @@ export class ChromiumCdp {
|
||||
private readonly api: CdpDebuggerApi;
|
||||
private readonly attachedTabs = new Set<number>();
|
||||
private readonly attachInFlight = new Map<number, Promise<void>>();
|
||||
private readonly detachInFlight = new Map<number, Promise<void>>();
|
||||
private readonly tabOwners = new Map<number, Set<string>>();
|
||||
private readonly dialogBuffers = new Map<number, JavaScriptDialogInfo[]>();
|
||||
private readonly dialogSequences = new Map<number, number>();
|
||||
@@ -182,7 +183,13 @@ export class ChromiumCdp {
|
||||
private networkSubscription: { dispose(): void } | null = null;
|
||||
private frameTargetSubscription: { dispose(): void } | null = null;
|
||||
|
||||
constructor(api: CdpDebuggerApi = chromeDebuggerApi) {
|
||||
constructor(
|
||||
api: CdpDebuggerApi = chromeDebuggerApi,
|
||||
private readonly options: {
|
||||
/** CDP observation alone does not authorize dismissing user dialogs. */
|
||||
shouldAutoAcceptDialog?: (tabId: number) => boolean | Promise<boolean>;
|
||||
} = {},
|
||||
) {
|
||||
this.api = api;
|
||||
this.bindAutoDetach();
|
||||
this.bindDialogHandler();
|
||||
@@ -193,6 +200,10 @@ export class ChromiumCdp {
|
||||
|
||||
/** Attach to `tabId` if we haven't already in this driver. */
|
||||
async ensureAttached(tabId: number): Promise<void> {
|
||||
// Returning a tab clears the cache before Chrome finishes detaching.
|
||||
// New observers must wait before opening the next connection to that tab.
|
||||
const detaching = this.detachInFlight.get(tabId);
|
||||
if (detaching) await detaching;
|
||||
if (this.attachedTabs.has(tabId)) return;
|
||||
const existing = this.attachInFlight.get(tabId);
|
||||
if (existing) {
|
||||
@@ -425,6 +436,11 @@ export class ChromiumCdp {
|
||||
|
||||
/** Detach if attached; never throws. */
|
||||
async detach(tabId: number): Promise<void> {
|
||||
const existing = this.detachInFlight.get(tabId);
|
||||
if (existing) {
|
||||
await existing;
|
||||
return;
|
||||
}
|
||||
this.attachInFlight.delete(tabId);
|
||||
if (!this.attachedTabs.has(tabId)) return;
|
||||
this.attachedTabs.delete(tabId);
|
||||
@@ -432,13 +448,19 @@ export class ChromiumCdp {
|
||||
this.clearConsoleState(tabId);
|
||||
this.clearNetworkState(tabId);
|
||||
this.clearFrameState(tabId);
|
||||
try {
|
||||
await this.api.detach({ tabId });
|
||||
} catch (err) {
|
||||
// Tab may already be gone — Chrome auto-detaches on close. Log
|
||||
// at debug so production builds aren't noisy.
|
||||
console.debug("[bsk cdp] detach failed (likely tab already closed)", err);
|
||||
}
|
||||
const detach = (async () => {
|
||||
try {
|
||||
await this.api.detach({ tabId });
|
||||
} catch (err) {
|
||||
// Tab may already be gone — Chrome auto-detaches on close. Log
|
||||
// at debug so production builds aren't noisy.
|
||||
console.debug("[bsk cdp] detach failed (likely tab already closed)", err);
|
||||
}
|
||||
})().finally(() => {
|
||||
this.detachInFlight.delete(tabId);
|
||||
});
|
||||
this.detachInFlight.set(tabId, detach);
|
||||
await detach;
|
||||
}
|
||||
|
||||
/** True iff `ensureAttached(tabId)` has succeeded since the last detach. */
|
||||
@@ -453,6 +475,17 @@ export class ChromiumCdp {
|
||||
this.tabOwners.set(tabId, owners);
|
||||
}
|
||||
|
||||
/** Release one session's claim, preserving attachments still used by another. */
|
||||
async releaseSessionTab(sessionId: string, tabId: number): Promise<void> {
|
||||
const owners = this.tabOwners.get(tabId);
|
||||
if (!owners?.delete(sessionId) || owners.size > 0) return;
|
||||
this.tabOwners.delete(tabId);
|
||||
// An observation may still be attaching when the tab is returned. Wait
|
||||
// for it so detach cannot miss the attachment or remove a new owner's claim.
|
||||
await this.attachInFlight.get(tabId)?.catch(() => {});
|
||||
if (!this.tabOwners.has(tabId)) await this.detach(tabId);
|
||||
}
|
||||
|
||||
/** Subscribe to all CDP events. Returned disposable removes the listener. */
|
||||
onEvent(handler: (source: CdpDebuggee, method: string, params: unknown) => void): {
|
||||
dispose(): void;
|
||||
@@ -639,13 +672,23 @@ export class ChromiumCdp {
|
||||
}
|
||||
|
||||
private async onJavaScriptDialogOpening(tabId: number, params: unknown): Promise<void> {
|
||||
if (!this.attachedTabs.has(tabId) && !this.attachInFlight.has(tabId)) return;
|
||||
const parsed = parseDialogOpeningParams(params);
|
||||
try {
|
||||
if (
|
||||
this.options.shouldAutoAcceptDialog &&
|
||||
!(await this.options.shouldAutoAcceptDialog(tabId))
|
||||
) {
|
||||
return;
|
||||
}
|
||||
// The tab may have been returned while its current scope was checked.
|
||||
if (!this.attachedTabs.has(tabId) && !this.attachInFlight.has(tabId)) return;
|
||||
const handleParams: { accept: boolean; promptText?: string } = { accept: true };
|
||||
if (parsed.type === "prompt") {
|
||||
handleParams.promptText = parsed.defaultPrompt ?? "";
|
||||
}
|
||||
await this.api.sendCommand({ tabId }, "Page.handleJavaScriptDialog", handleParams);
|
||||
if (!this.attachedTabs.has(tabId) && !this.attachInFlight.has(tabId)) return;
|
||||
const sequence = (this.dialogSequences.get(tabId) ?? 0) + 1;
|
||||
this.dialogSequences.set(tabId, sequence);
|
||||
this.appendDialog(tabId, {
|
||||
@@ -835,15 +878,9 @@ export class ChromiumCdp {
|
||||
|
||||
/** Detach tabs only when no other live session has claimed them. */
|
||||
async detachSession(sessionId: string): Promise<void> {
|
||||
const tabsToDetach: number[] = [];
|
||||
for (const [tabId, owners] of this.tabOwners) {
|
||||
owners.delete(sessionId);
|
||||
if (owners.size === 0) {
|
||||
this.tabOwners.delete(tabId);
|
||||
tabsToDetach.push(tabId);
|
||||
}
|
||||
}
|
||||
await Promise.all(tabsToDetach.map((tabId) => this.detach(tabId)));
|
||||
await Promise.all(
|
||||
Array.from(this.tabOwners.keys(), (tabId) => this.releaseSessionTab(sessionId, tabId)),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -23,7 +23,7 @@ describe("BorrowConfirmationOverlay", () => {
|
||||
id: "req-1",
|
||||
isActiveTab: true,
|
||||
tabTitle: "Example",
|
||||
timeoutMs: 5000,
|
||||
timeoutMs: 60_000,
|
||||
onAllow,
|
||||
onDeny,
|
||||
},
|
||||
@@ -32,7 +32,7 @@ describe("BorrowConfirmationOverlay", () => {
|
||||
);
|
||||
|
||||
expect(screen.getByText("允许借用标签页?")).toBeTruthy();
|
||||
for (let i = 0; i < 5; i++) {
|
||||
for (let i = 0; i < 60; i++) {
|
||||
await act(async () => {
|
||||
await vi.advanceTimersByTimeAsync(1000);
|
||||
});
|
||||
@@ -66,7 +66,7 @@ describe("BorrowConfirmationOverlay", () => {
|
||||
id: "req-fallback",
|
||||
isActiveTab: true,
|
||||
tabTitle: "Example",
|
||||
timeoutMs: 5000,
|
||||
timeoutMs: 60_000,
|
||||
onAllow,
|
||||
onDeny,
|
||||
},
|
||||
@@ -74,7 +74,7 @@ describe("BorrowConfirmationOverlay", () => {
|
||||
/>,
|
||||
);
|
||||
|
||||
for (let i = 0; i < 5; i++) {
|
||||
for (let i = 0; i < 60; i++) {
|
||||
await act(async () => {
|
||||
await vi.advanceTimersByTimeAsync(1000);
|
||||
});
|
||||
|
||||
@@ -1,10 +1,12 @@
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { RECORD_STOP, type RecordStepPayload } from "@/lib/record-bridge";
|
||||
import { handleRecordContentMessage, startRecordCapture } from "../record-capture";
|
||||
import type { RecordStepPayload } from "@/lib/record-bridge";
|
||||
import { startRecordCapture } from "../record-capture";
|
||||
|
||||
vi.stubGlobal("chrome", {
|
||||
runtime: {
|
||||
sendMessage: vi.fn(() => Promise.resolve()),
|
||||
sendMessage: vi.fn((message: { sequence?: number }) =>
|
||||
Promise.resolve({ ok: true, sequence: message.sequence }),
|
||||
),
|
||||
},
|
||||
});
|
||||
|
||||
@@ -46,58 +48,6 @@ function click(el: Element): void {
|
||||
el.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 }));
|
||||
}
|
||||
|
||||
describe("handleRecordContentMessage stop/cancel", () => {
|
||||
it("ignores STOP when no recording is active", () => {
|
||||
const dispose = vi.fn();
|
||||
const onStop = vi.fn();
|
||||
const setActiveRequestId = vi.fn();
|
||||
const setCapture = vi.fn();
|
||||
const sendResponse = vi.fn();
|
||||
|
||||
const needsAsync = handleRecordContentMessage(
|
||||
{ type: RECORD_STOP, requestId: "rec-stale" },
|
||||
{
|
||||
activeRequestId: null,
|
||||
capture: { dispose },
|
||||
setActiveRequestId,
|
||||
setCapture,
|
||||
onStart: vi.fn(),
|
||||
onStop,
|
||||
},
|
||||
sendResponse,
|
||||
);
|
||||
|
||||
expect(needsAsync).toBe(false);
|
||||
expect(dispose).not.toHaveBeenCalled();
|
||||
expect(onStop).not.toHaveBeenCalled();
|
||||
expect(setActiveRequestId).not.toHaveBeenCalled();
|
||||
expect(setCapture).not.toHaveBeenCalled();
|
||||
expect(sendResponse).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("ignores STOP for a mismatched requestId", () => {
|
||||
const dispose = vi.fn();
|
||||
const onStop = vi.fn();
|
||||
|
||||
const needsAsync = handleRecordContentMessage(
|
||||
{ type: RECORD_STOP, requestId: "rec-other" },
|
||||
{
|
||||
activeRequestId: "rec-1",
|
||||
capture: { dispose },
|
||||
setActiveRequestId: vi.fn(),
|
||||
setCapture: vi.fn(),
|
||||
onStart: vi.fn(),
|
||||
onStop,
|
||||
},
|
||||
vi.fn(),
|
||||
);
|
||||
|
||||
expect(needsAsync).toBe(false);
|
||||
expect(dispose).not.toHaveBeenCalled();
|
||||
expect(onStop).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe("record-capture semantic", () => {
|
||||
let steps: RecordStepPayload[];
|
||||
|
||||
@@ -174,6 +124,19 @@ describe("record-capture semantic", () => {
|
||||
]);
|
||||
});
|
||||
|
||||
it("does not emit page navigation from a child Document capture", () => {
|
||||
const originalUrl = location.href;
|
||||
const capture = startRecordCapture("rec-child", (step) => steps.push(step), {
|
||||
captureNavigation: false,
|
||||
});
|
||||
|
||||
history.pushState({}, "", "#inside-frame");
|
||||
expect(steps).toEqual([]);
|
||||
|
||||
capture.dispose();
|
||||
history.replaceState({}, "", originalUrl);
|
||||
});
|
||||
|
||||
it("does not record clicks on anonymous layout divs", () => {
|
||||
document.body.innerHTML = `
|
||||
<div id="chrome">page chrome</div>
|
||||
@@ -228,6 +191,7 @@ describe("record-capture semantic", () => {
|
||||
expect(steps[0]).toMatchObject({
|
||||
op: "hover",
|
||||
target: { role: "button", name: "Open user navigation menu" },
|
||||
geometry: { tag: "button", rect: { x: 900, y: 8, w: 32, h: 32 } },
|
||||
});
|
||||
expect(steps[1]).toMatchObject({
|
||||
op: "click",
|
||||
|
||||
@@ -0,0 +1,124 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import type { RecordFramePortMessage } from "@/lib/recording/frame-bridge";
|
||||
import { RECORD_FRAME_START } from "@/lib/recording/frame-bridge";
|
||||
import { RECORD_DOCUMENT_ATTRIBUTE } from "@/shared/recording-document-identity";
|
||||
import { RecordFrameAgent } from "../recording/frame-agent";
|
||||
|
||||
class PortListeners<T extends (...args: never[]) => unknown> {
|
||||
readonly values = new Set<T>();
|
||||
addListener = (listener: T) => this.values.add(listener);
|
||||
removeListener = (listener: T) => this.values.delete(listener);
|
||||
}
|
||||
|
||||
function portHarness() {
|
||||
const onMessage = new PortListeners<(message: unknown) => void>();
|
||||
const onDisconnect = new PortListeners<() => void>();
|
||||
const outbound: RecordFramePortMessage[] = [];
|
||||
const port = {
|
||||
name: "bsk-record-frame",
|
||||
onMessage,
|
||||
onDisconnect,
|
||||
postMessage(message: RecordFramePortMessage) {
|
||||
outbound.push(message);
|
||||
if (message.type === "ready") {
|
||||
queueMicrotask(() => {
|
||||
for (const listener of onMessage.values) {
|
||||
listener({
|
||||
type: "ready_ack",
|
||||
requestId: message.requestId,
|
||||
producerId: message.producerId,
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
},
|
||||
disconnect: vi.fn(),
|
||||
} as unknown as chrome.runtime.Port;
|
||||
return {
|
||||
port,
|
||||
outbound,
|
||||
receive(message: RecordFramePortMessage) {
|
||||
for (const listener of onMessage.values) listener(message);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe("RecordFrameAgent", () => {
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals();
|
||||
document.documentElement.removeAttribute(RECORD_DOCUMENT_ATTRIBUTE);
|
||||
document.body.replaceChildren();
|
||||
});
|
||||
|
||||
it("starts when crypto.randomUUID is unavailable", async () => {
|
||||
const harness = portHarness();
|
||||
const getRandomValues = vi.fn((bytes: Uint8Array) => {
|
||||
bytes.fill(0x11);
|
||||
return bytes;
|
||||
});
|
||||
vi.stubGlobal("crypto", { getRandomValues });
|
||||
vi.stubGlobal("chrome", {
|
||||
runtime: {
|
||||
connect: vi.fn(() => harness.port),
|
||||
sendMessage: vi.fn(),
|
||||
},
|
||||
});
|
||||
const agent = new RecordFrameAgent();
|
||||
|
||||
await expect(
|
||||
agent.start({ type: RECORD_FRAME_START, requestId: "rec-http", startedAtMs: 10 }),
|
||||
).resolves.toEqual({ ok: true });
|
||||
expect(getRandomValues).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("keeps a failed stop retryable and flushes the final dirty fill", async () => {
|
||||
const harness = portHarness();
|
||||
const sendMessage = vi
|
||||
.fn<(message: { sequence: number }) => Promise<unknown>>()
|
||||
.mockRejectedValueOnce(new Error("offline"))
|
||||
.mockRejectedValueOnce(new Error("still offline"))
|
||||
.mockImplementation(async (message) => ({ ok: true, sequence: message.sequence }));
|
||||
vi.stubGlobal("chrome", {
|
||||
runtime: {
|
||||
connect: vi.fn(() => harness.port),
|
||||
sendMessage,
|
||||
},
|
||||
});
|
||||
document.body.innerHTML = `<label for="draft">Draft</label><input id="draft" />`;
|
||||
const agent = new RecordFrameAgent();
|
||||
|
||||
await expect(
|
||||
agent.start({ type: RECORD_FRAME_START, requestId: "rec-1", startedAtMs: 10 }),
|
||||
).resolves.toEqual({ ok: true });
|
||||
expect(document.documentElement.hasAttribute(RECORD_DOCUMENT_ATTRIBUTE)).toBe(true);
|
||||
|
||||
const input = document.querySelector("input")!;
|
||||
input.dispatchEvent(new FocusEvent("focusin", { bubbles: true }));
|
||||
input.value = "final value";
|
||||
input.dispatchEvent(new Event("input", { bubbles: true }));
|
||||
|
||||
harness.receive({ type: "stop", requestId: "rec-1", commandId: "stop-1" });
|
||||
await vi.waitFor(() =>
|
||||
expect(harness.outbound).toContainEqual({
|
||||
type: "stopped",
|
||||
requestId: "rec-1",
|
||||
commandId: "stop-1",
|
||||
ok: false,
|
||||
error: "failed to deliver one or more recorded steps",
|
||||
}),
|
||||
);
|
||||
expect(document.documentElement.hasAttribute(RECORD_DOCUMENT_ATTRIBUTE)).toBe(true);
|
||||
|
||||
harness.receive({ type: "stop", requestId: "rec-1", commandId: "stop-2" });
|
||||
await vi.waitFor(() =>
|
||||
expect(harness.outbound).toContainEqual({
|
||||
type: "stopped",
|
||||
requestId: "rec-1",
|
||||
commandId: "stop-2",
|
||||
ok: true,
|
||||
}),
|
||||
);
|
||||
expect(sendMessage).toHaveBeenCalledTimes(3);
|
||||
expect(document.documentElement.hasAttribute(RECORD_DOCUMENT_ATTRIBUTE)).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,36 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import type { RecordStepMessage } from "@/lib/record-bridge";
|
||||
import { RecordStepDelivery } from "../record-step-delivery";
|
||||
|
||||
describe("RecordStepDelivery", () => {
|
||||
it("retries from the first unacknowledged sequence before sending later steps", async () => {
|
||||
const sent: RecordStepMessage[] = [];
|
||||
const send = vi.fn(async (message: RecordStepMessage) => {
|
||||
sent.push(message);
|
||||
if (sent.length === 1) throw new Error("service worker unavailable");
|
||||
return { ok: true, sequence: message.sequence };
|
||||
});
|
||||
const delivery = new RecordStepDelivery("rec-ordered", send, "document-1");
|
||||
|
||||
delivery.enqueue({ op: "click", target: { tag: "button", name: "First" } });
|
||||
delivery.enqueue({ op: "click", target: { tag: "button", name: "Second" } });
|
||||
|
||||
await expect(delivery.flush()).resolves.toBe(true);
|
||||
expect(sent.map((message) => message.sequence)).toEqual([1, 1, 2]);
|
||||
expect(sent.map((message) => message.step.target?.name)).toEqual(["First", "First", "Second"]);
|
||||
});
|
||||
|
||||
it("keeps unacknowledged steps pending for a later flush", async () => {
|
||||
const send = vi
|
||||
.fn<(message: RecordStepMessage) => Promise<unknown>>()
|
||||
.mockRejectedValueOnce(new Error("offline"))
|
||||
.mockRejectedValueOnce(new Error("still offline"))
|
||||
.mockImplementation(async (message) => ({ ok: true, sequence: message.sequence }));
|
||||
const delivery = new RecordStepDelivery("rec-retry", send, "document-1");
|
||||
delivery.enqueue({ op: "click", target: { tag: "button", name: "Save" } });
|
||||
|
||||
await expect(delivery.flush()).resolves.toBe(false);
|
||||
await expect(delivery.flush()).resolves.toBe(true);
|
||||
expect(send).toHaveBeenCalledTimes(3);
|
||||
});
|
||||
});
|
||||
@@ -12,21 +12,7 @@ import {
|
||||
hasDirectHoverInteractiveSignal,
|
||||
hasStrongHoverExpansionSignal,
|
||||
} from "@/lib/hover-trigger-policy";
|
||||
import {
|
||||
isRecordCancelMessage,
|
||||
isRecordStartMessage,
|
||||
isRecordStopMessage,
|
||||
RECORD_CANCEL,
|
||||
RECORD_START,
|
||||
RECORD_STEP,
|
||||
RECORD_STOP,
|
||||
type RecordCancelMessage,
|
||||
type RecordStartAck,
|
||||
type RecordStartMessage,
|
||||
type RecordStepPayload,
|
||||
type RecordStopAck,
|
||||
type RecordStopMessage,
|
||||
} from "@/lib/record-bridge";
|
||||
import type { RecordStepPayload } from "@/lib/record-bridge";
|
||||
import { shouldRecordPress } from "@/lib/recording/draft-policy";
|
||||
import {
|
||||
closestHoverSurfaceCandidate,
|
||||
@@ -37,28 +23,6 @@ import {
|
||||
isLikelyHoverSurfaceOwner,
|
||||
} from "./record-hover-surface";
|
||||
|
||||
const pendingStepSends = new Map<string, Set<Promise<boolean>>>();
|
||||
const failedStepDeliveries = new Set<string>();
|
||||
const knownRecordRequests = new Set<string>();
|
||||
|
||||
function sendRecordStep(requestId: string, step: RecordStepPayload): void {
|
||||
const payload = { type: RECORD_STEP, requestId, step };
|
||||
const pending = Promise.resolve(chrome.runtime.sendMessage(payload)).then(
|
||||
() => true,
|
||||
() => {
|
||||
failedStepDeliveries.add(requestId);
|
||||
return false;
|
||||
},
|
||||
);
|
||||
const sends = pendingStepSends.get(requestId) ?? new Set<Promise<boolean>>();
|
||||
sends.add(pending);
|
||||
pendingStepSends.set(requestId, sends);
|
||||
void pending.then(() => {
|
||||
sends.delete(pending);
|
||||
if (sends.size === 0) pendingStepSends.delete(requestId);
|
||||
});
|
||||
}
|
||||
|
||||
export interface RecordCaptureController {
|
||||
dispose(): void;
|
||||
}
|
||||
@@ -68,6 +32,7 @@ interface FillSession {
|
||||
target: CaptureTargetDescriptor;
|
||||
baselineValue: string;
|
||||
lastValue: string;
|
||||
pendingCommit?: "enter" | "suggestion" | "blur";
|
||||
}
|
||||
|
||||
interface HoverCandidate {
|
||||
@@ -157,6 +122,24 @@ function nearbyFillableFromSearchChrome(target: Element): FillableElement | null
|
||||
return null;
|
||||
}
|
||||
|
||||
function captureGeometry(el: Element): RecordStepPayload["geometry"] {
|
||||
const rect = el.getBoundingClientRect();
|
||||
return {
|
||||
rect: { x: rect.x, y: rect.y, w: rect.width, h: rect.height },
|
||||
tag: el.tagName.toLowerCase(),
|
||||
};
|
||||
}
|
||||
|
||||
function geometryForEventTarget(
|
||||
target: EventTarget | null,
|
||||
): RecordStepPayload["geometry"] | undefined {
|
||||
if (!(target instanceof Element)) return undefined;
|
||||
const clickable = target.closest(
|
||||
'a,button,input,select,textarea,[role="button"],[role="link"],[role="menuitem"],[contenteditable="true"]',
|
||||
);
|
||||
return captureGeometry(clickable ?? target);
|
||||
}
|
||||
|
||||
function fillableValue(el: FillableElement): string {
|
||||
if (el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement) {
|
||||
return el.value;
|
||||
@@ -376,6 +359,7 @@ function scheduleInputCompletionCommit(
|
||||
export function startRecordCapture(
|
||||
_requestId: string,
|
||||
sendStep: (step: RecordStepPayload) => void,
|
||||
options: { captureNavigation?: boolean } = {},
|
||||
): RecordCaptureController {
|
||||
const emitStep = (step: RecordStepPayload) => {
|
||||
sendStep({ page_url: location.href, ...step });
|
||||
@@ -414,14 +398,17 @@ export function startRecordCapture(
|
||||
emitStep({
|
||||
op: "fill",
|
||||
target: session.target,
|
||||
geometry: captureGeometry(session.element),
|
||||
value,
|
||||
commit: session.pendingCommit ?? "blur",
|
||||
...(isPassword ? { redacted: true } : {}),
|
||||
});
|
||||
};
|
||||
|
||||
const commitFillSession = () => {
|
||||
const commitFillSession = (commit: "enter" | "suggestion" | "blur" = "blur") => {
|
||||
if (!fillSession || composing) return;
|
||||
const session = fillSession;
|
||||
session.pendingCommit = commit;
|
||||
fillSession = null;
|
||||
emitFill(session);
|
||||
committedValues.set(session.element, session.lastValue);
|
||||
@@ -627,6 +614,7 @@ export function startRecordCapture(
|
||||
emitStep({
|
||||
op: "hover",
|
||||
target: hover.target,
|
||||
geometry: captureGeometry(hover.element),
|
||||
});
|
||||
emittedHoverElements.add(hover.element);
|
||||
};
|
||||
@@ -681,6 +669,7 @@ export function startRecordCapture(
|
||||
emitStep({
|
||||
op: "click",
|
||||
target,
|
||||
geometry: geometryForEventTarget(eventTarget(event)),
|
||||
expects_navigation: true,
|
||||
});
|
||||
};
|
||||
@@ -782,7 +771,7 @@ export function startRecordCapture(
|
||||
scheduleInputCompletionCommit(
|
||||
sessionElement,
|
||||
syncFillSessionValue,
|
||||
commitFillSession,
|
||||
() => commitFillSession("suggestion"),
|
||||
(el) => fillSession?.element === el,
|
||||
);
|
||||
return;
|
||||
@@ -842,6 +831,7 @@ export function startRecordCapture(
|
||||
emitStep({
|
||||
op: "select",
|
||||
target: desc,
|
||||
geometry: captureGeometry(target),
|
||||
values,
|
||||
labels,
|
||||
expects_navigation: true,
|
||||
@@ -882,7 +872,7 @@ export function startRecordCapture(
|
||||
};
|
||||
}
|
||||
if (fillable) {
|
||||
commitFillSession();
|
||||
commitFillSession(event.key === "Enter" ? "enter" : "blur");
|
||||
}
|
||||
const desc = describeEventTarget(target);
|
||||
if (!desc && !event.key) return;
|
||||
@@ -890,7 +880,7 @@ export function startRecordCapture(
|
||||
emitStep({
|
||||
op: "press",
|
||||
key: event.key,
|
||||
...(desc ? { target: desc } : {}),
|
||||
...(desc ? { target: desc, geometry: geometryForEventTarget(target) } : {}),
|
||||
...(modifiers.length ? { modifiers } : {}),
|
||||
expects_navigation: event.key === "Enter",
|
||||
});
|
||||
@@ -906,23 +896,33 @@ export function startRecordCapture(
|
||||
document.addEventListener("change", onChange, true);
|
||||
document.addEventListener("keydown", onKeyDown, true);
|
||||
|
||||
const urlObserver = new MutationObserver(() => emitNavigateIfChanged());
|
||||
urlObserver.observe(document, { subtree: true, childList: true });
|
||||
const captureNavigation = options.captureNavigation ?? true;
|
||||
const urlObserver = captureNavigation
|
||||
? new MutationObserver(() => emitNavigateIfChanged())
|
||||
: null;
|
||||
urlObserver?.observe(document, { subtree: true, childList: true });
|
||||
const onUrlEvent = () => emitNavigateIfChanged();
|
||||
window.addEventListener("hashchange", onUrlEvent);
|
||||
window.addEventListener("popstate", onUrlEvent);
|
||||
window.addEventListener("pagehide", commitFillSession);
|
||||
// Wrap: passing commitFillSession directly would forward the DOM event as
|
||||
// the `commit` argument and stamp it onto the recorded fill step.
|
||||
const onPageHide = () => commitFillSession();
|
||||
if (captureNavigation) {
|
||||
window.addEventListener("hashchange", onUrlEvent);
|
||||
window.addEventListener("popstate", onUrlEvent);
|
||||
}
|
||||
window.addEventListener("pagehide", onPageHide);
|
||||
|
||||
const originalPushState = history.pushState;
|
||||
const originalReplaceState = history.replaceState;
|
||||
history.pushState = function (...args: Parameters<History["pushState"]>) {
|
||||
originalPushState.apply(this, args);
|
||||
emitNavigateIfChanged(navigationActionPending ? true : undefined);
|
||||
};
|
||||
history.replaceState = function (...args: Parameters<History["replaceState"]>) {
|
||||
originalReplaceState.apply(this, args);
|
||||
emitNavigateIfChanged(navigationActionPending ? true : undefined);
|
||||
};
|
||||
if (captureNavigation) {
|
||||
history.pushState = function (...args: Parameters<History["pushState"]>) {
|
||||
originalPushState.apply(this, args);
|
||||
emitNavigateIfChanged(navigationActionPending ? true : undefined);
|
||||
};
|
||||
history.replaceState = function (...args: Parameters<History["replaceState"]>) {
|
||||
originalReplaceState.apply(this, args);
|
||||
emitNavigateIfChanged(navigationActionPending ? true : undefined);
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
dispose() {
|
||||
@@ -936,93 +936,16 @@ export function startRecordCapture(
|
||||
document.removeEventListener("compositionend", onCompositionEnd, true);
|
||||
document.removeEventListener("change", onChange, true);
|
||||
document.removeEventListener("keydown", onKeyDown, true);
|
||||
urlObserver.disconnect();
|
||||
window.removeEventListener("hashchange", onUrlEvent);
|
||||
window.removeEventListener("popstate", onUrlEvent);
|
||||
window.removeEventListener("pagehide", commitFillSession);
|
||||
history.pushState = originalPushState;
|
||||
history.replaceState = originalReplaceState;
|
||||
urlObserver?.disconnect();
|
||||
if (captureNavigation) {
|
||||
window.removeEventListener("hashchange", onUrlEvent);
|
||||
window.removeEventListener("popstate", onUrlEvent);
|
||||
}
|
||||
window.removeEventListener("pagehide", onPageHide);
|
||||
if (captureNavigation) {
|
||||
history.pushState = originalPushState;
|
||||
history.replaceState = originalReplaceState;
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export type RecordContentMessage = RecordStartMessage | RecordStopMessage | RecordCancelMessage;
|
||||
|
||||
export function isRecordContentMessage(msg: unknown): msg is RecordContentMessage {
|
||||
return isRecordStartMessage(msg) || isRecordStopMessage(msg) || isRecordCancelMessage(msg);
|
||||
}
|
||||
|
||||
export function handleRecordContentMessage(
|
||||
message: RecordContentMessage,
|
||||
state: {
|
||||
activeRequestId: string | null;
|
||||
capture: RecordCaptureController | null;
|
||||
setActiveRequestId(id: string | null): void;
|
||||
setCapture(capture: RecordCaptureController | null): void;
|
||||
onStart(requestId: string, startedAtMs?: number): void;
|
||||
onStop(): void;
|
||||
},
|
||||
sendResponse?: (response: RecordStartAck | RecordStopAck) => void,
|
||||
): boolean {
|
||||
if (isRecordStartMessage(message)) {
|
||||
if (!knownRecordRequests.has(message.requestId)) {
|
||||
knownRecordRequests.add(message.requestId);
|
||||
failedStepDeliveries.delete(message.requestId);
|
||||
}
|
||||
state.capture?.dispose();
|
||||
state.setCapture(
|
||||
startRecordCapture(message.requestId, (step) => {
|
||||
sendRecordStep(message.requestId, step);
|
||||
}),
|
||||
);
|
||||
state.setActiveRequestId(message.requestId);
|
||||
state.onStart(message.requestId, message.startedAtMs);
|
||||
sendResponse?.({ ok: true });
|
||||
return sendResponse !== undefined;
|
||||
}
|
||||
|
||||
if (isRecordStopMessage(message) || isRecordCancelMessage(message)) {
|
||||
// Require an active recording that matches this requestId — otherwise a
|
||||
// stray STOP/CANCEL (e.g. after teardown) would still run finishStop and
|
||||
// clear overlay state even though nothing was capturing.
|
||||
if (!state.activeRequestId || state.activeRequestId !== message.requestId) {
|
||||
return false;
|
||||
}
|
||||
state.capture?.dispose();
|
||||
state.setCapture(null);
|
||||
const finishStop = () => {
|
||||
state.onStop();
|
||||
state.setActiveRequestId(null);
|
||||
};
|
||||
if (isRecordStopMessage(message) && sendResponse) {
|
||||
const pending = [...(pendingStepSends.get(message.requestId) ?? [])];
|
||||
void Promise.all(pending).then((delivered) => {
|
||||
finishStop();
|
||||
const succeeded = delivered.every(Boolean) && !failedStepDeliveries.has(message.requestId);
|
||||
if (succeeded) {
|
||||
failedStepDeliveries.delete(message.requestId);
|
||||
knownRecordRequests.delete(message.requestId);
|
||||
}
|
||||
sendResponse(
|
||||
succeeded
|
||||
? { ok: true }
|
||||
: {
|
||||
ok: false,
|
||||
error: "failed to deliver one or more recorded steps",
|
||||
},
|
||||
);
|
||||
});
|
||||
return true;
|
||||
}
|
||||
if (isRecordCancelMessage(message)) {
|
||||
failedStepDeliveries.delete(message.requestId);
|
||||
knownRecordRequests.delete(message.requestId);
|
||||
}
|
||||
finishStop();
|
||||
return false;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
export { RECORD_CANCEL, RECORD_START, RECORD_STEP, RECORD_STOP };
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
import { createRandomUuid } from "@/lib/random-uuid";
|
||||
import {
|
||||
isAcceptedRecordStepAck,
|
||||
RECORD_STEP,
|
||||
type RecordStepMessage,
|
||||
type RecordStepPayload,
|
||||
} from "@/lib/record-bridge";
|
||||
|
||||
export type SendRecordStepMessage = (message: RecordStepMessage) => Promise<unknown>;
|
||||
|
||||
export class RecordStepDelivery {
|
||||
readonly #requestId: string;
|
||||
readonly #producerId: string;
|
||||
readonly #send: SendRecordStepMessage;
|
||||
readonly #pending = new Map<number, RecordStepPayload>();
|
||||
#nextSequence = 1;
|
||||
#tail = Promise.resolve();
|
||||
|
||||
constructor(
|
||||
requestId: string,
|
||||
send: SendRecordStepMessage = (message) => chrome.runtime.sendMessage(message),
|
||||
producerId: string = createRandomUuid(),
|
||||
) {
|
||||
this.#requestId = requestId;
|
||||
this.#producerId = producerId;
|
||||
this.#send = send;
|
||||
}
|
||||
|
||||
enqueue(step: RecordStepPayload): void {
|
||||
const sequence = this.#nextSequence;
|
||||
this.#nextSequence += 1;
|
||||
this.#pending.set(sequence, step);
|
||||
void this.#schedule();
|
||||
}
|
||||
|
||||
async flush(): Promise<boolean> {
|
||||
await this.#tail;
|
||||
await this.#schedule();
|
||||
return this.#pending.size === 0;
|
||||
}
|
||||
|
||||
#schedule(): Promise<void> {
|
||||
const drain = () => this.#drain();
|
||||
this.#tail = this.#tail.then(drain, drain);
|
||||
return this.#tail;
|
||||
}
|
||||
|
||||
async #drain(): Promise<void> {
|
||||
for (const [sequence, step] of this.#pending) {
|
||||
const message: RecordStepMessage = {
|
||||
type: RECORD_STEP,
|
||||
requestId: this.#requestId,
|
||||
producerId: this.#producerId,
|
||||
sequence,
|
||||
step,
|
||||
};
|
||||
try {
|
||||
const response = await this.#send(message);
|
||||
if (!isAcceptedRecordStepAck(response, sequence)) return;
|
||||
this.#pending.delete(sequence);
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,210 @@
|
||||
import { createRandomUuid } from "@/lib/random-uuid";
|
||||
import {
|
||||
markRecordingDocument,
|
||||
type RecordingDocumentMarker,
|
||||
waitForDocumentElement,
|
||||
} from "@/lib/recording/document-marker";
|
||||
import {
|
||||
isRecordFrameStartMessage,
|
||||
RECORD_FRAME_PORT,
|
||||
RECORD_FRAME_QUERY,
|
||||
RECORD_FRAME_START,
|
||||
type RecordFramePortMessage,
|
||||
type RecordFrameQueryResponse,
|
||||
type RecordFrameStartMessage,
|
||||
} from "@/lib/recording/frame-bridge";
|
||||
import { type RecordCaptureController, startRecordCapture } from "../record-capture";
|
||||
import { RecordStepDelivery } from "../record-step-delivery";
|
||||
|
||||
interface ActiveFrameRecording {
|
||||
requestId: string;
|
||||
producerId: string;
|
||||
port: chrome.runtime.Port;
|
||||
delivery: RecordStepDelivery;
|
||||
capture: RecordCaptureController;
|
||||
marker: RecordingDocumentMarker;
|
||||
}
|
||||
|
||||
const RECORD_FRAME_REGISTER_TIMEOUT_MS = 5_000;
|
||||
|
||||
export class RecordFrameAgent {
|
||||
#active: ActiveFrameRecording | null = null;
|
||||
#startPromise: Promise<{ ok: true } | { ok: false; error: string }> | null = null;
|
||||
#disposed = false;
|
||||
|
||||
async start(
|
||||
message: RecordFrameStartMessage,
|
||||
): Promise<{ ok: true } | { ok: false; error: string }> {
|
||||
if (this.#active?.requestId === message.requestId) return { ok: true };
|
||||
if (this.#disposed) return { ok: false, error: "recording document is disposed" };
|
||||
if (this.#startPromise) {
|
||||
await this.#startPromise;
|
||||
return this.start(message);
|
||||
}
|
||||
if (this.#active) this.cancel(this.#active.requestId);
|
||||
const starting = this.#start(message);
|
||||
this.#startPromise = starting;
|
||||
try {
|
||||
return await starting;
|
||||
} finally {
|
||||
if (this.#startPromise === starting) this.#startPromise = null;
|
||||
}
|
||||
}
|
||||
|
||||
cancel(requestId: string): void {
|
||||
const active = this.#active;
|
||||
if (!active || active.requestId !== requestId) return;
|
||||
active.capture.dispose();
|
||||
active.marker.restore();
|
||||
active.port.disconnect();
|
||||
this.#active = null;
|
||||
}
|
||||
|
||||
dispose(): void {
|
||||
this.#disposed = true;
|
||||
if (this.#active) this.cancel(this.#active.requestId);
|
||||
}
|
||||
|
||||
async #start(
|
||||
message: RecordFrameStartMessage,
|
||||
): Promise<{ ok: true } | { ok: false; error: string }> {
|
||||
const producerId = createRandomUuid();
|
||||
const root = await waitForDocumentElement();
|
||||
if (this.#disposed) return { ok: false, error: "recording document is disposed" };
|
||||
const marker = markRecordingDocument(producerId, root);
|
||||
const port = chrome.runtime.connect({ name: RECORD_FRAME_PORT });
|
||||
try {
|
||||
await registerPort(port, message.requestId, producerId);
|
||||
} catch {
|
||||
marker.restore();
|
||||
port.disconnect();
|
||||
return { ok: false, error: "failed to register recording document" };
|
||||
}
|
||||
if (this.#disposed) {
|
||||
marker.restore();
|
||||
port.disconnect();
|
||||
return { ok: false, error: "recording document is disposed" };
|
||||
}
|
||||
|
||||
const delivery = new RecordStepDelivery(message.requestId, undefined, producerId);
|
||||
const capture = startRecordCapture(
|
||||
message.requestId,
|
||||
(step) => {
|
||||
marker.ensure();
|
||||
delivery.enqueue(step);
|
||||
},
|
||||
{ captureNavigation: window.top === window },
|
||||
);
|
||||
const active: ActiveFrameRecording = {
|
||||
requestId: message.requestId,
|
||||
producerId,
|
||||
port,
|
||||
delivery,
|
||||
capture,
|
||||
marker,
|
||||
};
|
||||
this.#active = active;
|
||||
port.onMessage.addListener((raw: unknown) => {
|
||||
const command = raw as Partial<RecordFramePortMessage>;
|
||||
if (command.type === "stop" && command.requestId === active.requestId) {
|
||||
void this.#stop(active, command.commandId);
|
||||
} else if (command.type === "cancel" && command.requestId === active.requestId) {
|
||||
this.cancel(active.requestId);
|
||||
}
|
||||
});
|
||||
port.onDisconnect.addListener(() => {
|
||||
if (this.#active !== active) return;
|
||||
active.capture.dispose();
|
||||
active.marker.restore();
|
||||
this.#active = null;
|
||||
});
|
||||
return { ok: true };
|
||||
}
|
||||
|
||||
async #stop(active: ActiveFrameRecording, commandId: string | undefined): Promise<void> {
|
||||
if (!commandId || this.#active !== active) return;
|
||||
active.capture.dispose();
|
||||
const succeeded = await active.delivery.flush();
|
||||
try {
|
||||
active.port.postMessage({
|
||||
type: "stopped",
|
||||
requestId: active.requestId,
|
||||
commandId,
|
||||
ok: succeeded,
|
||||
...(succeeded ? {} : { error: "failed to deliver one or more recorded steps" }),
|
||||
} satisfies RecordFramePortMessage);
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
if (!succeeded || this.#active !== active) return;
|
||||
active.marker.restore();
|
||||
this.#active = null;
|
||||
active.port.disconnect();
|
||||
}
|
||||
}
|
||||
|
||||
function registerPort(
|
||||
port: chrome.runtime.Port,
|
||||
requestId: string,
|
||||
producerId: string,
|
||||
): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const timer = setTimeout(() => {
|
||||
cleanup();
|
||||
reject(new Error("recording document registration timed out"));
|
||||
}, RECORD_FRAME_REGISTER_TIMEOUT_MS);
|
||||
const cleanup = () => {
|
||||
clearTimeout(timer);
|
||||
port.onMessage.removeListener(onMessage);
|
||||
port.onDisconnect.removeListener(onDisconnect);
|
||||
};
|
||||
const onMessage = (raw: unknown) => {
|
||||
const message = raw as Partial<RecordFramePortMessage>;
|
||||
if (
|
||||
message.type !== "ready_ack" ||
|
||||
message.requestId !== requestId ||
|
||||
message.producerId !== producerId
|
||||
) {
|
||||
return;
|
||||
}
|
||||
cleanup();
|
||||
resolve();
|
||||
};
|
||||
const onDisconnect = () => {
|
||||
cleanup();
|
||||
reject(new Error("recording document port disconnected"));
|
||||
};
|
||||
port.onMessage.addListener(onMessage);
|
||||
port.onDisconnect.addListener(onDisconnect);
|
||||
port.postMessage({ type: "ready", requestId, producerId } satisfies RecordFramePortMessage);
|
||||
});
|
||||
}
|
||||
|
||||
export function attachRecordFrameAgent(): () => void {
|
||||
const agent = new RecordFrameAgent();
|
||||
const onMessage = (
|
||||
message: unknown,
|
||||
_sender: chrome.runtime.MessageSender,
|
||||
sendResponse: (response: { ok: true } | { ok: false; error: string }) => void,
|
||||
) => {
|
||||
if (!isRecordFrameStartMessage(message)) return false;
|
||||
void agent.start(message).then(sendResponse);
|
||||
return true;
|
||||
};
|
||||
chrome.runtime.onMessage.addListener(onMessage);
|
||||
void chrome.runtime
|
||||
.sendMessage({ type: RECORD_FRAME_QUERY })
|
||||
.then((response: RecordFrameQueryResponse | undefined) => {
|
||||
if (!response?.active || !response.requestId || response.startedAtMs === undefined) return;
|
||||
return agent.start({
|
||||
type: RECORD_FRAME_START,
|
||||
requestId: response.requestId,
|
||||
startedAtMs: response.startedAtMs,
|
||||
});
|
||||
})
|
||||
.catch(() => {});
|
||||
return () => {
|
||||
chrome.runtime.onMessage.removeListener(onMessage);
|
||||
agent.dispose();
|
||||
};
|
||||
}
|
||||
@@ -21,10 +21,11 @@ import {
|
||||
type OverlayMode,
|
||||
} from "@/lib/overlay-bridge";
|
||||
import { POPUP_PORT_NAME, type PopupInbound, type PopupOutbound } from "@/lib/popup-bridge";
|
||||
import { recordFrameCoordinator } from "@/lib/recording/frame-coordinator";
|
||||
import { attachSessionsLiveFlag } from "@/lib/sessions-live-flag";
|
||||
import { createDisconnectCleanup } from "@/session-manager/disconnect-cleanup";
|
||||
import { attachSessionEventHandler } from "@/session-manager/event-handler";
|
||||
import { SessionManager } from "@/session-manager/manager";
|
||||
import { isAgentControlledTab, SessionManager } from "@/session-manager/manager";
|
||||
import {
|
||||
attachBorrowNotificationButtonHandler,
|
||||
attachBorrowNotificationClickHandler,
|
||||
@@ -38,7 +39,10 @@ import {
|
||||
attachRecordFinishListener,
|
||||
attachRecordQueryListener,
|
||||
attachRecordStepListener,
|
||||
type RecordRuntimeDeps,
|
||||
} from "@/tools/record";
|
||||
import { chromeTabsApi } from "@/tools/shared";
|
||||
import { chromeTabMutationApi } from "@/tools/tabs";
|
||||
import { detectBrowserMeta } from "@/transport/handshake";
|
||||
import type { Transport } from "@/transport/transport";
|
||||
import { WSTransport } from "@/transport/ws-transport";
|
||||
@@ -47,7 +51,12 @@ export default defineBackground(() => {
|
||||
const controller = new ConnectionController();
|
||||
const transport = new WSTransport({ url: __BSK_DAEMON_WS_URL__ });
|
||||
const sessions = new SessionManager();
|
||||
const cdp = new ChromiumCdp();
|
||||
const cdp = new ChromiumCdp(undefined, {
|
||||
shouldAutoAcceptDialog: async (tabId) => {
|
||||
const tab = await chrome.tabs.get(tabId);
|
||||
return sessions.findByWindowId(tab.windowId) !== null;
|
||||
},
|
||||
});
|
||||
const sessionsLive = attachSessionsLiveFlag({ manager: sessions });
|
||||
let overlayGeneration = 0;
|
||||
const controlModes = new Map<string, OverlayMode>();
|
||||
@@ -78,6 +87,24 @@ export default defineBackground(() => {
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Authoritative overlay state for a specific tab. Agent Window tabs are
|
||||
* free by default; only tabs explicitly claimed through session startup,
|
||||
* `tab_create`, or `tab_borrow` receive the control overlay.
|
||||
*/
|
||||
function overlayStateForTab(tabId?: number, windowId?: number): OverlayAgentStateMessage {
|
||||
if (typeof tabId === "number" && typeof windowId === "number") {
|
||||
const ctx = sessions.findByWindowId(windowId);
|
||||
if (ctx && isAgentControlledTab(ctx, tabId)) return overlayStateForWindow(windowId);
|
||||
}
|
||||
return {
|
||||
type: OVERLAY_AGENT_STATE,
|
||||
sessionId: null,
|
||||
mode: "hidden",
|
||||
generation: overlayGeneration,
|
||||
};
|
||||
}
|
||||
|
||||
async function pushOverlayStateToTab(
|
||||
tabId: number,
|
||||
state: OverlayAgentStateMessage,
|
||||
@@ -89,13 +116,18 @@ export default defineBackground(() => {
|
||||
}
|
||||
}
|
||||
|
||||
async function pushOverlayStateForTab(tabId: number, windowId?: number): Promise<void> {
|
||||
const state = overlayStateForTab(tabId, windowId);
|
||||
await pushOverlayStateToTab(tabId, state);
|
||||
}
|
||||
|
||||
async function pushOverlayStateForWindow(windowId: number): Promise<void> {
|
||||
const state = overlayStateForWindow(windowId);
|
||||
const tabs = await chrome.tabs.query({ windowId });
|
||||
await Promise.all(
|
||||
tabs.map((tab) =>
|
||||
typeof tab.id === "number" ? pushOverlayStateToTab(tab.id, state) : Promise.resolve(),
|
||||
),
|
||||
tabs.map((tab) => {
|
||||
if (typeof tab.id !== "number") return Promise.resolve();
|
||||
return pushOverlayStateForTab(tab.id, windowId);
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
@@ -134,23 +166,66 @@ export default defineBackground(() => {
|
||||
if (typeof tab.windowId !== "number") return;
|
||||
pushOverlayStateForAgentWindow(tab.windowId);
|
||||
});
|
||||
// Newly observed tabs are free unless the agent's own creation path has
|
||||
// already claimed their concrete id. This listener only renders that
|
||||
// state; it never infers or mutates ownership from event ordering.
|
||||
chrome.tabs.onCreated.addListener((tab) => {
|
||||
if (typeof tab.windowId !== "number" || typeof tab.id !== "number") return;
|
||||
if (!sessions.findByWindowId(tab.windowId)) return;
|
||||
void pushOverlayStateForTab(tab.id, tab.windowId);
|
||||
});
|
||||
chrome.tabs.onRemoved.addListener((tabId, removeInfo) => {
|
||||
sessions.forgetClosedTab(tabId, { isWindowClosing: removeInfo.isWindowClosing });
|
||||
});
|
||||
// Re-sync the storage.session flag on SW startup so a previous SW's
|
||||
// stale `true` does not keep waking us on every page load until the
|
||||
// first mutation (review M4/M5 round 3 m-R3-1).
|
||||
void sessionsLive.refresh();
|
||||
const cleanupAfterDisconnect = createDisconnectCleanup({
|
||||
manager: sessions,
|
||||
sessionStopDeps: { cdp },
|
||||
// Same contract as the dispatcher's `tool.session_stop`: without these
|
||||
// deps the agent-tab cleanup and the window-release path are dead code.
|
||||
sessionStopDeps: {
|
||||
cdp,
|
||||
tabManagement: { tabs: chromeTabMutationApi },
|
||||
tabsQuery: chromeTabsApi,
|
||||
},
|
||||
onSessionsChanged: () => {
|
||||
void sessionsLive.syncFromManager();
|
||||
},
|
||||
});
|
||||
const recordDeps = {
|
||||
tabsApi: chrome.tabs,
|
||||
cdp,
|
||||
frameCoordinator: recordFrameCoordinator,
|
||||
sendToTab: (tabId: number, msg: Parameters<typeof chrome.tabs.sendMessage>[1]) =>
|
||||
chrome.tabs.sendMessage(tabId, msg),
|
||||
bypassOverlay: async (tabId: number, enabled: boolean) => {
|
||||
try {
|
||||
await chrome.tabs.sendMessage(tabId, {
|
||||
type: OVERLAY_AUTOMATION_BYPASS,
|
||||
enabled,
|
||||
});
|
||||
} catch {
|
||||
// Content script may be unavailable on restricted pages.
|
||||
}
|
||||
},
|
||||
} satisfies RecordRuntimeDeps;
|
||||
recordFrameCoordinator.attach();
|
||||
attachRecordStepListener(recordDeps);
|
||||
attachRecordFinishListener(recordDeps);
|
||||
attachRecordQueryListener(recordDeps);
|
||||
|
||||
const dispatcher = new ToolDispatcher({
|
||||
transport,
|
||||
sessions,
|
||||
cdp,
|
||||
recording: recordDeps,
|
||||
onSessionsChanged: onOverlaySessionStateChanged,
|
||||
onBrowserControlResumed,
|
||||
onAgentTabClaimed: (tabId, windowId) => {
|
||||
void pushOverlayStateForTab(tabId, windowId);
|
||||
},
|
||||
approveBorrow: (ctx) =>
|
||||
requestBorrowConfirmation(ctx.tabId, {
|
||||
...(ctx.signal !== undefined ? { signal: ctx.signal } : {}),
|
||||
@@ -170,27 +245,6 @@ export default defineBackground(() => {
|
||||
}),
|
||||
});
|
||||
dispatcher.start();
|
||||
const recordDeps = {
|
||||
tabsApi: chrome.tabs,
|
||||
sendToTab: (tabId: number, msg: Parameters<typeof chrome.tabs.sendMessage>[1]) =>
|
||||
chrome.tabs.sendMessage(tabId, msg),
|
||||
bypassOverlay: async (tabId: number, enabled: boolean) => {
|
||||
try {
|
||||
await chrome.tabs.sendMessage(tabId, {
|
||||
type: OVERLAY_AUTOMATION_BYPASS,
|
||||
enabled,
|
||||
});
|
||||
} catch {
|
||||
// Content script may be unavailable on restricted pages.
|
||||
}
|
||||
},
|
||||
};
|
||||
// Message listeners stay up (cheap; fire only for record message types).
|
||||
// Tab / webNavigation observation attaches lazily while a recording is
|
||||
// active — see ensureBrowserObservationListeners in tools/record.ts.
|
||||
attachRecordStepListener();
|
||||
attachRecordFinishListener(recordDeps);
|
||||
attachRecordQueryListener(recordDeps);
|
||||
if (typeof chrome.notifications?.onClicked?.addListener === "function") {
|
||||
attachBorrowNotificationClickHandler({
|
||||
onClicked: chrome.notifications.onClicked,
|
||||
@@ -305,7 +359,11 @@ export default defineBackground(() => {
|
||||
}
|
||||
|
||||
if (msg.kind === OVERLAY_MSG_READY) {
|
||||
sendResponse(overlayStateForWindow(sender.tab?.windowId));
|
||||
// Resolve from explicit tab ownership before sending so free tabs never
|
||||
// flash the control mask.
|
||||
if (typeof sender.tab?.id === "number") {
|
||||
void pushOverlayStateForTab(sender.tab.id, sender.tab.windowId);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
@@ -11,11 +11,6 @@ import { createHelpRequestData } from "@/content/help-request";
|
||||
import overlayCss from "@/content/overlay.css?inline";
|
||||
import { OverlayController, shouldShowAgentControlOverlay } from "@/content/overlay-controller";
|
||||
import { RecordOverlay } from "@/content/RecordOverlay";
|
||||
import {
|
||||
handleRecordContentMessage,
|
||||
isRecordContentMessage,
|
||||
type RecordCaptureController,
|
||||
} from "@/content/record-capture";
|
||||
import {
|
||||
type CaptureSuppressAck,
|
||||
type CaptureSuppressMessage,
|
||||
@@ -45,11 +40,17 @@ import {
|
||||
} from "@/lib/overlay-bridge";
|
||||
import { sendInterrupt } from "@/lib/overlay-interrupt-client";
|
||||
import {
|
||||
isRecordCancelMessage,
|
||||
isRecordStartMessage,
|
||||
isRecordStopMessage,
|
||||
RECORD_FINISH,
|
||||
RECORD_QUERY,
|
||||
type RecordCancelMessage,
|
||||
type RecordQueryResponse,
|
||||
type RecordStartAck,
|
||||
type RecordStartMessage,
|
||||
type RecordStopAck,
|
||||
type RecordStopMessage,
|
||||
} from "@/lib/record-bridge";
|
||||
import type {
|
||||
BorrowCancelMessage,
|
||||
@@ -69,7 +70,6 @@ export default defineContentScript({
|
||||
if (window.top !== window) return;
|
||||
|
||||
const overlays = new OverlayController();
|
||||
let recordCapture: RecordCaptureController | null = null;
|
||||
let activeRecordRequestId: string | null = null;
|
||||
let reactRoot: ReactDOM.Root | null = null;
|
||||
let overlayHost: HTMLElement | null = null;
|
||||
@@ -228,8 +228,6 @@ export default defineContentScript({
|
||||
if (previousHelp) {
|
||||
void sendHelpFinish(previousHelp.id, "cancelled");
|
||||
}
|
||||
recordCapture?.dispose();
|
||||
recordCapture = null;
|
||||
activeRecordRequestId = null;
|
||||
renderAll();
|
||||
}
|
||||
@@ -259,6 +257,9 @@ export default defineContentScript({
|
||||
| HelpRequestMessage
|
||||
| HelpCancelMessage
|
||||
| CaptureSuppressMessage
|
||||
| RecordStartMessage
|
||||
| RecordStopMessage
|
||||
| RecordCancelMessage
|
||||
| OverlayAgentOverlayResetMessage
|
||||
| OverlayAgentStateMessage
|
||||
| OverlayAutomationBypassMessage,
|
||||
@@ -269,41 +270,32 @@ export default defineContentScript({
|
||||
return captureSuppress.handleMessage(message, sendResponse);
|
||||
}
|
||||
|
||||
if (isRecordContentMessage(message)) {
|
||||
const needsAsync = handleRecordContentMessage(
|
||||
message,
|
||||
{
|
||||
activeRequestId: activeRecordRequestId,
|
||||
capture: recordCapture,
|
||||
setActiveRequestId: (id) => {
|
||||
activeRecordRequestId = id;
|
||||
},
|
||||
setCapture: (capture) => {
|
||||
recordCapture = capture;
|
||||
},
|
||||
onStart: (requestId, startedAtMs) => {
|
||||
overlays.setAgentRecordRequest({
|
||||
id: requestId,
|
||||
...(typeof startedAtMs === "number" ? { startedAtMs } : {}),
|
||||
onFinish: () => {
|
||||
void chrome.runtime.sendMessage({
|
||||
type: RECORD_FINISH,
|
||||
requestId,
|
||||
});
|
||||
},
|
||||
});
|
||||
renderAll();
|
||||
},
|
||||
onStop: () => {
|
||||
overlays.clearAgentRecordRequest(activeRecordRequestId ?? undefined);
|
||||
renderAll();
|
||||
},
|
||||
if (isRecordStartMessage(message)) {
|
||||
activeRecordRequestId = message.requestId;
|
||||
overlays.setAgentRecordRequest({
|
||||
id: message.requestId,
|
||||
...(typeof message.startedAtMs === "number" ? { startedAtMs: message.startedAtMs } : {}),
|
||||
onFinish: () => {
|
||||
void chrome.runtime.sendMessage({
|
||||
type: RECORD_FINISH,
|
||||
requestId: message.requestId,
|
||||
});
|
||||
},
|
||||
sendResponse as unknown as
|
||||
| ((response: RecordStartAck | RecordStopAck) => void)
|
||||
| undefined,
|
||||
);
|
||||
return needsAsync;
|
||||
});
|
||||
renderAll();
|
||||
(sendResponse as unknown as (response: RecordStartAck) => void)({ ok: true });
|
||||
return false;
|
||||
}
|
||||
|
||||
if (isRecordStopMessage(message) || isRecordCancelMessage(message)) {
|
||||
if (activeRecordRequestId !== message.requestId) return false;
|
||||
overlays.clearAgentRecordRequest(message.requestId);
|
||||
activeRecordRequestId = null;
|
||||
renderAll();
|
||||
if (isRecordStopMessage(message)) {
|
||||
(sendResponse as unknown as (response: RecordStopAck) => void)({ ok: true });
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
if (
|
||||
@@ -526,9 +518,6 @@ export default defineContentScript({
|
||||
chrome.runtime.onMessage.removeListener(onMessage);
|
||||
chrome.storage.onChanged.removeListener(onStorageChange);
|
||||
window.removeEventListener("pageshow", onPageShow);
|
||||
// Restore history hooks / remove capture listeners before the CS unloads.
|
||||
recordCapture?.dispose();
|
||||
recordCapture = null;
|
||||
activeRecordRequestId = null;
|
||||
});
|
||||
},
|
||||
|
||||
@@ -122,8 +122,8 @@ describe("App", () => {
|
||||
|
||||
expect(screen.queryByText(/^扩展 v/)).toBeNull();
|
||||
expect(screen.queryByText(/^daemon v/)).toBeNull();
|
||||
expect(screen.getByTitle("扩展版本").textContent).toBe(EXTENSION_VERSION);
|
||||
expect(screen.getByTitle("bsk 版本").textContent).toBe(mockDaemonVersion);
|
||||
expect(screen.getByTitle("扩展版本").textContent).toBe(`Ext ${EXTENSION_VERSION}`);
|
||||
expect(screen.getByTitle("bsk 版本").textContent).toBe(`CLI ${mockDaemonVersion}`);
|
||||
expect(screen.getByText("03c3e47f")).toBeTruthy();
|
||||
|
||||
const copyButton = screen.getByRole("button", { name: "复制实例 ID" });
|
||||
@@ -200,6 +200,9 @@ describe("App", () => {
|
||||
expect(copied).toContain("--browser 03c3e47f");
|
||||
expect(copied).toContain('--purpose "发布 wiki 文档"');
|
||||
expect(copied).not.toMatch(/bsk record start[^\n]*--url/);
|
||||
expect(copied).toContain("./trace");
|
||||
expect(copied).toContain("trace.json");
|
||||
expect(copied).toContain("states/");
|
||||
// Button label stays static; a transient toast confirms the copy.
|
||||
expect(copyButton.textContent).toContain("复制录制指令");
|
||||
await waitFor(() => expect(screen.getByRole("status").textContent).toContain("已复制"));
|
||||
@@ -244,6 +247,41 @@ describe("App", () => {
|
||||
expect(copyButton.getAttribute("disabled")).not.toBeNull();
|
||||
expect(screen.getByText("连接后可用")).toBeTruthy();
|
||||
});
|
||||
|
||||
it("treats protocol drift as still connected and prompts an upgrade", async () => {
|
||||
mockUseConnectionState.mockReturnValue({
|
||||
snapshot: {
|
||||
...baseSnapshot,
|
||||
state: "version_skew",
|
||||
instanceId: "03c3e47f",
|
||||
handshake: {
|
||||
server: "bh",
|
||||
version: mockDaemonVersion,
|
||||
protocol_version: "1.0",
|
||||
},
|
||||
},
|
||||
statusState: "version_skew",
|
||||
setLabel,
|
||||
setConnectionEnabled,
|
||||
});
|
||||
|
||||
render(<App />);
|
||||
|
||||
expect(screen.getByText("已连接")).toBeTruthy();
|
||||
expect(screen.getByText("可升级")).toBeTruthy();
|
||||
expect(screen.queryByText("协议不一致")).toBeNull();
|
||||
expect(screen.queryByText("Action needed")).toBeNull();
|
||||
expect(screen.queryByText("兼容")).toBeNull();
|
||||
const warning = screen.getByText(/协议版本不同,请及时升级/);
|
||||
expect(warning.textContent).toContain("CLI 协议");
|
||||
expect(warning.textContent).toContain("扩展协议");
|
||||
|
||||
openRecordView();
|
||||
const copyButton = screen.getByRole("button", { name: "复制录制指令" });
|
||||
expect(copyButton.getAttribute("disabled")).toBeNull();
|
||||
fireEvent.click(copyButton);
|
||||
expect(navigator.clipboard.writeText).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("control hints toggle", () => {
|
||||
|
||||
@@ -79,6 +79,7 @@ export function App() {
|
||||
}, [copiedTick]);
|
||||
|
||||
const isSkewed = statusState === "version_skew";
|
||||
const connectionLive = statusState === "connected" || isSkewed;
|
||||
const daemonVersion = snapshot.handshake?.version ?? "—";
|
||||
const daemonProtocol = snapshot.handshake?.protocol_version ?? "—";
|
||||
const extensionVersion = snapshot.extensionVersion || "—";
|
||||
@@ -90,7 +91,7 @@ export function App() {
|
||||
setCopiedInstanceId(true);
|
||||
};
|
||||
|
||||
const recordReady = statusState === "connected" && Boolean(snapshot.instanceId);
|
||||
const recordReady = connectionLive && Boolean(snapshot.instanceId);
|
||||
const recordPurpose = purposeDraft.trim();
|
||||
const recordStartUrl = startUrlDraft.trim();
|
||||
const recordCommand = snapshot.instanceId
|
||||
@@ -206,12 +207,12 @@ export function App() {
|
||||
</div>
|
||||
{isSkewed && (
|
||||
<p
|
||||
className="mt-2 text-xs leading-snug text-amber-600 dark:text-amber-400"
|
||||
className="mt-2 text-xs leading-snug text-muted-foreground"
|
||||
data-slot="popup-version-skew-warning"
|
||||
>
|
||||
{t("popup.versionSkewWarning", {
|
||||
extensionProtocol: PROTOCOL_VERSION,
|
||||
daemonProtocol,
|
||||
cliProtocol: daemonProtocol,
|
||||
})}
|
||||
</p>
|
||||
)}
|
||||
@@ -266,9 +267,9 @@ export function App() {
|
||||
data-slot="popup-meta"
|
||||
>
|
||||
<div className="flex shrink-0 items-center gap-1">
|
||||
<span title={t("popup.extensionVersionHint")}>{extensionVersion}</span>
|
||||
<span title={t("popup.daemonVersionHint")}>CLI {daemonVersion}</span>
|
||||
<span aria-hidden>/</span>
|
||||
<span title={t("popup.daemonVersionHint")}>{daemonVersion}</span>
|
||||
<span title={t("popup.extensionVersionHint")}>Ext {extensionVersion}</span>
|
||||
</div>
|
||||
<div className="flex min-w-0 items-center justify-end gap-1">
|
||||
<span className="shrink-0">{t("popup.instanceTitle")}</span>
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
import { attachRecordFrameAgent } from "@/content/recording/frame-agent";
|
||||
|
||||
export default defineContentScript({
|
||||
matches: ["<all_urls>"],
|
||||
runAt: "document_start",
|
||||
allFrames: true,
|
||||
matchAboutBlank: true,
|
||||
matchOriginAsFallback: true,
|
||||
|
||||
main(ctx) {
|
||||
const dispose = attachRecordFrameAgent();
|
||||
ctx.onInvalidated(dispose);
|
||||
},
|
||||
});
|
||||
@@ -2,7 +2,7 @@ import { describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
CAPTURE_SUPPRESS,
|
||||
type CaptureSuppressMessage,
|
||||
withOverlaysHiddenForCapture,
|
||||
withExtensionOverlayHidden,
|
||||
} from "../capture-suppress-bridge";
|
||||
|
||||
function recordingSendToTab(events: string[]) {
|
||||
@@ -12,12 +12,12 @@ function recordingSendToTab(events: string[]) {
|
||||
});
|
||||
}
|
||||
|
||||
describe("withOverlaysHiddenForCapture", () => {
|
||||
describe("withExtensionOverlayHidden", () => {
|
||||
it("sends begin → fn → end and returns fn's result", async () => {
|
||||
const events: string[] = [];
|
||||
const sendToTab = recordingSendToTab(events);
|
||||
|
||||
const result = await withOverlaysHiddenForCapture(
|
||||
const result = await withExtensionOverlayHidden(
|
||||
7,
|
||||
async () => {
|
||||
events.push("capture");
|
||||
@@ -37,7 +37,7 @@ describe("withOverlaysHiddenForCapture", () => {
|
||||
const sendToTab = recordingSendToTab(events);
|
||||
|
||||
await expect(
|
||||
withOverlaysHiddenForCapture(
|
||||
withExtensionOverlayHidden(
|
||||
7,
|
||||
async () => {
|
||||
events.push("capture");
|
||||
@@ -55,7 +55,7 @@ describe("withOverlaysHiddenForCapture", () => {
|
||||
});
|
||||
const fn = vi.fn(async () => "shot");
|
||||
|
||||
const result = await withOverlaysHiddenForCapture(7, fn, sendToTab);
|
||||
const result = await withExtensionOverlayHidden(7, fn, sendToTab);
|
||||
|
||||
expect(result).toBe("shot");
|
||||
expect(fn).toHaveBeenCalledTimes(1);
|
||||
@@ -69,7 +69,7 @@ describe("withOverlaysHiddenForCapture", () => {
|
||||
return { type: CAPTURE_SUPPRESS, ok: true };
|
||||
});
|
||||
|
||||
const result = await withOverlaysHiddenForCapture(7, async () => "shot", sendToTab);
|
||||
const result = await withExtensionOverlayHidden(7, async () => "shot", sendToTab);
|
||||
|
||||
expect(result).toBe("shot");
|
||||
expect(sendToTab).toHaveBeenCalledTimes(2);
|
||||
|
||||
@@ -26,20 +26,20 @@ function handshake(
|
||||
}
|
||||
|
||||
describe("computeConnectedState (protocol-based compat)", () => {
|
||||
it("returns connected when protocol strings match", () => {
|
||||
expect(computeConnectedState(handshake("1.0", "1.0"), MIN_COMPATIBLE_PROTOCOL)).toEqual({
|
||||
it("returns connected when daemon protocol equals extension protocol", () => {
|
||||
expect(computeConnectedState(handshake("1.1", "1.0"), MIN_COMPATIBLE_PROTOCOL)).toEqual({
|
||||
kind: "connected",
|
||||
});
|
||||
});
|
||||
|
||||
it("returns version_skew when daemon protocol minor is newer", () => {
|
||||
expect(computeConnectedState(handshake("1.1", "1.0"))).toEqual({
|
||||
expect(computeConnectedState(handshake("1.2", "1.0"))).toEqual({
|
||||
kind: "version_skew",
|
||||
});
|
||||
});
|
||||
|
||||
it("returns version_skew when daemon protocol string differs but floor is satisfied", () => {
|
||||
expect(computeConnectedState(handshake("1", "1.0"))).toEqual({
|
||||
expect(computeConnectedState(handshake("1.1.0", "1.0"))).toEqual({
|
||||
kind: "version_skew",
|
||||
});
|
||||
});
|
||||
@@ -53,7 +53,7 @@ describe("computeConnectedState (protocol-based compat)", () => {
|
||||
});
|
||||
|
||||
it("rejects when extension is below daemon min_compatible_protocol", () => {
|
||||
const result = computeConnectedState(handshake("1.0", "1.5"));
|
||||
const result = computeConnectedState(handshake("1.1", "1.5"));
|
||||
expect(result.kind).toBe("rejected");
|
||||
if (result.kind === "rejected") {
|
||||
expect(result.reason).toContain("min_compatible_protocol");
|
||||
@@ -65,7 +65,7 @@ describe("computeConnectedState (protocol-based compat)", () => {
|
||||
const result = computeConnectedState({
|
||||
server: "browser-skill-daemon",
|
||||
version: "0.1.0",
|
||||
protocol_version: "1.0",
|
||||
protocol_version: "1.1",
|
||||
min_compatible_peer: "0.1.0",
|
||||
});
|
||||
expect(result).toEqual({ kind: "connected" });
|
||||
@@ -80,7 +80,7 @@ describe("computeConnectedState (protocol-based compat)", () => {
|
||||
});
|
||||
|
||||
it("rejects malformed daemon min_compatible_protocol with a daemon-floor reason", () => {
|
||||
const result = computeConnectedState(handshake("1.0", "not-a-protocol"));
|
||||
const result = computeConnectedState(handshake("1.1", "not-a-protocol"));
|
||||
expect(result.kind).toBe("rejected");
|
||||
if (result.kind === "rejected") {
|
||||
expect(result.reason).toContain("daemon min_compatible_protocol");
|
||||
@@ -242,11 +242,11 @@ describe("ConnectionController connectionEnabled", () => {
|
||||
const second = transport.send.mock.calls[1]?.[0] as { id: string };
|
||||
expect(second.id).not.toBe(first.id);
|
||||
|
||||
transport.emitMessage({ id: first.id, result: handshake("1.0", "1.0") });
|
||||
transport.emitMessage({ id: first.id, result: handshake("1.1", "1.0") });
|
||||
await Promise.resolve();
|
||||
expect(controller.snapshot().state).not.toBe("connected");
|
||||
|
||||
transport.emitMessage({ id: second.id, result: handshake("1.0", "1.0") });
|
||||
transport.emitMessage({ id: second.id, result: handshake("1.1", "1.0") });
|
||||
await vi.waitFor(() => expect(controller.snapshot().state).toBe("connected"));
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { createRandomUuid } from "../random-uuid";
|
||||
|
||||
describe("createRandomUuid", () => {
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
it("uses crypto.randomUUID when it is available", () => {
|
||||
const randomUUID = vi.fn(() => "00000000-0000-4000-8000-000000000000");
|
||||
vi.stubGlobal("crypto", { randomUUID, getRandomValues: vi.fn() });
|
||||
|
||||
expect(createRandomUuid()).toBe("00000000-0000-4000-8000-000000000000");
|
||||
expect(randomUUID).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("falls back to getRandomValues and sets UUID v4 bits", () => {
|
||||
const bytes = new Uint8Array(16).fill(0xff);
|
||||
const getRandomValues = vi.fn((target: Uint8Array) => {
|
||||
target.set(bytes);
|
||||
return target;
|
||||
});
|
||||
vi.stubGlobal("crypto", { getRandomValues });
|
||||
|
||||
const uuid = createRandomUuid();
|
||||
|
||||
expect(uuid).toBe("ffffffff-ffff-4fff-bfff-ffffffffffff");
|
||||
expect(getRandomValues).toHaveBeenCalledOnce();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,32 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
isAcceptedRecordStepAck,
|
||||
isRecordStepMessage,
|
||||
RECORD_STEP,
|
||||
type RecordStepMessage,
|
||||
} from "../record-bridge";
|
||||
|
||||
const message: RecordStepMessage = {
|
||||
type: RECORD_STEP,
|
||||
requestId: "rec-1",
|
||||
producerId: "document-1",
|
||||
sequence: 1,
|
||||
step: { op: "click", target: { tag: "button", name: "Submit" } },
|
||||
};
|
||||
|
||||
describe("record step delivery protocol", () => {
|
||||
it("requires a producer and a positive integer sequence", () => {
|
||||
expect(isRecordStepMessage(message)).toBe(true);
|
||||
expect(isRecordStepMessage({ ...message, producerId: "" })).toBe(false);
|
||||
expect(isRecordStepMessage({ ...message, sequence: 0 })).toBe(false);
|
||||
expect(isRecordStepMessage({ ...message, sequence: 1.5 })).toBe(false);
|
||||
});
|
||||
|
||||
it("accepts only the acknowledgement for the sent sequence", () => {
|
||||
expect(isAcceptedRecordStepAck({ ok: true, sequence: 2 }, 2)).toBe(true);
|
||||
expect(isAcceptedRecordStepAck({ ok: true, sequence: 1 }, 2)).toBe(false);
|
||||
expect(isAcceptedRecordStepAck({ ok: false, expectedSequence: 2, error: "gap" }, 2)).toBe(
|
||||
false,
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,213 @@
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { RECORD_FRAME_PORT, RECORD_FRAME_QUERY } from "../recording/frame-bridge";
|
||||
import { RecordFrameCoordinator } from "../recording/frame-coordinator";
|
||||
|
||||
class ListenerSet<T extends (...args: never[]) => unknown> {
|
||||
readonly listeners = new Set<T>();
|
||||
addListener = (listener: T) => this.listeners.add(listener);
|
||||
removeListener = (listener: T) => this.listeners.delete(listener);
|
||||
}
|
||||
|
||||
function fakePort(sender: chrome.runtime.MessageSender) {
|
||||
const messages: unknown[] = [];
|
||||
const onMessage = new ListenerSet<(message: unknown) => void>();
|
||||
const onDisconnect = new ListenerSet<() => void>();
|
||||
return {
|
||||
port: {
|
||||
name: RECORD_FRAME_PORT,
|
||||
sender,
|
||||
onMessage,
|
||||
onDisconnect,
|
||||
postMessage: vi.fn((message: unknown) => messages.push(message)),
|
||||
disconnect: vi.fn(),
|
||||
} as unknown as chrome.runtime.Port,
|
||||
messages,
|
||||
receive(message: unknown) {
|
||||
for (const listener of onMessage.listeners) listener(message);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe("RecordFrameCoordinator", () => {
|
||||
const onConnect = new ListenerSet<(port: chrome.runtime.Port) => void>();
|
||||
const onMessage = new ListenerSet<
|
||||
(
|
||||
message: unknown,
|
||||
sender: chrome.runtime.MessageSender,
|
||||
sendResponse: (response: unknown) => void,
|
||||
) => boolean
|
||||
>();
|
||||
|
||||
beforeEach(() => {
|
||||
onConnect.listeners.clear();
|
||||
onMessage.listeners.clear();
|
||||
vi.stubGlobal("chrome", {
|
||||
runtime: { onConnect, onMessage },
|
||||
});
|
||||
});
|
||||
|
||||
it("answers child-document queries for the initial tab before it is armed", () => {
|
||||
const coordinator = new RecordFrameCoordinator({
|
||||
getAllFrames: async () => [],
|
||||
sendToDocument: async () => ({ ok: true }),
|
||||
});
|
||||
coordinator.attach();
|
||||
coordinator.begin("rec-1", 10, 3);
|
||||
|
||||
const query = (tabId: number) => {
|
||||
let response: unknown;
|
||||
for (const listener of onMessage.listeners) {
|
||||
listener(
|
||||
{ type: RECORD_FRAME_QUERY },
|
||||
{
|
||||
tab: { id: tabId },
|
||||
frameId: 7,
|
||||
documentId: "child-document",
|
||||
} as chrome.runtime.MessageSender,
|
||||
(value) => {
|
||||
response = value;
|
||||
},
|
||||
);
|
||||
}
|
||||
return response;
|
||||
};
|
||||
|
||||
expect(query(3)).toEqual({ active: true, requestId: "rec-1", startedAtMs: 10 });
|
||||
expect(query(4)).toEqual({ active: false });
|
||||
});
|
||||
|
||||
it("starts a child Document that appears after the initial frame snapshot", async () => {
|
||||
let notifyFrameNavigation:
|
||||
| ((frame: {
|
||||
tabId: number;
|
||||
frameId: number;
|
||||
documentId?: string;
|
||||
lifecycle: "committed" | "completed";
|
||||
}) => void)
|
||||
| undefined;
|
||||
const sendToDocument = vi.fn(async () => ({ ok: true }));
|
||||
const coordinator = new RecordFrameCoordinator({
|
||||
getAllFrames: async () => [{ frameId: 0, documentId: "top-document" }],
|
||||
sendToDocument,
|
||||
subscribeFrameNavigation(listener) {
|
||||
notifyFrameNavigation = listener;
|
||||
return () => {};
|
||||
},
|
||||
});
|
||||
coordinator.attach();
|
||||
const onDocumentReady = vi.fn();
|
||||
coordinator.begin("rec-1", 10, 3, onDocumentReady);
|
||||
await coordinator.armTab("rec-1", 3);
|
||||
sendToDocument.mockClear();
|
||||
|
||||
notifyFrameNavigation?.({
|
||||
tabId: 3,
|
||||
frameId: 7,
|
||||
documentId: "late-child",
|
||||
lifecycle: "completed",
|
||||
});
|
||||
await vi.waitFor(() => {
|
||||
expect(sendToDocument).toHaveBeenCalledWith(
|
||||
3,
|
||||
{ type: "bsk-record-frame-start", requestId: "rec-1", startedAtMs: 10 },
|
||||
{ documentId: "late-child" },
|
||||
);
|
||||
});
|
||||
expect(onDocumentReady).not.toHaveBeenCalled();
|
||||
|
||||
const child = fakePort({
|
||||
tab: { id: 3 },
|
||||
frameId: 7,
|
||||
documentId: "late-child",
|
||||
} as chrome.runtime.MessageSender);
|
||||
for (const listener of onConnect.listeners) listener(child.port);
|
||||
child.receive({ type: "ready", requestId: "rec-1", producerId: "producer-1" });
|
||||
notifyFrameNavigation?.({
|
||||
tabId: 3,
|
||||
frameId: 7,
|
||||
documentId: "late-child",
|
||||
lifecycle: "completed",
|
||||
});
|
||||
await vi.waitFor(() => {
|
||||
expect(onDocumentReady).toHaveBeenCalledWith({
|
||||
tabId: 3,
|
||||
documentId: "late-child",
|
||||
browserFrameId: 7,
|
||||
producerId: "producer-1",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it("binds a producer to its sender Document and keeps final steps valid while stopping", async () => {
|
||||
const sendToDocument = vi.fn(async () => ({ ok: true }));
|
||||
const coordinator = new RecordFrameCoordinator({
|
||||
getAllFrames: async () => [
|
||||
{ frameId: 0, documentId: "top-document" },
|
||||
{ frameId: 7, documentId: "child-document" },
|
||||
],
|
||||
sendToDocument,
|
||||
});
|
||||
coordinator.attach();
|
||||
coordinator.begin("rec-1", 10, 3);
|
||||
await expect(coordinator.armTab("rec-1", 3)).resolves.toBe(true);
|
||||
|
||||
const sender = {
|
||||
tab: { id: 3 },
|
||||
frameId: 7,
|
||||
documentId: "child-document",
|
||||
} as chrome.runtime.MessageSender;
|
||||
const frame = fakePort(sender);
|
||||
for (const listener of onConnect.listeners) listener(frame.port);
|
||||
frame.receive({ type: "ready", requestId: "rec-1", producerId: "producer-1" });
|
||||
|
||||
expect(coordinator.sourceFor("rec-1", "producer-1", sender)).toEqual({
|
||||
tabId: 3,
|
||||
documentId: "child-document",
|
||||
browserFrameId: 7,
|
||||
producerId: "producer-1",
|
||||
});
|
||||
|
||||
const stopping = coordinator.stop("rec-1");
|
||||
expect(coordinator.sourceFor("rec-1", "producer-1", sender)).not.toBeNull();
|
||||
const stop = frame.messages.find(
|
||||
(message): message is { type: "stop"; commandId: string } =>
|
||||
typeof message === "object" &&
|
||||
message !== null &&
|
||||
(message as { type?: string }).type === "stop",
|
||||
);
|
||||
expect(stop).toBeDefined();
|
||||
frame.receive({
|
||||
type: "stopped",
|
||||
requestId: "rec-1",
|
||||
commandId: stop!.commandId,
|
||||
ok: true,
|
||||
});
|
||||
await expect(stopping).resolves.toBe(true);
|
||||
expect(coordinator.sourceFor("rec-1", "producer-1", sender)).toBeNull();
|
||||
});
|
||||
|
||||
it("rejects a producer used from another Document", async () => {
|
||||
const coordinator = new RecordFrameCoordinator({
|
||||
getAllFrames: async () => [{ frameId: 0, documentId: "top-document" }],
|
||||
sendToDocument: async () => ({ ok: true }),
|
||||
});
|
||||
coordinator.attach();
|
||||
coordinator.begin("rec-1", 10, 3);
|
||||
await coordinator.armTab("rec-1", 3);
|
||||
const sender = {
|
||||
tab: { id: 3 },
|
||||
frameId: 0,
|
||||
documentId: "top-document",
|
||||
} as chrome.runtime.MessageSender;
|
||||
const frame = fakePort(sender);
|
||||
for (const listener of onConnect.listeners) listener(frame.port);
|
||||
frame.receive({ type: "ready", requestId: "rec-1", producerId: "producer-1" });
|
||||
|
||||
expect(
|
||||
coordinator.sourceFor("rec-1", "producer-1", {
|
||||
...sender,
|
||||
documentId: "other-document",
|
||||
}),
|
||||
).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,70 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
RECORD_DOCUMENT_ATTRIBUTE,
|
||||
recordingDocumentMarkerValue,
|
||||
} from "@/shared/recording-document-identity";
|
||||
import { markRecordingDocument } from "../recording/document-marker";
|
||||
import { ObservationNodeIndex } from "../recording/observation-capture";
|
||||
|
||||
describe("recording document marker", () => {
|
||||
it("rejects a non-random Document identity", () => {
|
||||
expect(() => markRecordingDocument("user@example.com")).toThrow(
|
||||
"recording Document identity must be a random UUID",
|
||||
);
|
||||
});
|
||||
|
||||
it("restores the Document attribute exactly", () => {
|
||||
document.documentElement.setAttribute(RECORD_DOCUMENT_ATTRIBUTE, "page-value");
|
||||
const producerId = "123e4567-e89b-42d3-a456-426614174000";
|
||||
const marker = markRecordingDocument(producerId);
|
||||
|
||||
expect(document.documentElement.getAttribute(RECORD_DOCUMENT_ATTRIBUTE)).toBe(
|
||||
recordingDocumentMarkerValue(producerId),
|
||||
);
|
||||
marker.restore();
|
||||
expect(document.documentElement.getAttribute(RECORD_DOCUMENT_ATTRIBUTE)).toBe("page-value");
|
||||
document.documentElement.removeAttribute(RECORD_DOCUMENT_ATTRIBUTE);
|
||||
});
|
||||
|
||||
it("indexes the CDP scope from the safe frame identity", () => {
|
||||
const index = new ObservationNodeIndex({
|
||||
rootFrameId: "root",
|
||||
frames: [
|
||||
{
|
||||
frameId: "child",
|
||||
target: { tabId: 3, sessionId: "oopif-session" },
|
||||
recordingDocumentId: "producer-1",
|
||||
},
|
||||
],
|
||||
matchNodes: [],
|
||||
refs: [],
|
||||
});
|
||||
|
||||
expect(index.documentScope("producer-1")).toEqual({
|
||||
frameId: "child",
|
||||
target: { tabId: 3, sessionId: "oopif-session" },
|
||||
});
|
||||
});
|
||||
|
||||
it("fails closed when a recording identity maps to multiple Documents", () => {
|
||||
const index = new ObservationNodeIndex({
|
||||
rootFrameId: "root",
|
||||
frames: [
|
||||
{
|
||||
frameId: "left",
|
||||
target: { tabId: 3, sessionId: "left-session" },
|
||||
recordingDocumentId: "duplicate",
|
||||
},
|
||||
{
|
||||
frameId: "right",
|
||||
target: { tabId: 3, sessionId: "right-session" },
|
||||
recordingDocumentId: "duplicate",
|
||||
},
|
||||
],
|
||||
matchNodes: [],
|
||||
refs: [],
|
||||
});
|
||||
|
||||
expect(index.documentScope("duplicate")).toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -11,7 +11,7 @@ function sessionWithInput(redactValues = false): RecordingObservationSession {
|
||||
const session = new RecordingObservationSession({ redactValues });
|
||||
const state = session.registry.register({
|
||||
url: URL,
|
||||
rawVomText: '@vom 1\ntextbox "Password" value="••••••" [ref=e1]',
|
||||
vomText: '@vom 1\ntextbox "Password" value="••••••" [ref=e1]',
|
||||
});
|
||||
session.cursor.lastSettled = {
|
||||
stateId: state.id,
|
||||
@@ -34,7 +34,7 @@ function sessionWithInput(redactValues = false): RecordingObservationSession {
|
||||
return session;
|
||||
}
|
||||
|
||||
function finalizedFillBody(value: string, redactValues: boolean): string {
|
||||
function finalizedFillTrace(value: string, redactValues: boolean) {
|
||||
const session = sessionWithInput(redactValues);
|
||||
const draft: RecordingDraftStep = {
|
||||
op: "fill",
|
||||
@@ -53,15 +53,20 @@ function finalizedFillBody(value: string, redactValues: boolean): string {
|
||||
startedAt: "2026-08-12T00:00:00.000Z",
|
||||
stoppedBy: "user_finish",
|
||||
bskVersion: "test",
|
||||
}).states[0]!.body;
|
||||
redactValues,
|
||||
});
|
||||
}
|
||||
|
||||
function finalizedFillBody(value: string, redactValues: boolean): string {
|
||||
return finalizedFillTrace(value, redactValues).states[0]!.body;
|
||||
}
|
||||
|
||||
describe("record observation annotations", () => {
|
||||
it("omits a fill literal when values are redacted", () => {
|
||||
const secret = "hunter2-private";
|
||||
const body = finalizedFillBody(secret, true);
|
||||
expect(body).toContain("step 1: fill");
|
||||
expect(body).not.toContain(secret);
|
||||
const dumped = JSON.stringify(finalizedFillTrace(secret, true));
|
||||
expect(dumped).toContain("step 1: fill");
|
||||
expect(dumped).not.toContain(secret);
|
||||
});
|
||||
|
||||
it("keeps ordinary fill details", () => {
|
||||
@@ -73,7 +78,7 @@ describe("record observation annotations", () => {
|
||||
const state = registry.register({
|
||||
url: "https://example.com/a\nb",
|
||||
title: "hello\nworld",
|
||||
rawVomText: "@vom 1",
|
||||
vomText: "@vom 1",
|
||||
});
|
||||
const trace = buildTraceV3({
|
||||
registry,
|
||||
@@ -91,18 +96,18 @@ describe("recording state ownership", () => {
|
||||
it("deduplicates within one recording and isolates ids between recordings", () => {
|
||||
const first = new RecordingStateRegistry();
|
||||
const second = new RecordingStateRegistry();
|
||||
expect(first.register({ url: URL, rawVomText: "same" }).id).toBe("s1");
|
||||
expect(first.register({ url: URL, rawVomText: "same" }).id).toBe("s1");
|
||||
expect(second.register({ url: URL, rawVomText: "other" }).id).toBe("s1");
|
||||
expect(first.register({ url: URL, vomText: "same" }).id).toBe("s1");
|
||||
expect(first.register({ url: URL, vomText: "same" }).id).toBe("s1");
|
||||
expect(second.register({ url: URL, vomText: "other" }).id).toBe("s1");
|
||||
});
|
||||
|
||||
it("enriches metadata when a deduplicated observation becomes more complete", () => {
|
||||
const registry = new RecordingStateRegistry();
|
||||
registry.register({ url: URL, rawVomText: "same" });
|
||||
registry.register({ url: URL, vomText: "same" });
|
||||
const state = registry.register({
|
||||
url: URL,
|
||||
title: "Login",
|
||||
rawVomText: "same",
|
||||
vomText: "same",
|
||||
truncated: true,
|
||||
});
|
||||
expect(state).toMatchObject({ id: "s1", title: "Login", truncated: true });
|
||||
|
||||
@@ -0,0 +1,289 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
|
||||
|
||||
const captureRecordingObservation = vi.hoisted(() => vi.fn());
|
||||
const waitForDocumentSettled = vi.hoisted(() => vi.fn(async () => "quiet" as const));
|
||||
|
||||
vi.mock("../recording/observation-capture", async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import("../recording/observation-capture")>();
|
||||
return { ...actual, captureRecordingObservation };
|
||||
});
|
||||
|
||||
vi.mock("../recording/document-settle", async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import("../recording/document-settle")>();
|
||||
return { ...actual, waitForDocumentSettled };
|
||||
});
|
||||
|
||||
import { ObservationNodeIndex } from "../recording/observation-capture";
|
||||
import { RecordingObservationRuntime } from "../recording/recording-runtime";
|
||||
import type { RecordingDraftStep } from "../recording/types";
|
||||
|
||||
function observation(url = "https://example.com/") {
|
||||
return {
|
||||
rootFrameId: "root",
|
||||
index: new ObservationNodeIndex({ rootFrameId: "root", matchNodes: [], refs: [] }),
|
||||
url,
|
||||
title: "Example",
|
||||
vomText: '@vom 1\nRootWebArea "Example"',
|
||||
truncated: false,
|
||||
};
|
||||
}
|
||||
|
||||
function runtime(): RecordingObservationRuntime {
|
||||
return new RecordingObservationRuntime({
|
||||
cdp: { send: vi.fn() as unknown as CdpRunner["send"] },
|
||||
tabsApi: { get: vi.fn(), query: vi.fn() } as unknown as ChromeTabsApi,
|
||||
});
|
||||
}
|
||||
|
||||
describe("RecordingObservationRuntime", () => {
|
||||
afterEach(() => {
|
||||
captureRecordingObservation.mockReset();
|
||||
waitForDocumentSettled.mockClear();
|
||||
});
|
||||
|
||||
it("shares one initial capture and includes it in flush", async () => {
|
||||
let release!: (value: ReturnType<typeof observation>) => void;
|
||||
captureRecordingObservation.mockReturnValueOnce(
|
||||
new Promise((resolve) => {
|
||||
release = resolve;
|
||||
}),
|
||||
);
|
||||
const recording = runtime();
|
||||
|
||||
const first = recording.captureInitial(7);
|
||||
const second = recording.captureInitial(7);
|
||||
const flushed = recording.flush();
|
||||
let flushCompleted = false;
|
||||
void flushed.then(() => {
|
||||
flushCompleted = true;
|
||||
});
|
||||
|
||||
await Promise.resolve();
|
||||
expect(captureRecordingObservation).toHaveBeenCalledTimes(1);
|
||||
expect(flushCompleted).toBe(false);
|
||||
|
||||
release(observation());
|
||||
await Promise.all([first, second, flushed]);
|
||||
expect(flushCompleted).toBe(true);
|
||||
});
|
||||
|
||||
it("cancels an in-flight initial capture", async () => {
|
||||
captureRecordingObservation.mockImplementationOnce(
|
||||
(input: { signal?: AbortSignal }) =>
|
||||
new Promise((_, reject) => {
|
||||
input.signal?.addEventListener(
|
||||
"abort",
|
||||
() => reject(new DOMException("observation aborted", "AbortError")),
|
||||
{ once: true },
|
||||
);
|
||||
}),
|
||||
);
|
||||
const recording = runtime();
|
||||
|
||||
const pending = recording.captureInitial(7);
|
||||
recording.cancel();
|
||||
|
||||
await expect(pending).rejects.toMatchObject({ name: "AbortError" });
|
||||
});
|
||||
|
||||
it("retries a failed initial capture when the recording settles at stop", async () => {
|
||||
captureRecordingObservation
|
||||
.mockRejectedValueOnce(new Error("document swapped"))
|
||||
.mockResolvedValueOnce(observation());
|
||||
const recording = runtime();
|
||||
|
||||
await expect(recording.captureInitial(7)).rejects.toThrow("document swapped");
|
||||
await recording.settleTrailing(7, []);
|
||||
const trace = recording.buildTrace({
|
||||
drafts: [],
|
||||
startedAt: "2026-08-19T00:00:00.000Z",
|
||||
stoppedBy: "user_finish",
|
||||
bskVersion: "0.1.6",
|
||||
});
|
||||
|
||||
expect(captureRecordingObservation).toHaveBeenCalledTimes(2);
|
||||
expect(trace.states).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("keeps runtime matching data out of the serialized trace", async () => {
|
||||
const captured = observation();
|
||||
captured.index = new ObservationNodeIndex({
|
||||
rootFrameId: "root",
|
||||
matchNodes: [
|
||||
{
|
||||
frameId: "root",
|
||||
backendNodeId: 42,
|
||||
tag: "button",
|
||||
rect: { x: 10, y: 20, w: 100, h: 30 },
|
||||
localRect: { x: 10, y: 20, w: 100, h: 30 },
|
||||
},
|
||||
],
|
||||
refs: [{ ref: "e1", backendNodeId: 42, role: "button", name: "Submit", line: 1 }],
|
||||
});
|
||||
captureRecordingObservation.mockResolvedValueOnce(captured);
|
||||
const recording = runtime();
|
||||
|
||||
await recording.captureInitial(7);
|
||||
const dumped = JSON.stringify(
|
||||
recording.buildTrace({
|
||||
drafts: [],
|
||||
startedAt: "2026-08-19T00:00:00.000Z",
|
||||
stoppedBy: "user_finish",
|
||||
bskVersion: "0.1.6",
|
||||
}),
|
||||
);
|
||||
|
||||
expect(dumped).toContain("RootWebArea");
|
||||
expect(dumped).not.toContain("matchNodes");
|
||||
expect(dumped).not.toContain("localRect");
|
||||
expect(dumped).not.toContain("backendNodeId");
|
||||
expect(dumped).not.toContain("ObservationNodeIndex");
|
||||
});
|
||||
|
||||
it("captures a tab transition without sharing observation cursors", async () => {
|
||||
captureRecordingObservation
|
||||
.mockResolvedValueOnce(observation("https://example.com/first"))
|
||||
.mockResolvedValueOnce(observation("https://example.com/second"));
|
||||
const recording = runtime();
|
||||
|
||||
await recording.captureInitial(4);
|
||||
const transition = await recording.captureTabTransition(4, 5);
|
||||
await recording.captureInitial(4);
|
||||
await recording.captureInitial(5);
|
||||
|
||||
expect(transition).toEqual({
|
||||
preStateId: "s1",
|
||||
postStateId: "s2",
|
||||
targetUrl: "https://example.com/second",
|
||||
});
|
||||
expect(captureRecordingObservation.mock.calls.map(([input]) => input.tabId)).toEqual([4, 5]);
|
||||
});
|
||||
|
||||
it("binds an iframe draft to its marked CDP Document scope", async () => {
|
||||
captureRecordingObservation.mockResolvedValueOnce({
|
||||
...observation(),
|
||||
index: new ObservationNodeIndex({
|
||||
rootFrameId: "root",
|
||||
frames: [
|
||||
{
|
||||
frameId: "child",
|
||||
target: { tabId: 7, sessionId: "oopif-session" },
|
||||
recordingDocumentId: "producer-1",
|
||||
},
|
||||
],
|
||||
matchNodes: [
|
||||
{
|
||||
backendNodeId: 42,
|
||||
frameId: "child",
|
||||
tag: "button",
|
||||
rect: { x: 410, y: 20, w: 100, h: 30 },
|
||||
localRect: { x: 10, y: 20, w: 100, h: 30 },
|
||||
},
|
||||
],
|
||||
refs: [{ ref: "e1", backendNodeId: 42, frameId: "child", line: 1 }],
|
||||
}),
|
||||
});
|
||||
const recording = runtime();
|
||||
const drafts: RecordingDraftStep[] = [
|
||||
{
|
||||
op: "click" as const,
|
||||
captureTarget: { tag: "button", role: "button", name: "Save" },
|
||||
targetHint: {
|
||||
geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" },
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
await recording.captureInitial(7);
|
||||
await recording.processDraft(7, drafts, 0, "producer-1");
|
||||
recording.cancel();
|
||||
|
||||
const draft = drafts[0];
|
||||
expect(draft?.op).toBe("click");
|
||||
if (!draft || draft.op !== "click") throw new Error("expected click draft");
|
||||
expect(draft.targetHint).toMatchObject({
|
||||
frameId: "child",
|
||||
geometrySpace: "local",
|
||||
});
|
||||
expect(draft.matchedTarget?.ref).toBe("e1");
|
||||
});
|
||||
|
||||
it("refreshes a late iframe before matching its first action", async () => {
|
||||
const iframeObservation = {
|
||||
...observation(),
|
||||
index: new ObservationNodeIndex({
|
||||
rootFrameId: "root",
|
||||
frames: [
|
||||
{
|
||||
frameId: "child",
|
||||
target: { tabId: 7, sessionId: "oopif-session" },
|
||||
recordingDocumentId: "producer-1",
|
||||
},
|
||||
],
|
||||
matchNodes: [
|
||||
{
|
||||
backendNodeId: 42,
|
||||
frameId: "child",
|
||||
tag: "button",
|
||||
rect: { x: 410, y: 20, w: 100, h: 30 },
|
||||
localRect: { x: 10, y: 20, w: 100, h: 30 },
|
||||
},
|
||||
],
|
||||
refs: [
|
||||
{
|
||||
ref: "e1",
|
||||
backendNodeId: 42,
|
||||
frameId: "child",
|
||||
role: "button",
|
||||
name: "表格视图",
|
||||
line: 1,
|
||||
},
|
||||
],
|
||||
}),
|
||||
};
|
||||
captureRecordingObservation
|
||||
.mockResolvedValueOnce(observation())
|
||||
.mockResolvedValueOnce(iframeObservation)
|
||||
.mockResolvedValueOnce(iframeObservation);
|
||||
const recording = runtime();
|
||||
|
||||
await recording.captureInitial(7);
|
||||
await recording.refreshDocument(7, "producer-1");
|
||||
const drafts: RecordingDraftStep[] = [
|
||||
{
|
||||
op: "click",
|
||||
captureTarget: { tag: "button", role: "button", name: "表格视图" },
|
||||
targetHint: {
|
||||
geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" },
|
||||
},
|
||||
},
|
||||
];
|
||||
await recording.processDraft(7, drafts, 0, "producer-1");
|
||||
recording.cancel();
|
||||
|
||||
expect(captureRecordingObservation).toHaveBeenCalledTimes(3);
|
||||
expect(waitForDocumentSettled).toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
{ frameId: "child", target: { tabId: 7, sessionId: "oopif-session" } },
|
||||
{ signal: expect.any(AbortSignal) },
|
||||
);
|
||||
const draft = drafts[0];
|
||||
expect(draft?.op).toBe("click");
|
||||
if (!draft || draft.op !== "click") throw new Error("expected click draft");
|
||||
expect(draft.matchedTarget).toMatchObject({ ref: "e1", name: "表格视图" });
|
||||
});
|
||||
|
||||
it("does not poison final flush when a frame readiness refresh fails", async () => {
|
||||
captureRecordingObservation
|
||||
.mockResolvedValueOnce(observation())
|
||||
.mockRejectedValueOnce(new Error("child Document replaced"));
|
||||
const recording = runtime();
|
||||
|
||||
await recording.captureInitial(7);
|
||||
await expect(recording.refreshDocument(7, "producer-1")).rejects.toThrow(
|
||||
"child Document replaced",
|
||||
);
|
||||
await expect(recording.flush()).resolves.toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -3,7 +3,7 @@ import { appendRecordedPayload, observeRecordedNavigation } from "../recording/s
|
||||
|
||||
describe("recording-step-buffer", () => {
|
||||
it("stores semantic click without summary", () => {
|
||||
const buffer = { steps: [], pendingNavigation: false };
|
||||
const buffer = { steps: [], navigation: { pendingNavigation: false } };
|
||||
appendRecordedPayload(buffer, {
|
||||
op: "click",
|
||||
target: { tag: "button", role: "button", name: "发布" },
|
||||
@@ -15,7 +15,40 @@ describe("recording-step-buffer", () => {
|
||||
captureTarget: { tag: "button", role: "button", name: "发布" },
|
||||
},
|
||||
]);
|
||||
expect(buffer.pendingNavigation).toBe(true);
|
||||
expect(buffer.navigation.pendingNavigation).toBe(true);
|
||||
});
|
||||
|
||||
it("keeps the hovered element description as a capture fallback", () => {
|
||||
const buffer = { steps: [], navigation: { pendingNavigation: false } };
|
||||
appendRecordedPayload(
|
||||
buffer,
|
||||
{
|
||||
op: "hover",
|
||||
target: { tag: "button", role: "button", name: "新建" },
|
||||
geometry: {
|
||||
rect: { x: 900, y: 8, w: 60, h: 32 },
|
||||
tag: "button",
|
||||
},
|
||||
},
|
||||
{
|
||||
geometry: {
|
||||
rect: { x: 900, y: 8, w: 60, h: 32 },
|
||||
tag: "button",
|
||||
},
|
||||
},
|
||||
);
|
||||
expect(buffer.steps).toEqual([
|
||||
{
|
||||
op: "hover",
|
||||
captureTarget: { tag: "button", role: "button", name: "新建" },
|
||||
targetHint: {
|
||||
geometry: {
|
||||
rect: { x: 900, y: 8, w: 60, h: 32 },
|
||||
tag: "button",
|
||||
},
|
||||
},
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("annotates navigated_to on action-caused navigation instead of wait_for_navigation", () => {
|
||||
@@ -26,9 +59,11 @@ describe("recording-step-buffer", () => {
|
||||
captureTarget: { tag: "button", role: "button", name: "发布" },
|
||||
},
|
||||
],
|
||||
currentUrl: "https://example.com/a",
|
||||
pendingNavigation: true,
|
||||
pendingNavigationDeadline: Date.now() + 5_000,
|
||||
navigation: {
|
||||
currentUrl: "https://example.com/a",
|
||||
pendingNavigation: true,
|
||||
pendingNavigationDeadline: Date.now() + 5_000,
|
||||
},
|
||||
};
|
||||
observeRecordedNavigation(buffer, "https://example.com/b", true);
|
||||
expect(buffer.steps).toEqual([
|
||||
@@ -50,9 +85,11 @@ describe("recording-step-buffer", () => {
|
||||
values: ["tech"],
|
||||
},
|
||||
],
|
||||
currentUrl: "https://example.com/list",
|
||||
pendingNavigation: true,
|
||||
pendingNavigationDeadline: Date.now() + 5_000,
|
||||
navigation: {
|
||||
currentUrl: "https://example.com/list",
|
||||
pendingNavigation: true,
|
||||
pendingNavigationDeadline: Date.now() + 5_000,
|
||||
},
|
||||
};
|
||||
observeRecordedNavigation(buffer, "https://example.com/list?cat=tech", true);
|
||||
expect(buffer.steps).toEqual([
|
||||
@@ -68,13 +105,60 @@ describe("recording-step-buffer", () => {
|
||||
it("emits navigate for uncaused URL changes", () => {
|
||||
const buffer = {
|
||||
steps: [],
|
||||
currentUrl: "https://example.com/a",
|
||||
pendingNavigation: false,
|
||||
navigation: { currentUrl: "https://example.com/a", pendingNavigation: false },
|
||||
};
|
||||
observeRecordedNavigation(buffer, "https://example.com/b", false);
|
||||
const result = observeRecordedNavigation(buffer, "https://example.com/b", false);
|
||||
expect(result).toEqual({ kind: "appended", index: 0 });
|
||||
expect(buffer.steps[0]).toMatchObject({
|
||||
op: "navigate",
|
||||
url: "https://example.com/b",
|
||||
});
|
||||
});
|
||||
|
||||
it("asks the recorder to coalesce redirect hops instead of emitting each one", () => {
|
||||
const buffer = {
|
||||
steps: [],
|
||||
navigation: {
|
||||
currentUrl: "https://passport.example/login",
|
||||
pendingNavigation: false,
|
||||
},
|
||||
};
|
||||
const hop1 = observeRecordedNavigation(
|
||||
buffer,
|
||||
"https://passport.example/callback",
|
||||
false,
|
||||
"link",
|
||||
["server_redirect"],
|
||||
);
|
||||
expect(hop1).toEqual({
|
||||
kind: "coalesce_redirect",
|
||||
url: "https://passport.example/callback",
|
||||
});
|
||||
expect(buffer.steps).toEqual([]);
|
||||
|
||||
const hop2 = observeRecordedNavigation(buffer, "https://app.example/dashboard", false, "link", [
|
||||
"client_redirect",
|
||||
]);
|
||||
expect(hop2).toEqual({
|
||||
kind: "coalesce_redirect",
|
||||
url: "https://app.example/dashboard",
|
||||
});
|
||||
expect(buffer.steps).toEqual([]);
|
||||
expect(buffer.navigation.currentUrl).toBe("https://app.example/dashboard");
|
||||
});
|
||||
|
||||
it("does not reuse redirect metadata for a later content-observed URL change", () => {
|
||||
const buffer = {
|
||||
steps: [],
|
||||
navigation: {
|
||||
currentUrl: "https://example.com/redirect",
|
||||
pendingNavigation: false,
|
||||
},
|
||||
};
|
||||
|
||||
const result = observeRecordedNavigation(buffer, "https://example.com/spa");
|
||||
|
||||
expect(result).toEqual({ kind: "appended", index: 0 });
|
||||
expect(buffer.steps[0]).toMatchObject({ op: "navigate", url: "https://example.com/spa" });
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { RecordingTabCoordinator } from "../recording/tab-coordinator";
|
||||
|
||||
describe("RecordingTabCoordinator", () => {
|
||||
it("keeps navigation state isolated per tab", () => {
|
||||
const tabs = new RecordingTabCoordinator(4, "https://example.com/first");
|
||||
const first = tabs.navigation(4);
|
||||
first.pendingNavigation = true;
|
||||
const second = tabs.navigation(5, "https://example.com/second");
|
||||
|
||||
expect(second).toEqual({
|
||||
currentUrl: "https://example.com/second",
|
||||
pendingNavigation: false,
|
||||
});
|
||||
expect(tabs.navigation(4)).toBe(first);
|
||||
expect(tabs.navigation(4).pendingNavigation).toBe(true);
|
||||
});
|
||||
|
||||
it("invalidates an earlier activation when a newer tab becomes active", () => {
|
||||
const tabs = new RecordingTabCoordinator(4);
|
||||
const second = tabs.noteActivation(5);
|
||||
const third = tabs.noteActivation(6);
|
||||
|
||||
expect(tabs.isLatest(second)).toBe(false);
|
||||
expect(tabs.isLatest(third)).toBe(true);
|
||||
tabs.commit(6);
|
||||
expect(tabs.currentTabId).toBe(6);
|
||||
});
|
||||
|
||||
it("commits the freshly observed URL when returning to an existing tab", () => {
|
||||
const tabs = new RecordingTabCoordinator(4);
|
||||
tabs.navigation(5, "https://example.com/old");
|
||||
|
||||
tabs.commit(5, "https://example.com/new");
|
||||
|
||||
expect(tabs.currentTabId).toBe(5);
|
||||
expect(tabs.navigation(5).currentUrl).toBe("https://example.com/new");
|
||||
});
|
||||
});
|
||||
@@ -69,6 +69,23 @@ describe("matchObservationTarget", () => {
|
||||
expect(target.ref).toBe("e2");
|
||||
});
|
||||
|
||||
it("matches iframe capture geometry in the child viewport coordinate space", () => {
|
||||
const child = node(42, "child", 410);
|
||||
child.localRect = { x: 10, y: 20, w: 100, h: 30 };
|
||||
const target = matchObservationTarget({
|
||||
observation: observation(
|
||||
[child],
|
||||
[{ ref: "e1", backendNodeId: 42, frameId: "child", line: 1 }],
|
||||
),
|
||||
hint: {
|
||||
frameId: "child",
|
||||
geometrySpace: "local",
|
||||
geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" },
|
||||
},
|
||||
});
|
||||
expect(target.ref).toBe("e1");
|
||||
});
|
||||
|
||||
it("restricts a missing frame hint to the root frame", () => {
|
||||
const target = matchObservationTarget({
|
||||
observation: observation(
|
||||
@@ -81,6 +98,21 @@ describe("matchObservationTarget", () => {
|
||||
expect(target).toEqual({ name: "发布", unmatched: true });
|
||||
});
|
||||
|
||||
it("fails closed when the source Document has no VOM frame mapping", () => {
|
||||
const target = matchObservationTarget({
|
||||
observation: observation(
|
||||
[node(42, "root")],
|
||||
[{ ref: "e1", backendNodeId: 42, role: "button", name: "发布", line: 1 }],
|
||||
),
|
||||
hint: {
|
||||
frameId: null,
|
||||
geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" },
|
||||
},
|
||||
fallback: { tag: "button", role: "button", name: "发布" },
|
||||
});
|
||||
expect(target).toEqual({ role: "button", name: "发布", unmatched: true });
|
||||
});
|
||||
|
||||
it("returns unmatched for ambiguous geometry", () => {
|
||||
const target = matchObservationTarget({
|
||||
observation: observation(
|
||||
|
||||
@@ -27,6 +27,17 @@ describe("shouldRecordPress", () => {
|
||||
});
|
||||
|
||||
describe("reduceTraceSteps", () => {
|
||||
it("does not expose internal tab transitions in trace v2", () => {
|
||||
const trace = reduceTraceSteps([
|
||||
{
|
||||
op: "switch_tab",
|
||||
preStateId: "s1",
|
||||
postStateId: "s2",
|
||||
},
|
||||
]);
|
||||
expect(trace.steps).toEqual([]);
|
||||
});
|
||||
|
||||
it("builds steps with pages dictionary and page id references", () => {
|
||||
const drafts: RecordingDraftStep[] = [
|
||||
{
|
||||
@@ -160,7 +171,7 @@ describe("reduceTraceSteps", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps hover steps before menu clicks", () => {
|
||||
it("drops hover steps, which only exist in trace v3", () => {
|
||||
const { steps } = reduceTraceSteps(
|
||||
[
|
||||
{
|
||||
@@ -176,12 +187,7 @@ describe("reduceTraceSteps", () => {
|
||||
],
|
||||
"https://example.com/app",
|
||||
);
|
||||
expect(steps.map((s) => s.op)).toEqual(["hover", "click"]);
|
||||
expect(steps[0]).toMatchObject({
|
||||
op: "hover",
|
||||
page: "p1",
|
||||
target: { name: "Account" },
|
||||
});
|
||||
expect(steps.map((s) => s.op)).toEqual(["click"]);
|
||||
});
|
||||
|
||||
it("resolveTraceStartUrl prefers explicit start URL", () => {
|
||||
|
||||
@@ -6,6 +6,76 @@ import { reduceTraceStepsV3 } from "../recording/trace-reducer-v3";
|
||||
import type { RecordingDraftStep } from "../recording/types";
|
||||
|
||||
describe("trace reducer v3", () => {
|
||||
it("removes form literals when value redaction is requested", () => {
|
||||
const reduced = reduceTraceStepsV3(
|
||||
[
|
||||
{
|
||||
op: "fill",
|
||||
value: "private@example.com",
|
||||
preStateId: "s1",
|
||||
postStateId: "s1",
|
||||
},
|
||||
{
|
||||
op: "select",
|
||||
values: ["private-account-id"],
|
||||
labels: ["Private account"],
|
||||
preStateId: "s1",
|
||||
postStateId: "s1",
|
||||
},
|
||||
],
|
||||
{ redactValues: true },
|
||||
);
|
||||
|
||||
expect(reduced.steps[0]).toMatchObject({ op: "fill", value: "***", redacted: true });
|
||||
expect(reduced.steps[1]).toMatchObject({ op: "select" });
|
||||
expect(reduced.steps[1]).not.toHaveProperty("selection");
|
||||
expect(JSON.stringify(reduced.steps)).not.toContain("private@example.com");
|
||||
expect(JSON.stringify(reduced.steps)).not.toContain("private-account-id");
|
||||
});
|
||||
|
||||
it("emits tab transitions only for callers that advertised support", () => {
|
||||
const drafts: RecordingDraftStep[] = [
|
||||
{
|
||||
op: "switch_tab",
|
||||
preStateId: "s1",
|
||||
postStateId: "s2",
|
||||
},
|
||||
];
|
||||
|
||||
expect(reduceTraceStepsV3(drafts).steps).toEqual([]);
|
||||
expect(reduceTraceStepsV3(drafts, { includeTabSwitches: true }).steps).toEqual([
|
||||
{ op: "switch_tab", id: 1, state: "s1", result: { state: "s2" } },
|
||||
]);
|
||||
});
|
||||
|
||||
it("keeps hover steps before menu clicks", () => {
|
||||
const drafts: RecordingDraftStep[] = [
|
||||
{
|
||||
op: "hover",
|
||||
captureTarget: { tag: "span", role: "button", name: "Account" },
|
||||
preStateId: "s1",
|
||||
postStateId: "s1",
|
||||
},
|
||||
{
|
||||
op: "click",
|
||||
captureTarget: { tag: "a", role: "link", name: "Profile" },
|
||||
preStateId: "s1",
|
||||
postStateId: "s1",
|
||||
},
|
||||
];
|
||||
|
||||
expect(reduceTraceStepsV3(drafts).steps.map((s) => s.op)).toEqual(["hover", "click"]);
|
||||
});
|
||||
|
||||
it("does not export a tab transition without both observation endpoints", () => {
|
||||
const sourceOnly: RecordingDraftStep = { op: "switch_tab", preStateId: "s1" };
|
||||
const targetOnly: RecordingDraftStep = { op: "switch_tab", postStateId: "s2" };
|
||||
|
||||
expect(
|
||||
reduceTraceStepsV3([sourceOnly, targetOnly], { includeTabSwitches: true }).steps,
|
||||
).toEqual([]);
|
||||
});
|
||||
|
||||
it("collapses redirect hops while retaining draft-to-step identity", () => {
|
||||
const drafts: RecordingDraftStep[] = [
|
||||
{ op: "navigate", url: "https://example.com/start", preStateId: "s1", postStateId: "s2" },
|
||||
@@ -36,7 +106,7 @@ describe("trace reducer v3", () => {
|
||||
|
||||
it("builds the wire model from protocol constants", () => {
|
||||
const registry = new RecordingStateRegistry();
|
||||
const state = registry.register({ url: "https://example.com", rawVomText: "@vom 1" });
|
||||
const state = registry.register({ url: "https://example.com", vomText: "@vom 1" });
|
||||
const trace = buildTraceV3({
|
||||
registry,
|
||||
drafts: [
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
/**
|
||||
* Wire protocol and background-side helper for hiding the extension's
|
||||
* in-page overlay while a screenshot is captured, so captured frames only
|
||||
* contain the page itself.
|
||||
* Wire protocol and background-side helper for temporarily excluding the
|
||||
* extension's in-page overlay. Screenshots use it to avoid captured chrome;
|
||||
* coordinate-driven native input uses it so Chrome hit-tests the page.
|
||||
*
|
||||
* The background wraps every capture in `withOverlaysHiddenForCapture`,
|
||||
* The background wraps each bounded operation in `withExtensionOverlayHidden`,
|
||||
* which sends `begin` (the content script hides the overlay host and only
|
||||
* acks once the compositor has produced an overlay-free frame) and always
|
||||
* follows up with `end`, even when the capture throws. Tabs without the
|
||||
@@ -46,7 +46,7 @@ const defaultSendToTab: CaptureSuppressSendToTab = (tabId, message) =>
|
||||
* (or any other `begin` failure) simply skips suppression — there is no
|
||||
* overlay to hide in that tab.
|
||||
*/
|
||||
export async function withOverlaysHiddenForCapture<T>(
|
||||
export async function withExtensionOverlayHidden<T>(
|
||||
tabId: number,
|
||||
fn: () => Promise<T>,
|
||||
sendToTab: CaptureSuppressSendToTab = defaultSendToTab,
|
||||
|
||||
@@ -3,8 +3,10 @@
|
||||
* between the content-script control overlay and the background SW.
|
||||
*
|
||||
* Content script → background:
|
||||
* - `{ kind: "overlay.ready" }` → background replies with the authoritative
|
||||
* overlay state for the sender's window.
|
||||
* - `{ kind: "overlay.ready" }` → background decides the per-tab state first
|
||||
* (see overlayStateForTab) and pushes the authoritative overlay state for
|
||||
* that tab. It also proactively pushes on tab create and control-mode
|
||||
* change, so a tab usually receives state before it ever sends ready.
|
||||
* - `{ kind: "overlay.interrupt", sessionId }` → background asks the
|
||||
* daemon (via a `session.user_interrupt` WS event) to cancel
|
||||
* every inflight + queued tool call for that session with
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
/**
|
||||
* Generate a UUID v4 in browser contexts where randomUUID() is unavailable.
|
||||
*
|
||||
* Crypto.randomUUID() is restricted to secure contexts, while
|
||||
* crypto.getRandomValues() is available to content scripts on ordinary HTTP
|
||||
* origins. BrowserSkill records many intranet pages served over HTTP, so the
|
||||
* fallback is required for recording to work on those pages.
|
||||
*/
|
||||
export function createRandomUuid(): string {
|
||||
const cryptoApi = globalThis.crypto;
|
||||
if (typeof cryptoApi.randomUUID === "function") return cryptoApi.randomUUID();
|
||||
|
||||
const bytes = new Uint8Array(16);
|
||||
cryptoApi.getRandomValues(bytes);
|
||||
bytes[6] = (bytes[6]! & 0x0f) | 0x40;
|
||||
bytes[8] = (bytes[8]! & 0x3f) | 0x80;
|
||||
|
||||
const hex = Array.from(bytes, (byte) => byte.toString(16).padStart(2, "0")).join("");
|
||||
return [
|
||||
hex.slice(0, 8),
|
||||
hex.slice(8, 12),
|
||||
hex.slice(12, 16),
|
||||
hex.slice(16, 20),
|
||||
hex.slice(20),
|
||||
].join("-");
|
||||
}
|
||||
@@ -16,6 +16,10 @@ export interface RecordStartAck {
|
||||
ok: true;
|
||||
}
|
||||
|
||||
export type RecordStepAck =
|
||||
| { ok: true; sequence: number }
|
||||
| { ok: false; expectedSequence: number; error: string };
|
||||
|
||||
export type RecordStopAck = { ok: true } | { ok: false; error: string };
|
||||
|
||||
export interface RecordQueryMessage {
|
||||
@@ -50,17 +54,30 @@ export interface RecordStepPayload {
|
||||
labels?: string[];
|
||||
url?: string;
|
||||
redacted?: boolean;
|
||||
commit?: "enter" | "suggestion" | "blur";
|
||||
/** Page URL when the step was captured. */
|
||||
page_url?: string;
|
||||
/** Event-target bounds in the source frame's viewport coordinate space. */
|
||||
geometry?: {
|
||||
rect: { x: number; y: number; w: number; h: number };
|
||||
tag: string;
|
||||
};
|
||||
/** Capture-only hint; never persisted unless converted to navigated_to. */
|
||||
expects_navigation?: boolean;
|
||||
/** Whether an observed URL change was synchronously caused by the action. */
|
||||
navigation_caused_by_action?: boolean;
|
||||
/** Raw webNavigation transition metadata for cause mapping. */
|
||||
transitionType?: string;
|
||||
transitionQualifiers?: string[];
|
||||
}
|
||||
|
||||
export interface RecordStepMessage {
|
||||
type: typeof RECORD_STEP;
|
||||
requestId: string;
|
||||
/** Stable for the lifetime of one content-script document. */
|
||||
producerId: string;
|
||||
/** Monotonic within one content-script producer and recording request. */
|
||||
sequence: number;
|
||||
step: RecordStepPayload;
|
||||
}
|
||||
|
||||
@@ -112,8 +129,27 @@ export function isRecordQueryMessage(msg: unknown): msg is RecordQueryMessage {
|
||||
export function isRecordStepMessage(msg: unknown): msg is RecordStepMessage {
|
||||
if (typeof msg !== "object" || msg === null) return false;
|
||||
const m = msg as Record<string, unknown>;
|
||||
if (m.type !== RECORD_STEP || typeof m.requestId !== "string") return false;
|
||||
if (
|
||||
m.type !== RECORD_STEP ||
|
||||
typeof m.requestId !== "string" ||
|
||||
typeof m.producerId !== "string" ||
|
||||
m.producerId.length === 0 ||
|
||||
typeof m.sequence !== "number" ||
|
||||
!Number.isSafeInteger(m.sequence) ||
|
||||
m.sequence < 1
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
const step = m.step;
|
||||
if (typeof step !== "object" || step === null) return false;
|
||||
return typeof (step as RecordStepPayload).op === "string";
|
||||
}
|
||||
|
||||
export function isAcceptedRecordStepAck(
|
||||
value: unknown,
|
||||
sequence: number,
|
||||
): value is Extract<RecordStepAck, { ok: true }> {
|
||||
if (typeof value !== "object" || value === null) return false;
|
||||
const ack = value as Partial<RecordStepAck>;
|
||||
return ack.ok === true && ack.sequence === sequence;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
import {
|
||||
RECORD_DOCUMENT_ATTRIBUTE,
|
||||
recordingDocumentMarkerValue,
|
||||
} from "@/shared/recording-document-identity";
|
||||
|
||||
export interface RecordingDocumentMarker {
|
||||
restore(): void;
|
||||
ensure(): void;
|
||||
}
|
||||
|
||||
export function waitForDocumentElement(): Promise<HTMLElement> {
|
||||
if (document.documentElement) return Promise.resolve(document.documentElement);
|
||||
return new Promise((resolve) => {
|
||||
const observer = new MutationObserver(() => {
|
||||
if (!document.documentElement) return;
|
||||
observer.disconnect();
|
||||
resolve(document.documentElement);
|
||||
});
|
||||
observer.observe(document, { childList: true });
|
||||
});
|
||||
}
|
||||
|
||||
export function markRecordingDocument(
|
||||
producerId: string,
|
||||
root: HTMLElement = document.documentElement,
|
||||
): RecordingDocumentMarker {
|
||||
const previous = root.getAttribute(RECORD_DOCUMENT_ATTRIBUTE);
|
||||
const markerValue = recordingDocumentMarkerValue(producerId);
|
||||
const ensure = () => root.setAttribute(RECORD_DOCUMENT_ATTRIBUTE, markerValue);
|
||||
ensure();
|
||||
return {
|
||||
ensure,
|
||||
restore() {
|
||||
if (root.getAttribute(RECORD_DOCUMENT_ATTRIBUTE) !== markerValue) return;
|
||||
if (previous === null) root.removeAttribute(RECORD_DOCUMENT_ATTRIBUTE);
|
||||
else root.setAttribute(RECORD_DOCUMENT_ATTRIBUTE, previous);
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
export const RECORD_FRAME_PORT = "bsk-record-frame";
|
||||
export const RECORD_FRAME_QUERY = "bsk-record-frame-query";
|
||||
export const RECORD_FRAME_START = "bsk-record-frame-start";
|
||||
|
||||
export interface RecordFrameQueryMessage {
|
||||
type: typeof RECORD_FRAME_QUERY;
|
||||
}
|
||||
|
||||
export interface RecordFrameQueryResponse {
|
||||
active: boolean;
|
||||
requestId?: string;
|
||||
startedAtMs?: number;
|
||||
}
|
||||
|
||||
export interface RecordFrameStartMessage {
|
||||
type: typeof RECORD_FRAME_START;
|
||||
requestId: string;
|
||||
startedAtMs: number;
|
||||
}
|
||||
|
||||
export type RecordFramePortMessage =
|
||||
| {
|
||||
type: "ready";
|
||||
requestId: string;
|
||||
producerId: string;
|
||||
}
|
||||
| {
|
||||
type: "ready_ack";
|
||||
requestId: string;
|
||||
producerId: string;
|
||||
}
|
||||
| {
|
||||
type: "stop";
|
||||
requestId: string;
|
||||
commandId: string;
|
||||
}
|
||||
| {
|
||||
type: "cancel";
|
||||
requestId: string;
|
||||
}
|
||||
| {
|
||||
type: "stopped";
|
||||
requestId: string;
|
||||
commandId: string;
|
||||
ok: boolean;
|
||||
error?: string;
|
||||
};
|
||||
|
||||
export function isRecordFrameQueryMessage(value: unknown): value is RecordFrameQueryMessage {
|
||||
return (
|
||||
typeof value === "object" &&
|
||||
value !== null &&
|
||||
(value as { type?: unknown }).type === RECORD_FRAME_QUERY
|
||||
);
|
||||
}
|
||||
|
||||
export function isRecordFrameStartMessage(value: unknown): value is RecordFrameStartMessage {
|
||||
if (typeof value !== "object" || value === null) return false;
|
||||
const message = value as Partial<RecordFrameStartMessage>;
|
||||
return (
|
||||
message.type === RECORD_FRAME_START &&
|
||||
typeof message.requestId === "string" &&
|
||||
typeof message.startedAtMs === "number"
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,400 @@
|
||||
import { createRandomUuid } from "../random-uuid";
|
||||
import {
|
||||
isRecordFrameQueryMessage,
|
||||
RECORD_FRAME_PORT,
|
||||
RECORD_FRAME_START,
|
||||
type RecordFramePortMessage,
|
||||
type RecordFrameQueryResponse,
|
||||
type RecordFrameStartMessage,
|
||||
} from "./frame-bridge";
|
||||
|
||||
export interface RecordingCaptureScope {
|
||||
tabId: number;
|
||||
documentId: string;
|
||||
browserFrameId: number;
|
||||
producerId: string;
|
||||
}
|
||||
|
||||
interface ArmedRecording {
|
||||
requestId: string;
|
||||
startedAtMs: number;
|
||||
tabIds: Set<number>;
|
||||
agents: Map<string, FrameAgent>;
|
||||
onDocumentReady?: (scope: RecordingCaptureScope) => void | Promise<void>;
|
||||
finishing: boolean;
|
||||
}
|
||||
|
||||
interface FrameAgent extends RecordingCaptureScope {
|
||||
requestId: string;
|
||||
port: chrome.runtime.Port;
|
||||
stopWaiters: Map<string, (ok: boolean) => void>;
|
||||
}
|
||||
|
||||
interface BrowserFrame {
|
||||
frameId: number;
|
||||
documentId?: string;
|
||||
}
|
||||
|
||||
interface BrowserFrameNavigation extends BrowserFrame {
|
||||
tabId: number;
|
||||
lifecycle: "committed" | "completed";
|
||||
}
|
||||
|
||||
const RECORD_FRAME_STOP_TIMEOUT_MS = 5_000;
|
||||
|
||||
export interface RecordFrameCoordinatorDeps {
|
||||
getAllFrames(tabId: number): Promise<BrowserFrame[]>;
|
||||
sendToDocument(
|
||||
tabId: number,
|
||||
message: RecordFrameStartMessage,
|
||||
target: { documentId?: string; frameId?: number },
|
||||
): Promise<unknown>;
|
||||
subscribeFrameNavigation?(listener: (frame: BrowserFrameNavigation) => void): () => void;
|
||||
}
|
||||
|
||||
function documentKey(tabId: number, documentId: string): string {
|
||||
return `${tabId}:${documentId}`;
|
||||
}
|
||||
|
||||
function senderAddress(
|
||||
sender: chrome.runtime.MessageSender,
|
||||
): { tabId: number; documentId: string; browserFrameId: number } | null {
|
||||
const tabId = sender.tab?.id;
|
||||
const documentId = sender.documentId;
|
||||
const browserFrameId = sender.frameId;
|
||||
if (
|
||||
typeof tabId !== "number" ||
|
||||
typeof documentId !== "string" ||
|
||||
typeof browserFrameId !== "number"
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
return { tabId, documentId, browserFrameId };
|
||||
}
|
||||
|
||||
function defaultDeps(): RecordFrameCoordinatorDeps {
|
||||
return {
|
||||
async getAllFrames(tabId) {
|
||||
return (await chrome.webNavigation.getAllFrames({ tabId })) ?? [];
|
||||
},
|
||||
sendToDocument(tabId, message, target) {
|
||||
return chrome.tabs.sendMessage(tabId, message, target);
|
||||
},
|
||||
subscribeFrameNavigation(listener) {
|
||||
const onCommitted = (details: chrome.webNavigation.WebNavigationTransitionCallbackDetails) =>
|
||||
listener({ ...details, lifecycle: "committed" });
|
||||
const onCompleted = (details: chrome.webNavigation.WebNavigationFramedCallbackDetails) =>
|
||||
listener({ ...details, lifecycle: "completed" });
|
||||
chrome.webNavigation.onCommitted.addListener(onCommitted);
|
||||
chrome.webNavigation.onCompleted.addListener(onCompleted);
|
||||
return () => {
|
||||
chrome.webNavigation.onCommitted.removeListener(onCommitted);
|
||||
chrome.webNavigation.onCompleted.removeListener(onCompleted);
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export class RecordFrameCoordinator {
|
||||
readonly #deps: RecordFrameCoordinatorDeps;
|
||||
readonly #recordings = new Map<string, ArmedRecording>();
|
||||
#attached = false;
|
||||
#detachFrameNavigation: (() => void) | null = null;
|
||||
|
||||
constructor(deps: RecordFrameCoordinatorDeps = defaultDeps()) {
|
||||
this.#deps = deps;
|
||||
}
|
||||
|
||||
attach(): () => void {
|
||||
if (this.#attached) return () => {};
|
||||
this.#attached = true;
|
||||
chrome.runtime.onConnect.addListener(this.#onConnect);
|
||||
chrome.runtime.onMessage.addListener(this.#onMessage);
|
||||
this.#detachFrameNavigation =
|
||||
this.#deps.subscribeFrameNavigation?.(this.#onFrameNavigation) ?? null;
|
||||
return () => {
|
||||
if (!this.#attached) return;
|
||||
this.#attached = false;
|
||||
chrome.runtime.onConnect.removeListener(this.#onConnect);
|
||||
chrome.runtime.onMessage.removeListener(this.#onMessage);
|
||||
this.#detachFrameNavigation?.();
|
||||
this.#detachFrameNavigation = null;
|
||||
for (const recording of this.#recordings.values()) this.#cancelAgents(recording);
|
||||
this.#recordings.clear();
|
||||
};
|
||||
}
|
||||
|
||||
begin(
|
||||
requestId: string,
|
||||
startedAtMs: number,
|
||||
initialTabId: number,
|
||||
onDocumentReady?: (scope: RecordingCaptureScope) => void | Promise<void>,
|
||||
): void {
|
||||
const previous = this.#recordings.get(requestId);
|
||||
if (previous) this.#cancelAgents(previous);
|
||||
this.#recordings.set(requestId, {
|
||||
requestId,
|
||||
startedAtMs,
|
||||
// Register the initial tab before navigation begins. A newly-created
|
||||
// child document can query at document_start before armTab's frame
|
||||
// snapshot sees it, especially when Chromium promotes it to an OOPIF.
|
||||
tabIds: new Set([initialTabId]),
|
||||
agents: new Map(),
|
||||
...(onDocumentReady ? { onDocumentReady } : {}),
|
||||
finishing: false,
|
||||
});
|
||||
}
|
||||
|
||||
async armTab(requestId: string, tabId: number): Promise<boolean> {
|
||||
const recording = this.#recordings.get(requestId);
|
||||
if (!recording || recording.finishing) return false;
|
||||
recording.tabIds.add(tabId);
|
||||
|
||||
let frames: BrowserFrame[];
|
||||
try {
|
||||
frames = await this.#deps.getAllFrames(tabId);
|
||||
} catch {
|
||||
frames = [{ frameId: 0 }];
|
||||
}
|
||||
if (!frames.some((frame) => frame.frameId === 0)) frames.unshift({ frameId: 0 });
|
||||
|
||||
const results = await Promise.all(
|
||||
frames.map(async (frame) => {
|
||||
const started = await this.#startFrame(recording, tabId, frame);
|
||||
return { frameId: frame.frameId, started };
|
||||
}),
|
||||
);
|
||||
return results.some((result) => result.frameId === 0 && result.started);
|
||||
}
|
||||
|
||||
sourceFor(
|
||||
requestId: string,
|
||||
producerId: string,
|
||||
sender: chrome.runtime.MessageSender,
|
||||
): RecordingCaptureScope | null {
|
||||
const address = senderAddress(sender);
|
||||
const recording = this.#recordings.get(requestId);
|
||||
if (!address || !recording) return null;
|
||||
const agent = recording.agents.get(documentKey(address.tabId, address.documentId));
|
||||
if (!agent || agent.producerId !== producerId) return null;
|
||||
return {
|
||||
tabId: agent.tabId,
|
||||
documentId: agent.documentId,
|
||||
browserFrameId: agent.browserFrameId,
|
||||
producerId: agent.producerId,
|
||||
};
|
||||
}
|
||||
|
||||
async stop(requestId: string): Promise<boolean> {
|
||||
const recording = this.#recordings.get(requestId);
|
||||
if (!recording) return true;
|
||||
recording.finishing = true;
|
||||
const commandId = createRandomUuid();
|
||||
const results = await Promise.all(
|
||||
[...recording.agents.values()].map((agent) => this.#stopAgent(agent, requestId, commandId)),
|
||||
);
|
||||
const succeeded = results.every(Boolean);
|
||||
if (succeeded) this.#recordings.delete(requestId);
|
||||
else recording.finishing = false;
|
||||
return succeeded;
|
||||
}
|
||||
|
||||
cancel(requestId: string): void {
|
||||
const recording = this.#recordings.get(requestId);
|
||||
if (!recording) return;
|
||||
this.#cancelAgents(recording);
|
||||
this.#recordings.delete(requestId);
|
||||
}
|
||||
|
||||
readonly #onMessage = (
|
||||
message: unknown,
|
||||
sender: chrome.runtime.MessageSender,
|
||||
sendResponse: (response: RecordFrameQueryResponse) => void,
|
||||
): boolean => {
|
||||
if (!isRecordFrameQueryMessage(message)) return false;
|
||||
const tabId = sender.tab?.id;
|
||||
const recording =
|
||||
typeof tabId === "number"
|
||||
? [...this.#recordings.values()].find(
|
||||
(candidate) => !candidate.finishing && candidate.tabIds.has(tabId),
|
||||
)
|
||||
: undefined;
|
||||
sendResponse(
|
||||
recording
|
||||
? {
|
||||
active: true,
|
||||
requestId: recording.requestId,
|
||||
startedAtMs: recording.startedAtMs,
|
||||
}
|
||||
: { active: false },
|
||||
);
|
||||
return false;
|
||||
};
|
||||
|
||||
readonly #onFrameNavigation = (frame: BrowserFrameNavigation): void => {
|
||||
if (frame.frameId === 0) return;
|
||||
for (const recording of this.#recordings.values()) {
|
||||
if (recording.finishing || !recording.tabIds.has(frame.tabId)) continue;
|
||||
// onCommitted can race document_start, while onCompleted runs after the
|
||||
// content script is available. Starting twice is safe and closes both
|
||||
// the early-query and late-SPA-frame gaps.
|
||||
void (async () => {
|
||||
const started = await this.#startFrame(recording, frame.tabId, frame);
|
||||
if (!started || frame.lifecycle !== "completed" || recording.finishing) return;
|
||||
const agent = frame.documentId
|
||||
? recording.agents.get(documentKey(frame.tabId, frame.documentId))
|
||||
: [...recording.agents.values()].find(
|
||||
(candidate) =>
|
||||
candidate.tabId === frame.tabId && candidate.browserFrameId === frame.frameId,
|
||||
);
|
||||
if (!agent) return;
|
||||
await recording.onDocumentReady?.({
|
||||
tabId: agent.tabId,
|
||||
documentId: agent.documentId,
|
||||
browserFrameId: agent.browserFrameId,
|
||||
producerId: agent.producerId,
|
||||
});
|
||||
})();
|
||||
}
|
||||
};
|
||||
|
||||
readonly #onConnect = (port: chrome.runtime.Port): void => {
|
||||
if (port.name !== RECORD_FRAME_PORT) return;
|
||||
const address = senderAddress(port.sender ?? {});
|
||||
if (!address) {
|
||||
port.disconnect();
|
||||
return;
|
||||
}
|
||||
let registered: FrameAgent | null = null;
|
||||
port.onMessage.addListener((raw: unknown) => {
|
||||
const message = raw as Partial<RecordFramePortMessage>;
|
||||
if (message.type === "ready") {
|
||||
if (typeof message.requestId !== "string" || typeof message.producerId !== "string") {
|
||||
port.disconnect();
|
||||
return;
|
||||
}
|
||||
const requestId = message.requestId;
|
||||
const producerId = message.producerId;
|
||||
if (registered) {
|
||||
if (registered.requestId === requestId && registered.producerId === producerId) {
|
||||
port.postMessage({
|
||||
type: "ready_ack",
|
||||
requestId,
|
||||
producerId,
|
||||
} satisfies RecordFramePortMessage);
|
||||
} else {
|
||||
port.disconnect();
|
||||
}
|
||||
return;
|
||||
}
|
||||
const recording = this.#recordings.get(requestId);
|
||||
if (
|
||||
!recording ||
|
||||
recording.finishing ||
|
||||
!recording.tabIds.has(address.tabId) ||
|
||||
producerId.length === 0
|
||||
) {
|
||||
port.disconnect();
|
||||
return;
|
||||
}
|
||||
const agent: FrameAgent = {
|
||||
...address,
|
||||
requestId,
|
||||
producerId,
|
||||
port,
|
||||
stopWaiters: new Map(),
|
||||
};
|
||||
registered = agent;
|
||||
const key = documentKey(address.tabId, address.documentId);
|
||||
const previous = recording.agents.get(key);
|
||||
if (previous && previous !== agent) previous.port.disconnect();
|
||||
recording.agents.set(key, agent);
|
||||
port.postMessage({
|
||||
type: "ready_ack",
|
||||
requestId,
|
||||
producerId,
|
||||
} satisfies RecordFramePortMessage);
|
||||
return;
|
||||
}
|
||||
if (
|
||||
message.type === "stopped" &&
|
||||
registered &&
|
||||
message.requestId === registered.requestId &&
|
||||
typeof message.commandId === "string"
|
||||
) {
|
||||
registered.stopWaiters.get(message.commandId)?.(message.ok === true);
|
||||
registered.stopWaiters.delete(message.commandId);
|
||||
}
|
||||
});
|
||||
port.onDisconnect.addListener(() => {
|
||||
if (!registered) return;
|
||||
for (const resolve of registered.stopWaiters.values()) resolve(true);
|
||||
const recording = this.#recordings.get(registered.requestId);
|
||||
const key = documentKey(registered.tabId, registered.documentId);
|
||||
if (recording?.agents.get(key) === registered) recording.agents.delete(key);
|
||||
});
|
||||
};
|
||||
|
||||
#stopAgent(agent: FrameAgent, requestId: string, commandId: string): Promise<boolean> {
|
||||
return new Promise((resolve) => {
|
||||
const timer = setTimeout(() => {
|
||||
agent.stopWaiters.delete(commandId);
|
||||
resolve(false);
|
||||
}, RECORD_FRAME_STOP_TIMEOUT_MS);
|
||||
agent.stopWaiters.set(commandId, (ok) => {
|
||||
clearTimeout(timer);
|
||||
resolve(ok);
|
||||
});
|
||||
try {
|
||||
agent.port.postMessage({
|
||||
type: "stop",
|
||||
requestId,
|
||||
commandId,
|
||||
} satisfies RecordFramePortMessage);
|
||||
} catch {
|
||||
clearTimeout(timer);
|
||||
agent.stopWaiters.delete(commandId);
|
||||
resolve(true);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async #startFrame(
|
||||
recording: ArmedRecording,
|
||||
tabId: number,
|
||||
frame: BrowserFrame,
|
||||
): Promise<boolean> {
|
||||
const message: RecordFrameStartMessage = {
|
||||
type: RECORD_FRAME_START,
|
||||
requestId: recording.requestId,
|
||||
startedAtMs: recording.startedAtMs,
|
||||
};
|
||||
try {
|
||||
const response = await this.#deps.sendToDocument(tabId, message, {
|
||||
...(frame.documentId ? { documentId: frame.documentId } : { frameId: frame.frameId }),
|
||||
});
|
||||
return isStarted(response);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
#cancelAgents(recording: ArmedRecording): void {
|
||||
for (const agent of recording.agents.values()) {
|
||||
try {
|
||||
agent.port.postMessage({
|
||||
type: "cancel",
|
||||
requestId: recording.requestId,
|
||||
} satisfies RecordFramePortMessage);
|
||||
} catch {
|
||||
// A destroyed Document has no remaining capture state to cancel.
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function isStarted(value: unknown): boolean {
|
||||
return typeof value === "object" && value !== null && (value as { ok?: unknown }).ok === true;
|
||||
}
|
||||
|
||||
export const recordFrameCoordinator = new RecordFrameCoordinator();
|
||||
@@ -0,0 +1,5 @@
|
||||
const REDIRECT_QUALIFIERS = new Set(["client_redirect", "server_redirect"]);
|
||||
|
||||
export function hasRedirectQualifier(qualifiers?: readonly string[]): boolean {
|
||||
return (qualifiers ?? []).some((qualifier) => REDIRECT_QUALIFIERS.has(qualifier));
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
import type { RenderedRef } from "@browser-skill/vom";
|
||||
import type { CdpTarget } from "@/browser-driver/frame-graph";
|
||||
import {
|
||||
type CaptureVomMatchNode,
|
||||
type CaptureVomObservationResult,
|
||||
@@ -28,6 +29,11 @@ export interface RegisteredObservation {
|
||||
url: string;
|
||||
}
|
||||
|
||||
export interface RecordingDocumentScope {
|
||||
frameId: string;
|
||||
target: CdpTarget;
|
||||
}
|
||||
|
||||
function nodeKey(frameId: string, backendNodeId: number): string {
|
||||
return `${frameId}:${backendNodeId}`;
|
||||
}
|
||||
@@ -40,8 +46,12 @@ export class ObservationNodeIndex {
|
||||
readonly #nodesByFrameTag = new Map<string, IndexedObservationNode[]>();
|
||||
readonly #refById = new Map<string, RenderedRef>();
|
||||
readonly #refsByFrame = new Map<string, RenderedRef[]>();
|
||||
readonly #scopeByProducer = new Map<string, RecordingDocumentScope | null>();
|
||||
|
||||
constructor(input: Pick<CaptureVomObservationResult, "rootFrameId" | "matchNodes" | "refs">) {
|
||||
constructor(
|
||||
input: Pick<CaptureVomObservationResult, "rootFrameId" | "matchNodes" | "refs"> &
|
||||
Partial<Pick<CaptureVomObservationResult, "frames">>,
|
||||
) {
|
||||
const refByNode = new Map<string, RenderedRef>();
|
||||
for (const ref of input.refs) {
|
||||
const frameId = ref.frameId ?? input.rootFrameId;
|
||||
@@ -51,6 +61,16 @@ export class ObservationNodeIndex {
|
||||
frameRefs.push(ref);
|
||||
this.#refsByFrame.set(frameId, frameRefs);
|
||||
}
|
||||
for (const frame of input.frames ?? []) {
|
||||
const documentId = frame.recordingDocumentId;
|
||||
if (!documentId) continue;
|
||||
this.#scopeByProducer.set(
|
||||
documentId,
|
||||
this.#scopeByProducer.has(documentId)
|
||||
? null
|
||||
: { frameId: frame.frameId, target: frame.target },
|
||||
);
|
||||
}
|
||||
for (const geometry of input.matchNodes) {
|
||||
const { frameId } = geometry;
|
||||
const entry = {
|
||||
@@ -76,6 +96,10 @@ export class ObservationNodeIndex {
|
||||
refs(frameId: string): readonly RenderedRef[] {
|
||||
return this.#refsByFrame.get(frameId) ?? [];
|
||||
}
|
||||
|
||||
documentScope(producerId: string): RecordingDocumentScope | undefined {
|
||||
return this.#scopeByProducer.get(producerId) ?? undefined;
|
||||
}
|
||||
}
|
||||
|
||||
async function readTabMeta(
|
||||
|
||||
@@ -28,7 +28,13 @@ function abortableDelay(ms: number, signal?: AbortSignal): Promise<void> {
|
||||
}
|
||||
|
||||
function isTargeted(draft: RecordingDraftStep): draft is TargetedRecordingDraft {
|
||||
return draft.op !== "navigate" && draft.op !== "scroll";
|
||||
return (
|
||||
draft.op === "click" ||
|
||||
draft.op === "hover" ||
|
||||
draft.op === "fill" ||
|
||||
draft.op === "press" ||
|
||||
draft.op === "select"
|
||||
);
|
||||
}
|
||||
|
||||
export class RecordingObservationSession {
|
||||
@@ -74,7 +80,7 @@ export class RecordingObservationSession {
|
||||
const state = this.registry.register({
|
||||
url: captured.url,
|
||||
title: captured.title,
|
||||
rawVomText: captured.vomText,
|
||||
vomText: captured.vomText,
|
||||
truncated: captured.truncated,
|
||||
});
|
||||
const observation: RegisteredObservation = {
|
||||
|
||||
@@ -0,0 +1,277 @@
|
||||
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
|
||||
import type { StopReason, TraceV3 } from "@/transport/types";
|
||||
import { type DocumentSettleScope, waitForDocumentSettled } from "./document-settle";
|
||||
import type { RegisteredObservation } from "./observation-capture";
|
||||
import { RecordingObservationSession } from "./observation-session";
|
||||
import { inferMissingPostStates, SettleController } from "./settle-controller";
|
||||
import { RecordingStateRegistry } from "./state-registry";
|
||||
import { buildTraceV3 } from "./trace-builder-v3";
|
||||
import type { RecordingDraftStep, StepAnnotation } from "./types";
|
||||
|
||||
interface TabRecordingContext {
|
||||
session: RecordingObservationSession;
|
||||
settle: SettleController;
|
||||
pendingCapture: PendingCapture | null;
|
||||
frameRefresh: Promise<void>;
|
||||
frameRefreshAbort: AbortController | null;
|
||||
}
|
||||
|
||||
interface PendingCapture {
|
||||
promise: Promise<RegisteredObservation>;
|
||||
abort: AbortController;
|
||||
}
|
||||
|
||||
export class RecordingObservationRuntime {
|
||||
readonly #cdp: CdpRunner;
|
||||
readonly #tabsApi: ChromeTabsApi;
|
||||
readonly #registry = new RecordingStateRegistry();
|
||||
readonly #annotations: StepAnnotation[] = [];
|
||||
readonly #contexts = new Map<number, TabRecordingContext>();
|
||||
readonly #maxTokens?: number;
|
||||
readonly #redactValues: boolean;
|
||||
|
||||
constructor(input: {
|
||||
cdp: CdpRunner;
|
||||
tabsApi: ChromeTabsApi;
|
||||
maxTokens?: number;
|
||||
redactValues?: boolean;
|
||||
}) {
|
||||
this.#cdp = input.cdp;
|
||||
this.#tabsApi = input.tabsApi;
|
||||
this.#maxTokens = input.maxTokens;
|
||||
this.#redactValues = input.redactValues ?? false;
|
||||
}
|
||||
|
||||
#context(tabId: number): TabRecordingContext {
|
||||
const existing = this.#contexts.get(tabId);
|
||||
if (existing) return existing;
|
||||
const session = new RecordingObservationSession({
|
||||
registry: this.#registry,
|
||||
annotations: this.#annotations,
|
||||
maxTokens: this.#maxTokens,
|
||||
redactValues: this.#redactValues,
|
||||
});
|
||||
const context: TabRecordingContext = {
|
||||
session,
|
||||
settle: new SettleController({
|
||||
session,
|
||||
cdp: this.#cdp,
|
||||
tabsApi: this.#tabsApi,
|
||||
tabId,
|
||||
}),
|
||||
pendingCapture: null,
|
||||
frameRefresh: Promise.resolve(),
|
||||
frameRefreshAbort: null,
|
||||
};
|
||||
this.#contexts.set(tabId, context);
|
||||
return context;
|
||||
}
|
||||
|
||||
async #capture(tabId: number): Promise<RegisteredObservation> {
|
||||
const context = this.#context(tabId);
|
||||
if (context.pendingCapture) return context.pendingCapture.promise;
|
||||
|
||||
const abort = new AbortController();
|
||||
let pending!: PendingCapture;
|
||||
const promise = context.session.capture(this.#cdp, this.#tabsApi, tabId, abort.signal);
|
||||
pending = {
|
||||
abort,
|
||||
promise: promise.finally(() => {
|
||||
if (context.pendingCapture === pending) context.pendingCapture = null;
|
||||
}),
|
||||
};
|
||||
context.pendingCapture = pending;
|
||||
return pending.promise;
|
||||
}
|
||||
|
||||
async captureInitial(tabId: number): Promise<void> {
|
||||
const context = this.#context(tabId);
|
||||
if (context.session.cursor.lastSettled) return;
|
||||
await this.#capture(tabId);
|
||||
}
|
||||
|
||||
/** Refresh the safe observation after a late child Document becomes usable. */
|
||||
async refreshDocument(tabId: number, producerId: string): Promise<void> {
|
||||
const context = this.#context(tabId);
|
||||
context.frameRefreshAbort?.abort();
|
||||
const abort = new AbortController();
|
||||
context.frameRefreshAbort = abort;
|
||||
const previous = context.frameRefresh;
|
||||
const refresh = (async () => {
|
||||
await previous;
|
||||
if (abort.signal.aborted) return;
|
||||
if (context.pendingCapture) await Promise.allSettled([context.pendingCapture.promise]);
|
||||
if (abort.signal.aborted) return;
|
||||
|
||||
// The first capture discovers the safe producer -> CDP Document scope.
|
||||
await this.#capture(tabId);
|
||||
if (abort.signal.aborted) return;
|
||||
const scope = context.session.cursor.lastSettled?.index.documentScope(producerId);
|
||||
if (scope) {
|
||||
await waitForDocumentSettled(this.#cdp, scope, { signal: abort.signal });
|
||||
}
|
||||
if (abort.signal.aborted) return;
|
||||
|
||||
// Capture again after the child SPA is quiet so the next action can be
|
||||
// matched against refs that actually exist in its pre-action state.
|
||||
await this.#capture(tabId);
|
||||
})();
|
||||
const tracked = refresh.catch(() => {});
|
||||
context.frameRefresh = tracked;
|
||||
try {
|
||||
await refresh;
|
||||
} finally {
|
||||
if (context.frameRefresh === tracked) context.frameRefreshAbort = null;
|
||||
}
|
||||
}
|
||||
|
||||
async captureTabTransition(
|
||||
fromTabId: number,
|
||||
toTabId: number,
|
||||
): Promise<{ preStateId?: string; postStateId?: string; targetUrl?: string }> {
|
||||
const from = this.#context(fromTabId);
|
||||
await this.#flushContext(from);
|
||||
if (!from.session.cursor.lastSettled) {
|
||||
try {
|
||||
await this.captureInitial(fromTabId);
|
||||
} catch {
|
||||
// A target-tab observation can still preserve the forward transition.
|
||||
}
|
||||
}
|
||||
|
||||
const to = this.#context(toTabId);
|
||||
let target = to.session.cursor.lastSettled;
|
||||
try {
|
||||
target = await this.#capture(toTabId);
|
||||
} catch {
|
||||
// Reuse the last settled target state when a refresh races navigation.
|
||||
}
|
||||
|
||||
const source = from.session.cursor.lastSettled;
|
||||
return {
|
||||
...(source ? { preStateId: source.stateId } : {}),
|
||||
...(target ? { postStateId: target.stateId, targetUrl: target.url } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
bindTabTransition(
|
||||
draft: Extract<RecordingDraftStep, { op: "switch_tab" }>,
|
||||
draftId: number,
|
||||
): void {
|
||||
if (draft.preStateId) this.#registry.markStep(draft.preStateId, draftId);
|
||||
}
|
||||
|
||||
async processDraft(
|
||||
tabId: number,
|
||||
drafts: RecordingDraftStep[],
|
||||
draftIndex: number,
|
||||
producerId?: string,
|
||||
): Promise<void> {
|
||||
const draft = drafts[draftIndex];
|
||||
if (!draft) return;
|
||||
const context = this.#context(tabId);
|
||||
if (!context.session.cursor.lastSettled) {
|
||||
try {
|
||||
await this.captureInitial(tabId);
|
||||
} catch {
|
||||
// Post-action settle can still provide a usable state.
|
||||
}
|
||||
}
|
||||
const scope = producerId
|
||||
? context.session.cursor.lastSettled?.index.documentScope(producerId)
|
||||
: undefined;
|
||||
if (
|
||||
producerId &&
|
||||
(draft.op === "click" ||
|
||||
draft.op === "hover" ||
|
||||
draft.op === "fill" ||
|
||||
draft.op === "press" ||
|
||||
draft.op === "select")
|
||||
) {
|
||||
draft.targetHint = {
|
||||
...(draft.targetHint ?? {}),
|
||||
frameId: scope?.frameId ?? null,
|
||||
geometrySpace: "local",
|
||||
};
|
||||
}
|
||||
context.session.bindDraft(draft, draftIndex + 1, context.settle.hasPending);
|
||||
context.settle.schedule(drafts, draftIndex, scope);
|
||||
}
|
||||
|
||||
scheduleSettle(
|
||||
tabId: number,
|
||||
drafts: RecordingDraftStep[],
|
||||
draftIndex: number,
|
||||
scope?: DocumentSettleScope,
|
||||
): void {
|
||||
this.#context(tabId).settle.schedule(drafts, draftIndex, scope);
|
||||
}
|
||||
|
||||
clearRedirect(tabId: number): void {
|
||||
this.#contexts.get(tabId)?.settle.clearRedirect();
|
||||
}
|
||||
|
||||
scheduleRedirect(tabId: number, drafts: RecordingDraftStep[], url: string): void {
|
||||
this.#context(tabId).settle.scheduleRedirect(drafts, url);
|
||||
}
|
||||
|
||||
async #flushContext(context: TabRecordingContext): Promise<void> {
|
||||
await context.frameRefresh;
|
||||
if (context.pendingCapture) await Promise.allSettled([context.pendingCapture.promise]);
|
||||
await context.settle.flushRedirects();
|
||||
await context.settle.flush();
|
||||
}
|
||||
|
||||
async flushRedirects(tabId: number): Promise<void> {
|
||||
await this.#contexts.get(tabId)?.settle.flushRedirects();
|
||||
}
|
||||
|
||||
async flush(): Promise<void> {
|
||||
for (const context of this.#contexts.values()) await this.#flushContext(context);
|
||||
}
|
||||
|
||||
async settleTrailing(tabId: number, drafts: RecordingDraftStep[]): Promise<void> {
|
||||
const context = this.#context(tabId);
|
||||
if (context.pendingCapture) await Promise.allSettled([context.pendingCapture.promise]);
|
||||
if (!context.session.cursor.lastSettled) {
|
||||
try {
|
||||
await this.captureInitial(tabId);
|
||||
} catch {
|
||||
// The trace may remain observation-free when CDP is unavailable.
|
||||
}
|
||||
}
|
||||
await context.settle.settleTrailing(drafts);
|
||||
inferMissingPostStates(drafts);
|
||||
}
|
||||
|
||||
cancel(): void {
|
||||
for (const context of this.#contexts.values()) {
|
||||
context.frameRefreshAbort?.abort();
|
||||
context.pendingCapture?.abort.abort();
|
||||
context.settle.cancel();
|
||||
}
|
||||
}
|
||||
|
||||
buildTrace(input: {
|
||||
drafts: RecordingDraftStep[];
|
||||
startedAt: string;
|
||||
purpose?: string;
|
||||
startUrl?: string;
|
||||
stoppedBy: StopReason;
|
||||
bskVersion: string;
|
||||
includeTabSwitches?: boolean;
|
||||
}): TraceV3 {
|
||||
return buildTraceV3({
|
||||
registry: this.#registry,
|
||||
drafts: input.drafts,
|
||||
annotations: this.#annotations,
|
||||
startedAt: input.startedAt,
|
||||
purpose: input.purpose,
|
||||
startUrl: input.startUrl,
|
||||
stoppedBy: input.stoppedBy,
|
||||
bskVersion: input.bskVersion,
|
||||
redactValues: this.#redactValues,
|
||||
includeTabSwitches: input.includeTabSwitches,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -212,6 +212,7 @@ export class SettleController {
|
||||
url: finalUrl,
|
||||
pageUrl: finalUrl,
|
||||
cause: "browser",
|
||||
transitionQualifiers: ["server_redirect"],
|
||||
preStateId: this.#session.cursor.lastSettled?.stateId,
|
||||
};
|
||||
drafts.push(draft);
|
||||
|
||||
@@ -2,7 +2,8 @@ export interface RecordedStateEntry {
|
||||
id: string;
|
||||
url: string;
|
||||
title?: string;
|
||||
rawVomText: string;
|
||||
/** Rendered VOM text; capture indexes and source DOM/AX data never enter the registry. */
|
||||
vomText: string;
|
||||
truncated: boolean;
|
||||
stepsHere: number[];
|
||||
}
|
||||
@@ -19,10 +20,10 @@ export class RecordingStateRegistry {
|
||||
register(input: {
|
||||
url: string;
|
||||
title?: string;
|
||||
rawVomText: string;
|
||||
vomText: string;
|
||||
truncated?: boolean;
|
||||
}): RecordedStateEntry {
|
||||
const identity = stateIdentity(input.url, input.rawVomText);
|
||||
const identity = stateIdentity(input.url, input.vomText);
|
||||
const existingId = this.#idByIdentity.get(identity);
|
||||
if (existingId) {
|
||||
const existing = this.#entriesById.get(existingId)!;
|
||||
@@ -35,7 +36,7 @@ export class RecordingStateRegistry {
|
||||
id: `s${this.#nextId}`,
|
||||
url: input.url,
|
||||
...(input.title ? { title: input.title } : {}),
|
||||
rawVomText: input.rawVomText,
|
||||
vomText: input.vomText,
|
||||
truncated: input.truncated ?? false,
|
||||
stepsHere: [],
|
||||
};
|
||||
|
||||
@@ -1,8 +1,13 @@
|
||||
import type { RecordStepPayload } from "../record-bridge";
|
||||
import type { RecordingDraftStep } from "./types";
|
||||
import { hasRedirectQualifier } from "./navigation-policy";
|
||||
import type { RecordingDraftStep, TargetMatchHint } from "./types";
|
||||
|
||||
export interface RecordingStepBuffer {
|
||||
steps: RecordingDraftStep[];
|
||||
navigation: RecordingNavigationCursor;
|
||||
}
|
||||
|
||||
export interface RecordingNavigationCursor {
|
||||
currentUrl?: string;
|
||||
pendingNavigation: boolean;
|
||||
pendingNavigationDeadline?: number;
|
||||
@@ -10,33 +15,29 @@ export interface RecordingStepBuffer {
|
||||
|
||||
const NAVIGATION_TRIGGER_WINDOW_MS = 3_000;
|
||||
|
||||
function toDraftStep(payload: RecordStepPayload): RecordingDraftStep | null {
|
||||
function toDraftStep(
|
||||
payload: RecordStepPayload,
|
||||
targetHint?: TargetMatchHint,
|
||||
): RecordingDraftStep | null {
|
||||
const pageUrl = payload.page_url;
|
||||
const common = {
|
||||
...(pageUrl ? { pageUrl } : {}),
|
||||
...(targetHint ? { targetHint } : {}),
|
||||
};
|
||||
switch (payload.op) {
|
||||
case "click":
|
||||
return payload.target
|
||||
? {
|
||||
op: "click",
|
||||
captureTarget: payload.target,
|
||||
...(pageUrl ? { pageUrl } : {}),
|
||||
}
|
||||
: null;
|
||||
return payload.target ? { op: "click", captureTarget: payload.target, ...common } : null;
|
||||
case "hover":
|
||||
return payload.target
|
||||
? {
|
||||
op: "hover",
|
||||
captureTarget: payload.target,
|
||||
...(pageUrl ? { pageUrl } : {}),
|
||||
}
|
||||
: null;
|
||||
return payload.target ? { op: "hover", captureTarget: payload.target, ...common } : null;
|
||||
case "fill":
|
||||
return payload.target
|
||||
? {
|
||||
op: "fill",
|
||||
captureTarget: payload.target,
|
||||
value: payload.value ?? "",
|
||||
...(payload.commit ? { commit: payload.commit } : {}),
|
||||
...(payload.redacted ? { redacted: true } : {}),
|
||||
...(pageUrl ? { pageUrl } : {}),
|
||||
...common,
|
||||
}
|
||||
: null;
|
||||
case "press":
|
||||
@@ -46,7 +47,7 @@ function toDraftStep(payload: RecordStepPayload): RecordingDraftStep | null {
|
||||
key: payload.key,
|
||||
...(payload.target ? { captureTarget: payload.target } : {}),
|
||||
...(payload.modifiers?.length ? { modifiers: payload.modifiers } : {}),
|
||||
...(pageUrl ? { pageUrl } : {}),
|
||||
...common,
|
||||
}
|
||||
: null;
|
||||
case "select":
|
||||
@@ -56,7 +57,7 @@ function toDraftStep(payload: RecordStepPayload): RecordingDraftStep | null {
|
||||
captureTarget: payload.target,
|
||||
values: payload.values,
|
||||
...(payload.labels?.length ? { labels: payload.labels } : {}),
|
||||
...(pageUrl ? { pageUrl } : {}),
|
||||
...common,
|
||||
}
|
||||
: null;
|
||||
case "navigate":
|
||||
@@ -64,73 +65,91 @@ function toDraftStep(payload: RecordStepPayload): RecordingDraftStep | null {
|
||||
}
|
||||
}
|
||||
|
||||
function annotateLastStepNavigation(buffer: RecordingStepBuffer, url: string): boolean {
|
||||
for (let i = buffer.steps.length - 1; i >= 0; i -= 1) {
|
||||
const step = buffer.steps[i];
|
||||
function annotateLastStepNavigation(buffer: RecordingStepBuffer, url: string): number {
|
||||
for (let index = buffer.steps.length - 1; index >= 0; index -= 1) {
|
||||
const step = buffer.steps[index];
|
||||
if (!step) continue;
|
||||
if (step.op === "click" || step.op === "press" || step.op === "select") {
|
||||
buffer.steps[i] = { ...step, navigatedTo: url };
|
||||
return true;
|
||||
if (step.op === "click" || step.op === "press" || step.op === "select" || step.op === "fill") {
|
||||
step.navigatedTo = url;
|
||||
return index;
|
||||
}
|
||||
break;
|
||||
}
|
||||
return false;
|
||||
return -1;
|
||||
}
|
||||
|
||||
export type NavigationObserveResult =
|
||||
| { kind: "noop" }
|
||||
| { kind: "annotated"; index: number }
|
||||
| { kind: "appended"; index: number }
|
||||
| { kind: "coalesce_redirect"; url: string };
|
||||
|
||||
export function observeRecordedNavigation(
|
||||
buffer: RecordingStepBuffer,
|
||||
url: string,
|
||||
causedByAction?: boolean,
|
||||
): void {
|
||||
if (!url || url === buffer.currentUrl) return;
|
||||
buffer.currentUrl = url;
|
||||
transitionType?: string,
|
||||
transitionQualifiers?: string[],
|
||||
): NavigationObserveResult {
|
||||
const navigation = buffer.navigation;
|
||||
if (!url || url === navigation.currentUrl) return { kind: "noop" };
|
||||
navigation.currentUrl = url;
|
||||
|
||||
const pendingIsCurrent =
|
||||
buffer.pendingNavigation &&
|
||||
(buffer.pendingNavigationDeadline === undefined ||
|
||||
buffer.pendingNavigationDeadline >= Date.now());
|
||||
navigation.pendingNavigation &&
|
||||
(navigation.pendingNavigationDeadline === undefined ||
|
||||
navigation.pendingNavigationDeadline >= Date.now());
|
||||
|
||||
if (causedByAction === true || (causedByAction === undefined && pendingIsCurrent)) {
|
||||
buffer.pendingNavigation = false;
|
||||
buffer.pendingNavigationDeadline = undefined;
|
||||
if (!annotateLastStepNavigation(buffer, url)) {
|
||||
buffer.steps.push({
|
||||
op: "navigate",
|
||||
url,
|
||||
pageUrl: url,
|
||||
});
|
||||
}
|
||||
return;
|
||||
navigation.pendingNavigation = false;
|
||||
navigation.pendingNavigationDeadline = undefined;
|
||||
const annotatedIndex = annotateLastStepNavigation(buffer, url);
|
||||
if (annotatedIndex >= 0) return { kind: "annotated", index: annotatedIndex };
|
||||
} else {
|
||||
navigation.pendingNavigation = false;
|
||||
navigation.pendingNavigationDeadline = undefined;
|
||||
}
|
||||
|
||||
buffer.pendingNavigation = false;
|
||||
buffer.pendingNavigationDeadline = undefined;
|
||||
if (hasRedirectQualifier(transitionQualifiers)) {
|
||||
return { kind: "coalesce_redirect", url };
|
||||
}
|
||||
buffer.steps.push({
|
||||
op: "navigate",
|
||||
url,
|
||||
pageUrl: url,
|
||||
transitionType,
|
||||
transitionQualifiers,
|
||||
});
|
||||
return { kind: "appended", index: buffer.steps.length - 1 };
|
||||
}
|
||||
|
||||
export function appendRecordedPayload(
|
||||
buffer: RecordingStepBuffer,
|
||||
payload: RecordStepPayload,
|
||||
): void {
|
||||
targetHint?: TargetMatchHint,
|
||||
): number | null {
|
||||
if (payload.op === "navigate") {
|
||||
if (payload.url) {
|
||||
observeRecordedNavigation(buffer, payload.url, payload.navigation_caused_by_action);
|
||||
}
|
||||
return;
|
||||
if (!payload.url) return null;
|
||||
const result = observeRecordedNavigation(
|
||||
buffer,
|
||||
payload.url,
|
||||
payload.navigation_caused_by_action,
|
||||
payload.transitionType,
|
||||
payload.transitionQualifiers,
|
||||
);
|
||||
return result.kind === "appended" ? result.index : null;
|
||||
}
|
||||
const step = toDraftStep({
|
||||
...payload,
|
||||
page_url: payload.page_url ?? buffer.currentUrl,
|
||||
});
|
||||
if (!step) return;
|
||||
const step = toDraftStep(
|
||||
{ ...payload, page_url: payload.page_url ?? buffer.navigation.currentUrl },
|
||||
targetHint,
|
||||
);
|
||||
if (!step) return null;
|
||||
buffer.steps.push(step);
|
||||
if (step.op === "click" || step.op === "press" || step.op === "select" || step.op === "fill") {
|
||||
buffer.pendingNavigation = payload.expects_navigation === true;
|
||||
buffer.pendingNavigationDeadline = buffer.pendingNavigation
|
||||
buffer.navigation.pendingNavigation = payload.expects_navigation === true;
|
||||
buffer.navigation.pendingNavigationDeadline = buffer.navigation.pendingNavigation
|
||||
? Date.now() + NAVIGATION_TRIGGER_WINDOW_MS
|
||||
: undefined;
|
||||
}
|
||||
return buffer.steps.length - 1;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
import type { RecordingNavigationCursor } from "./step-buffer";
|
||||
|
||||
export interface TabActivation {
|
||||
tabId: number;
|
||||
revision: number;
|
||||
}
|
||||
|
||||
export class RecordingTabCoordinator {
|
||||
readonly #navigationByTab = new Map<number, RecordingNavigationCursor>();
|
||||
#activeTabId: number;
|
||||
#currentTabId: number;
|
||||
#activationRevision = 0;
|
||||
|
||||
constructor(initialTabId: number, initialUrl?: string) {
|
||||
this.#activeTabId = initialTabId;
|
||||
this.#currentTabId = initialTabId;
|
||||
this.#navigationByTab.set(initialTabId, {
|
||||
currentUrl: initialUrl,
|
||||
pendingNavigation: false,
|
||||
});
|
||||
}
|
||||
|
||||
get activeTabId(): number {
|
||||
return this.#activeTabId;
|
||||
}
|
||||
|
||||
get currentTabId(): number {
|
||||
return this.#currentTabId;
|
||||
}
|
||||
|
||||
noteActivation(tabId: number): TabActivation {
|
||||
this.#activeTabId = tabId;
|
||||
this.#activationRevision += 1;
|
||||
return { tabId, revision: this.#activationRevision };
|
||||
}
|
||||
|
||||
isLatest(activation: TabActivation): boolean {
|
||||
return (
|
||||
activation.tabId === this.#activeTabId && activation.revision === this.#activationRevision
|
||||
);
|
||||
}
|
||||
|
||||
commit(tabId: number, currentUrl?: string): void {
|
||||
if (currentUrl !== undefined) this.navigation(tabId).currentUrl = currentUrl;
|
||||
this.#currentTabId = tabId;
|
||||
}
|
||||
|
||||
navigation(tabId: number, fallbackUrl?: string): RecordingNavigationCursor {
|
||||
const existing = this.#navigationByTab.get(tabId);
|
||||
if (existing) {
|
||||
if (!existing.currentUrl && fallbackUrl) existing.currentUrl = fallbackUrl;
|
||||
return existing;
|
||||
}
|
||||
const created: RecordingNavigationCursor = {
|
||||
currentUrl: fallbackUrl,
|
||||
pendingNavigation: false,
|
||||
};
|
||||
this.#navigationByTab.set(tabId, created);
|
||||
return created;
|
||||
}
|
||||
}
|
||||
@@ -14,8 +14,13 @@ function rectMatches(a: TargetGeometry["rect"], b: TargetGeometry["rect"]): bool
|
||||
return close(a.x, b.x) && close(a.y, b.y) && close(a.w, b.w) && close(a.h, b.h);
|
||||
}
|
||||
|
||||
function candidateMatches(candidate: IndexedObservationNode, geometry: TargetGeometry): boolean {
|
||||
return candidate.geometry.rect !== null && rectMatches(geometry.rect, candidate.geometry.rect);
|
||||
function candidateMatches(
|
||||
candidate: IndexedObservationNode,
|
||||
geometry: TargetGeometry,
|
||||
geometrySpace: "top" | "local",
|
||||
): boolean {
|
||||
const rect = geometrySpace === "local" ? candidate.geometry.localRect : candidate.geometry.rect;
|
||||
return rect != null && rectMatches(geometry.rect, rect);
|
||||
}
|
||||
|
||||
export function unmatchedTarget(fallback?: CaptureTargetDescriptor): TargetDescriptorV3 {
|
||||
@@ -53,12 +58,17 @@ export function matchObservationTarget(input: {
|
||||
hint?: TargetMatchHint;
|
||||
fallback?: CaptureTargetDescriptor;
|
||||
}): TargetDescriptorV3 {
|
||||
if (input.hint?.frameId === null) return unmatchedTarget(input.fallback);
|
||||
const frameId = input.hint?.frameId ?? input.observation.rootFrameId;
|
||||
const geometry = input.hint?.geometry;
|
||||
if (geometry) {
|
||||
const matches = input.observation.index
|
||||
.candidates(frameId, geometry.tag)
|
||||
.filter((candidate) => candidate.ref && candidateMatches(candidate, geometry));
|
||||
.filter(
|
||||
(candidate) =>
|
||||
candidate.ref &&
|
||||
candidateMatches(candidate, geometry, input.hint?.geometrySpace ?? "top"),
|
||||
);
|
||||
if (matches.length === 1) return descriptor(matches[0]!.ref!);
|
||||
const semanticMatches = matches.filter(
|
||||
(candidate) => candidate.ref && matchesSemantics(candidate.ref, input.fallback),
|
||||
|
||||
@@ -39,8 +39,13 @@ export function buildTraceV3(input: {
|
||||
startUrl?: string;
|
||||
stoppedBy: StopReason;
|
||||
bskVersion: string;
|
||||
redactValues?: boolean;
|
||||
includeTabSwitches?: boolean;
|
||||
}): TraceV3 {
|
||||
const reduced = reduceTraceStepsV3(input.drafts);
|
||||
const reduced = reduceTraceStepsV3(input.drafts, {
|
||||
includeTabSwitches: input.includeTabSwitches,
|
||||
redactValues: input.redactValues,
|
||||
});
|
||||
const entries = publishedEntries(input.registry, reduced.steps);
|
||||
const publishedId = new Map(entries.map((entry, index) => [entry.id, `s${index + 1}`]));
|
||||
const annotationsByState = new Map<string, StepAnnotation[]>();
|
||||
@@ -65,7 +70,7 @@ export function buildTraceV3(input: {
|
||||
url: entry.url,
|
||||
title: entry.title,
|
||||
stepIds: remapDraftIds(entry.stepsHere, reduced.stepIdByDraftId),
|
||||
vomText: entry.rawVomText,
|
||||
vomText: entry.vomText,
|
||||
annotations: annotationsByState.get(entry.id) ?? [],
|
||||
stepIdByDraftId: reduced.stepIdByDraftId,
|
||||
}),
|
||||
|
||||
@@ -119,14 +119,6 @@ function toV2Step(
|
||||
},
|
||||
effectForNavigation(step.navigatedTo, urlToId),
|
||||
);
|
||||
case "hover":
|
||||
if (!step.captureTarget) return null;
|
||||
return {
|
||||
op: "hover",
|
||||
id,
|
||||
page,
|
||||
target: step.captureTarget,
|
||||
};
|
||||
case "fill":
|
||||
if (!step.captureTarget) return null;
|
||||
return {
|
||||
@@ -161,7 +153,11 @@ function toV2Step(
|
||||
},
|
||||
effectForNavigation(step.navigatedTo, urlToId),
|
||||
);
|
||||
// `hover` only exists in Trace v3. Peers that negotiate v2 predate the
|
||||
// step variant and fail to decode the whole result if we emit it.
|
||||
case "hover":
|
||||
case "scroll":
|
||||
case "switch_tab":
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import type { NavigationCause, StepV3 } from "@/transport/types";
|
||||
import { shouldIncludeDraft } from "./draft-policy";
|
||||
import { hasRedirectQualifier } from "./navigation-policy";
|
||||
import { unmatchedTarget } from "./target-matcher";
|
||||
import type { RecordingDraftStep } from "./types";
|
||||
|
||||
@@ -8,7 +9,6 @@ interface CollapsedDraft {
|
||||
draftIds: number[];
|
||||
}
|
||||
|
||||
const REDIRECT_QUALIFIERS = new Set(["client_redirect", "server_redirect"]);
|
||||
const TRANSITION_CAUSES: Record<string, NavigationCause> = {
|
||||
typed: "user_typed",
|
||||
generated: "user_typed",
|
||||
@@ -22,7 +22,7 @@ const TRANSITION_CAUSES: Record<string, NavigationCause> = {
|
||||
};
|
||||
|
||||
function isRedirect(step: Extract<RecordingDraftStep, { op: "navigate" }>): boolean {
|
||||
return (step.transitionQualifiers ?? []).some((qualifier) => REDIRECT_QUALIFIERS.has(qualifier));
|
||||
return hasRedirectQualifier(step.transitionQualifiers);
|
||||
}
|
||||
|
||||
function collapseRedirects(steps: RecordingDraftStep[]): CollapsedDraft[] {
|
||||
@@ -58,8 +58,26 @@ function selection(values: string[], labels?: string[]): Array<{ value: string;
|
||||
}));
|
||||
}
|
||||
|
||||
function reduceDraft(draft: RecordingDraftStep, id: number): StepV3 | null {
|
||||
interface ReduceDraftOptions {
|
||||
includeTabSwitches: boolean;
|
||||
redactValues: boolean;
|
||||
}
|
||||
|
||||
function reduceDraft(
|
||||
draft: RecordingDraftStep,
|
||||
id: number,
|
||||
options: ReduceDraftOptions,
|
||||
): StepV3 | null {
|
||||
if (!shouldIncludeDraft(draft)) return null;
|
||||
if (draft.op === "switch_tab") {
|
||||
if (!options.includeTabSwitches || !draft.preStateId || !draft.postStateId) return null;
|
||||
return {
|
||||
op: "switch_tab",
|
||||
id,
|
||||
state: draft.preStateId,
|
||||
result: { state: draft.postStateId },
|
||||
};
|
||||
}
|
||||
const state = draft.preStateId ?? draft.postStateId;
|
||||
const resultState = draft.postStateId ?? draft.preStateId;
|
||||
if (!state || !resultState) return null;
|
||||
@@ -81,13 +99,14 @@ function reduceDraft(draft: RecordingDraftStep, id: number): StepV3 | null {
|
||||
target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget),
|
||||
};
|
||||
case "fill":
|
||||
const fillIsRedacted = options.redactValues || draft.redacted === true;
|
||||
return {
|
||||
op: "fill",
|
||||
...common,
|
||||
target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget),
|
||||
value: draft.value,
|
||||
value: fillIsRedacted ? "***" : draft.value,
|
||||
commit: draft.commit ?? "blur",
|
||||
...(draft.redacted ? { redacted: true } : {}),
|
||||
...(fillIsRedacted ? { redacted: true } : {}),
|
||||
};
|
||||
case "press":
|
||||
return {
|
||||
@@ -104,7 +123,7 @@ function reduceDraft(draft: RecordingDraftStep, id: number): StepV3 | null {
|
||||
op: "select",
|
||||
...common,
|
||||
target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget),
|
||||
selection: selection(draft.values, draft.labels),
|
||||
...(!options.redactValues ? { selection: selection(draft.values, draft.labels) } : {}),
|
||||
};
|
||||
case "scroll":
|
||||
return { op: "scroll", ...common };
|
||||
@@ -116,11 +135,17 @@ export interface ReducedTraceV3 {
|
||||
stepIdByDraftId: Map<number, number>;
|
||||
}
|
||||
|
||||
export function reduceTraceStepsV3(steps: RecordingDraftStep[]): ReducedTraceV3 {
|
||||
export function reduceTraceStepsV3(
|
||||
steps: RecordingDraftStep[],
|
||||
options: { includeTabSwitches?: boolean; redactValues?: boolean } = {},
|
||||
): ReducedTraceV3 {
|
||||
const output: StepV3[] = [];
|
||||
const stepIdByDraftId = new Map<number, number>();
|
||||
for (const { draft, draftIds } of collapseRedirects(steps)) {
|
||||
const step = reduceDraft(draft, output.length + 1);
|
||||
const step = reduceDraft(draft, output.length + 1, {
|
||||
includeTabSwitches: options.includeTabSwitches === true,
|
||||
redactValues: options.redactValues === true,
|
||||
});
|
||||
if (!step) continue;
|
||||
output.push(step);
|
||||
for (const draftId of draftIds) stepIdByDraftId.set(draftId, step.id);
|
||||
|
||||
@@ -15,8 +15,9 @@ export interface TargetGeometry {
|
||||
|
||||
export interface TargetMatchHint {
|
||||
geometry?: TargetGeometry;
|
||||
/** Missing means the current top frame, never an unrestricted frame search. */
|
||||
frameId?: string;
|
||||
/** Missing means top frame; null means the source Document could not be resolved. */
|
||||
frameId?: string | null;
|
||||
geometrySpace?: "top" | "local";
|
||||
}
|
||||
|
||||
export interface StepAnnotation {
|
||||
@@ -69,6 +70,7 @@ export type RecordingDraftStep =
|
||||
DraftTarget &
|
||||
DraftNavigationEffect)
|
||||
| ({ op: "scroll" } & DraftStateLink)
|
||||
| ({ op: "switch_tab" } & DraftStateLink)
|
||||
| ({
|
||||
op: "navigate";
|
||||
url: string;
|
||||
|
||||
@@ -9,7 +9,7 @@ describe("disconnect session cleanup", () => {
|
||||
const manager = new SessionManager({
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => nextWindowId++),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
remove,
|
||||
},
|
||||
});
|
||||
@@ -41,7 +41,7 @@ describe("disconnect session cleanup", () => {
|
||||
const manager = new SessionManager({
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 100),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
remove: vi.fn(() => removeGate),
|
||||
},
|
||||
});
|
||||
|
||||
@@ -39,7 +39,7 @@ describe("attachSessionEventHandler", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 4242),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
await manager.start("aa11");
|
||||
@@ -73,7 +73,7 @@ describe("attachSessionEventHandler", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 4242),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
const ctx = await manager.start("aa11");
|
||||
@@ -87,6 +87,7 @@ describe("attachSessionEventHandler", () => {
|
||||
windowEvents: events.api,
|
||||
});
|
||||
|
||||
manager.forgetClosedTab(7, { isWindowClosing: true });
|
||||
events.emit(4242);
|
||||
for (let i = 0; i < 4; i += 1) await Promise.resolve();
|
||||
|
||||
@@ -113,7 +114,7 @@ describe("attachSessionEventHandler", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 1),
|
||||
remove: vi.fn(),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
const transport = fakeTransport();
|
||||
@@ -129,7 +130,7 @@ describe("attachSessionEventHandler", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 1),
|
||||
remove: vi.fn(),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
const transport = fakeTransport();
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import type { AgentWindowApi, AgentWindowCreateOptions } from "../agent-window";
|
||||
import { SessionManager } from "../manager";
|
||||
import { isAgentControlledTab, SessionManager } from "../manager";
|
||||
|
||||
function fakeAgentWindow(): AgentWindowApi & {
|
||||
createMock: ReturnType<typeof vi.fn>;
|
||||
@@ -13,7 +13,7 @@ function fakeAgentWindow(): AgentWindowApi & {
|
||||
return id;
|
||||
});
|
||||
const removeMock = vi.fn(async (_id: number) => {});
|
||||
const ensureActiveTabMock = vi.fn(async (_windowId: number, _url: string) => {});
|
||||
const ensureActiveTabMock = vi.fn(async (_windowId: number, _url: string) => 0);
|
||||
return {
|
||||
create: createMock,
|
||||
remove: removeMock,
|
||||
@@ -151,6 +151,76 @@ describe("SessionManager", () => {
|
||||
expect(sm.list()).toEqual([]);
|
||||
});
|
||||
|
||||
describe("explicit tab claims", () => {
|
||||
it("claims the exact home tab returned by AgentWindowApi", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow() });
|
||||
const ctx = await sm.start("aa11");
|
||||
expect(ctx.agentCreatedTabs).toEqual(new Set([0]));
|
||||
expect(isAgentControlledTab(ctx, 0)).toBe(true);
|
||||
});
|
||||
|
||||
it("keeps every other tab free without classification state", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow() });
|
||||
const ctx = await sm.start("aa11");
|
||||
expect(isAgentControlledTab(ctx, 99)).toBe(false);
|
||||
expect(ctx).not.toHaveProperty("userTabs");
|
||||
expect(ctx).not.toHaveProperty("pendingAgentTabCount");
|
||||
});
|
||||
|
||||
it("treats a committed borrow as an explicit claim", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow() });
|
||||
const ctx = await sm.start("aa11");
|
||||
const reservation = sm.tryReserveBorrow(42, ctx.sessionId);
|
||||
if ("borrowedBy" in reservation) throw new Error("unexpected borrow conflict");
|
||||
reservation.commit({ tabId: 42, originalWindowId: 7, originalIndex: 3 });
|
||||
expect(isAgentControlledTab(ctx, 42)).toBe(true);
|
||||
});
|
||||
|
||||
it("forgets an agent-created tab after Chrome removes it", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow() });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.agentCreatedTabs.add(42);
|
||||
sm.forgetClosedTab(42);
|
||||
expect(ctx.agentCreatedTabs.has(42)).toBe(false);
|
||||
expect(isAgentControlledTab(ctx, 0)).toBe(true);
|
||||
});
|
||||
|
||||
it("forgets closed borrows idempotently without affecting other tabs or sessions", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow() });
|
||||
const a = await sm.start("aa11");
|
||||
const b = await sm.start("bb22");
|
||||
a.borrowedTabs.set(42, { tabId: 42, originalWindowId: 7, originalIndex: 0 });
|
||||
a.borrowedTabs.set(43, { tabId: 43, originalWindowId: 7, originalIndex: 1 });
|
||||
b.borrowedTabs.set(44, { tabId: 44, originalWindowId: 8, originalIndex: 0 });
|
||||
|
||||
sm.forgetClosedTab(42);
|
||||
sm.forgetClosedTab(42);
|
||||
sm.forgetClosedTab(999);
|
||||
|
||||
expect(isAgentControlledTab(a, 42)).toBe(false);
|
||||
expect(sm.findBorrowingSession(42, null)).toBeNull();
|
||||
expect([...a.borrowedTabs.keys()]).toEqual([43]);
|
||||
expect([...b.borrowedTabs.keys()]).toEqual([44]);
|
||||
expect(sm.list()).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("prevents an in-flight borrow from reclaiming a closed tab", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow() });
|
||||
const ctx = await sm.start("aa11");
|
||||
const reservation = sm.tryReserveBorrow(42, ctx.sessionId);
|
||||
if ("borrowedBy" in reservation) throw new Error("unexpected borrow conflict");
|
||||
|
||||
sm.forgetClosedTab(42);
|
||||
|
||||
expect(() =>
|
||||
reservation.commit({ tabId: 42, originalWindowId: 7, originalIndex: 0 }),
|
||||
).toThrow(/reservation disappeared/);
|
||||
reservation.release();
|
||||
expect(sm.findBorrowingSession(42, null)).toBeNull();
|
||||
expect(ctx.borrowedTabs.has(42)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("findBorrowingSession", () => {
|
||||
it("returns null when no session has borrowed the tab", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow() });
|
||||
|
||||
@@ -14,8 +14,11 @@ export interface AgentWindowApi {
|
||||
* Guarantee the Agent Window has an active, CDP-navigable tab.
|
||||
* `chrome://` pages (including the New Tab page) reject `Page.navigate`,
|
||||
* so sessions bootstrap with `about:blank` instead.
|
||||
*
|
||||
* Resolves with the id of the activated (or newly created) tab, so callers
|
||||
* can track the session's home tab without re-querying Chrome.
|
||||
*/
|
||||
ensureActiveTab(windowId: number, url: string): Promise<void>;
|
||||
ensureActiveTab(windowId: number, url: string): Promise<number>;
|
||||
}
|
||||
|
||||
/** Creation hints for a new Agent Window. */
|
||||
@@ -49,15 +52,19 @@ export const chromeAgentWindowApi: AgentWindowApi = {
|
||||
// cancellation success while the Agent Window remains open.
|
||||
await chrome.windows.remove(windowId);
|
||||
},
|
||||
async ensureActiveTab(windowId: number, url: string): Promise<void> {
|
||||
async ensureActiveTab(windowId: number, url: string): Promise<number> {
|
||||
const tabs = await chrome.tabs.query({ windowId });
|
||||
const first = tabs.find((t) => typeof t.id === "number");
|
||||
if (first?.id) {
|
||||
if (first?.id !== undefined) {
|
||||
if (!first.active) {
|
||||
await chrome.tabs.update(first.id, { active: true });
|
||||
}
|
||||
return;
|
||||
return first.id;
|
||||
}
|
||||
await chrome.tabs.create({ windowId, url, active: true });
|
||||
const created = await chrome.tabs.create({ windowId, url, active: true });
|
||||
if (typeof created?.id !== "number") {
|
||||
throw new Error("[bh] chrome.tabs.create returned no tab id");
|
||||
}
|
||||
return created.id;
|
||||
},
|
||||
};
|
||||
|
||||
@@ -6,9 +6,20 @@ export interface SessionContext {
|
||||
agentWindowId: number;
|
||||
refStore: RefStore;
|
||||
borrowedTabs: Map<number, BorrowedTab>;
|
||||
/**
|
||||
* Tabs explicitly claimed by the agent because it created them. This
|
||||
* includes the Agent Window's home tab and tabs created by `tool.tab_create`.
|
||||
* Tabs opened through Chrome UI never enter this set.
|
||||
*/
|
||||
agentCreatedTabs: Set<number>;
|
||||
createdAtMs: number;
|
||||
}
|
||||
|
||||
/** Whether this session has explicitly claimed control of `tabId`. */
|
||||
export function isAgentControlledTab(ctx: SessionContext, tabId: number): boolean {
|
||||
return ctx.agentCreatedTabs.has(tabId) || ctx.borrowedTabs.has(tabId);
|
||||
}
|
||||
|
||||
export interface BorrowedTab {
|
||||
tabId: number;
|
||||
originalWindowId: number;
|
||||
@@ -106,6 +117,19 @@ export class SessionManager {
|
||||
return Array.from(this.sessions.values());
|
||||
}
|
||||
|
||||
/**
|
||||
* Forget a tab Chrome has removed, including any uncommitted borrow.
|
||||
* Whole-window closures keep committed borrows until the window-removed
|
||||
* handler reports which user tabs could not be returned.
|
||||
*/
|
||||
forgetClosedTab(tabId: number, { isWindowClosing = false } = {}): void {
|
||||
this.borrowReservations.delete(tabId);
|
||||
for (const ctx of this.sessions.values()) {
|
||||
ctx.agentCreatedTabs.delete(tabId);
|
||||
if (!isWindowClosing) ctx.borrowedTabs.delete(tabId);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Look up whether `tabId` is currently borrowed by some *other*
|
||||
* session than the one calling. Used by M8 `tab_borrow` to refuse
|
||||
@@ -179,7 +203,7 @@ export class SessionManager {
|
||||
const { signal: _signal, ...createOptions } = opts;
|
||||
windowId = await this.agentWindow.create(AGENT_WINDOW_HOME, createOptions);
|
||||
throwIfSessionStartAborted(opts.signal);
|
||||
await this.agentWindow.ensureActiveTab(windowId, AGENT_WINDOW_HOME);
|
||||
const homeTabId = await this.agentWindow.ensureActiveTab(windowId, AGENT_WINDOW_HOME);
|
||||
throwIfSessionStartAborted(opts.signal);
|
||||
|
||||
const ctx: SessionContext = {
|
||||
@@ -187,6 +211,10 @@ export class SessionManager {
|
||||
agentWindowId: windowId,
|
||||
refStore: new RefStore(),
|
||||
borrowedTabs: new Map(),
|
||||
// The home tab is the session's first explicit claim. Every other
|
||||
// tab remains free until `tab_create` or `tab_borrow` identifies it
|
||||
// by its concrete Chrome tab id.
|
||||
agentCreatedTabs: new Set([homeTabId]),
|
||||
createdAtMs: this.now(),
|
||||
};
|
||||
this.sessions.set(sessionId, ctx);
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
export const RECORD_DOCUMENT_ATTRIBUTE = "data-bsk-record-document";
|
||||
|
||||
const RECORDING_DOCUMENT_ID_PATTERN =
|
||||
/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
|
||||
const RECORDING_DOCUMENT_MARKER_PREFIX = "bsk:";
|
||||
|
||||
export function recordingDocumentMarkerValue(identity: string): string {
|
||||
if (!RECORDING_DOCUMENT_ID_PATTERN.test(identity)) {
|
||||
throw new TypeError("recording Document identity must be a random UUID");
|
||||
}
|
||||
return `${RECORDING_DOCUMENT_MARKER_PREFIX}${identity}`;
|
||||
}
|
||||
|
||||
/** Read BrowserSkill's opaque per-Document recording identity from captured attributes. */
|
||||
export function readRecordingDocumentIdentity(
|
||||
attrs: Readonly<Record<string, string>>,
|
||||
): string | undefined {
|
||||
const marker = attrs[RECORD_DOCUMENT_ATTRIBUTE];
|
||||
if (!marker?.startsWith(RECORDING_DOCUMENT_MARKER_PREFIX)) return undefined;
|
||||
const identity = marker.slice(RECORDING_DOCUMENT_MARKER_PREFIX.length);
|
||||
return identity && RECORDING_DOCUMENT_ID_PATTERN.test(identity) ? identity : undefined;
|
||||
}
|
||||
@@ -127,6 +127,38 @@ describe("requestBorrowConfirmation", () => {
|
||||
expect(message.requestId.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("proactively focuses the user window before sending the overlay message", async () => {
|
||||
tabs.get.mockResolvedValueOnce({ id: 42, title: "Tab" });
|
||||
windows.getLastFocused.mockResolvedValueOnce(
|
||||
userWindowWithActiveTab({ windowId: 77, tabId: 7, url: "https://app.example/" }),
|
||||
);
|
||||
tabs.sendMessage.mockResolvedValueOnce({ type: "borrow-response", allowed: true });
|
||||
|
||||
const pending = requestBorrowConfirmation(42, {
|
||||
deps: { tabs, windows, notifications },
|
||||
});
|
||||
await vi.runAllTimersAsync();
|
||||
await pending;
|
||||
|
||||
expect(windows.update).toHaveBeenCalledWith(77, { focused: true });
|
||||
});
|
||||
|
||||
it("still proceeds when proactive window focus fails", async () => {
|
||||
tabs.get.mockResolvedValueOnce({ id: 42, title: "Tab" });
|
||||
windows.getLastFocused.mockResolvedValueOnce(
|
||||
userWindowWithActiveTab({ windowId: 77, tabId: 7, url: "https://app.example/" }),
|
||||
);
|
||||
windows.update.mockRejectedValueOnce(new Error("window gone"));
|
||||
tabs.sendMessage.mockResolvedValueOnce({ type: "borrow-response", allowed: true });
|
||||
|
||||
const pending = requestBorrowConfirmation(42, {
|
||||
deps: { tabs, windows, notifications },
|
||||
});
|
||||
await vi.runAllTimersAsync();
|
||||
|
||||
expect(await pending).toBe(true);
|
||||
});
|
||||
|
||||
it("sends confirmation to the borrow target when it is the active tab", async () => {
|
||||
tabs.get.mockResolvedValueOnce({ id: 42, title: "Active Tab" });
|
||||
windows.getLastFocused.mockResolvedValueOnce(
|
||||
|
||||
@@ -13,7 +13,7 @@ function fakeAgentWindow(ids: number[]) {
|
||||
return id;
|
||||
}),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -8,6 +8,7 @@ import type {
|
||||
RequestFrame,
|
||||
} from "@/transport/types";
|
||||
import { ToolDispatcher } from "../dispatcher";
|
||||
import { resetBrowserObservationForTests, setBrowserObservationAttachForTests } from "../record";
|
||||
|
||||
type TestDispatcherCdp = NonNullable<ConstructorParameters<typeof ToolDispatcher>[0]["cdp"]>;
|
||||
|
||||
@@ -44,16 +45,120 @@ function makeRequest(method: string, params: unknown): RequestFrame {
|
||||
|
||||
describe("ToolDispatcher", () => {
|
||||
afterEach(() => {
|
||||
resetBrowserObservationForTests();
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
it.each([
|
||||
"tool.snapshot",
|
||||
"tool.evaluate",
|
||||
"tool.get_html",
|
||||
"tool.console",
|
||||
])("annotates Chrome extension-access failures from %s", async (method) => {
|
||||
const tab = { id: 7, windowId: 4242, active: true, url: "https://example.test" };
|
||||
vi.stubGlobal("chrome", {
|
||||
tabs: {
|
||||
get: vi.fn(async () => tab),
|
||||
query: vi.fn(async () => [tab]),
|
||||
},
|
||||
});
|
||||
const { transport, sent, deliver } = fakeTransport();
|
||||
const sessions = new SessionManager({
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 4242),
|
||||
remove: vi.fn(),
|
||||
ensureActiveTab: vi.fn(),
|
||||
},
|
||||
});
|
||||
await sessions.start("test");
|
||||
const error = new Error("Cannot access a chrome-extension:// URL of different extension");
|
||||
const cdp = {
|
||||
send: vi.fn().mockRejectedValue(error),
|
||||
ensureConsoleCapture: vi.fn().mockRejectedValue(error),
|
||||
consoleEntriesSince: vi.fn(),
|
||||
} as unknown as TestDispatcherCdp;
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions, cdp });
|
||||
dispatcher.start();
|
||||
deliver(makeRequest(method, { session_id: "test", expression: "document.title" }));
|
||||
await vi.waitFor(() => expect(sent).toHaveLength(1));
|
||||
expect(sent[0]).toMatchObject({
|
||||
error: {
|
||||
code: "cdp_failed",
|
||||
data: { reason: "cdp_extension_access_denied" },
|
||||
},
|
||||
});
|
||||
dispatcher.stop();
|
||||
});
|
||||
|
||||
it("uses the configured recording runtime for start and stop", async () => {
|
||||
const { transport, sent, deliver } = fakeTransport();
|
||||
const sessions = new SessionManager({
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 4242),
|
||||
remove: vi.fn(),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
await sessions.start("aa11");
|
||||
const tab = {
|
||||
id: 7,
|
||||
windowId: 4242,
|
||||
active: true,
|
||||
status: "complete",
|
||||
url: "https://example.com/start",
|
||||
} as chrome.tabs.Tab;
|
||||
const frameCoordinator = {
|
||||
begin: vi.fn(),
|
||||
armTab: vi.fn(async () => true),
|
||||
sourceFor: vi.fn(() => null),
|
||||
stop: vi.fn(async () => true),
|
||||
cancel: vi.fn(),
|
||||
};
|
||||
const sendToTab = vi.fn(async () => ({ ok: true }));
|
||||
setBrowserObservationAttachForTests(
|
||||
() => () => {},
|
||||
() => () => {},
|
||||
);
|
||||
const dispatcher = new ToolDispatcher({
|
||||
transport,
|
||||
sessions,
|
||||
recording: {
|
||||
tabsApi: {
|
||||
get: vi.fn(async () => tab),
|
||||
query: vi.fn(async () => [tab]),
|
||||
},
|
||||
frameCoordinator,
|
||||
sendToTab,
|
||||
},
|
||||
});
|
||||
dispatcher.start();
|
||||
|
||||
deliver(
|
||||
makeRequest("tool.record_start", {
|
||||
session_id: "aa11",
|
||||
url: "https://example.com/start",
|
||||
}),
|
||||
);
|
||||
await vi.waitFor(() => expect(sent).toHaveLength(1));
|
||||
|
||||
expect(sent[0]).toMatchObject({ result: { tab_id: 7, recording: true } });
|
||||
expect(frameCoordinator.begin).toHaveBeenCalledOnce();
|
||||
expect(frameCoordinator.armTab).toHaveBeenCalledWith(expect.any(String), 7);
|
||||
|
||||
deliver({ id: "r-2", method: "tool.record_stop", params: { session_id: "aa11" } });
|
||||
await vi.waitFor(() => expect(sent).toHaveLength(2));
|
||||
|
||||
expect(frameCoordinator.stop).toHaveBeenCalledOnce();
|
||||
expect(sent[1]).toMatchObject({ id: "r-2", result: { trace: { steps: [] } } });
|
||||
});
|
||||
|
||||
it("routes tool.session_start to the SessionManager and replies with the window id", async () => {
|
||||
const { transport, sent, deliver } = fakeTransport();
|
||||
const sessions = new SessionManager({
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 4242),
|
||||
remove: vi.fn(),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions });
|
||||
@@ -76,7 +181,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create,
|
||||
remove: vi.fn(),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions });
|
||||
@@ -94,7 +199,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 4242),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
await sessions.start("aa11");
|
||||
@@ -108,6 +213,47 @@ describe("ToolDispatcher", () => {
|
||||
expect(sent[0]).toEqual({ id: "r-1", result: {} });
|
||||
});
|
||||
|
||||
it("wires the production tab APIs into tool.session_stop so a surviving user tab releases the window", async () => {
|
||||
// Regression guard for the deps that session_stop reads directly: when
|
||||
// `tabManagement.tabs` / `tabsQuery` are not injected by the dispatcher,
|
||||
// the agent-tab cleanup and the release decision silently no-op and the
|
||||
// Agent Window is closed even though a user tab is still open (issue #57).
|
||||
const removeTab = vi.fn(async () => {});
|
||||
const closeWindow = vi.fn(async () => {});
|
||||
// After cleanup the window still holds one tab (id 99) that is neither
|
||||
// the home tab nor an agent-created tab — i.e. a genuine user tab.
|
||||
vi.stubGlobal("chrome", {
|
||||
tabs: {
|
||||
remove: removeTab,
|
||||
query: vi.fn(async () => [{ id: 99, windowId: 4242, active: true }]),
|
||||
},
|
||||
});
|
||||
const { transport, sent, deliver } = fakeTransport();
|
||||
const sessions = new SessionManager({
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 4242),
|
||||
remove: closeWindow,
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
await sessions.start("aa11");
|
||||
// Register the exact tab id returned by the agent's creation path.
|
||||
sessions.get("aa11")?.agentCreatedTabs.add(7);
|
||||
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions });
|
||||
dispatcher.start();
|
||||
|
||||
deliver(makeRequest("tool.session_stop", { session_id: "aa11" }));
|
||||
await flushMicrotasks();
|
||||
|
||||
// The agent tab is closed through the real chrome.tabs surface...
|
||||
expect(removeTab).toHaveBeenCalledWith(7);
|
||||
// ...and the window is released rather than closed, so the user's tab survives.
|
||||
expect(closeWindow).not.toHaveBeenCalled();
|
||||
expect(sessions.has("aa11")).toBe(false);
|
||||
expect(sent[0]).toEqual({ id: "r-1", result: { window_released: true } });
|
||||
});
|
||||
|
||||
it("routes tool.console through the CDP console buffer", async () => {
|
||||
vi.stubGlobal("chrome", {
|
||||
tabs: {
|
||||
@@ -119,7 +265,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 4242),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
await sessions.start("aa11");
|
||||
@@ -168,13 +314,114 @@ describe("ToolDispatcher", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("bypasses and restores the control overlay for an upload trigger click", async () => {
|
||||
const sendMessage = vi.fn(async () => undefined);
|
||||
vi.stubGlobal("chrome", {
|
||||
tabs: {
|
||||
get: vi.fn(async () => ({ id: 7, windowId: 4242, active: true })),
|
||||
query: vi.fn(async () => [{ id: 7, windowId: 4242, active: true }]),
|
||||
sendMessage,
|
||||
},
|
||||
});
|
||||
const { transport, sent, deliver } = fakeTransport();
|
||||
const sessions = new SessionManager({
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 4242),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
const ctx = await sessions.start("aa11");
|
||||
ctx.refStore.set("e1", 123, { tabId: 7 });
|
||||
const send = vi.fn(async <T>(_tabId: number, method: string, params?: object) => {
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } } as T;
|
||||
}
|
||||
if (method === "DOM.getContentQuads") {
|
||||
return { quads: [[0, 0, 20, 0, 20, 20, 0, 20]] } as T;
|
||||
}
|
||||
if (method === "DOM.resolveNode") {
|
||||
return { object: { objectId: "trigger-object" } } as T;
|
||||
}
|
||||
if (method === "DOM.describeNode") return { node: { backendNodeId: 456 } } as T;
|
||||
if (method === "Runtime.callFunctionOn") {
|
||||
const declaration = (params as { functionDeclaration?: string }).functionDeclaration ?? "";
|
||||
if (declaration.includes("count: state.inputs.length")) {
|
||||
return { result: { value: { count: 1, multiple: false } } } as T;
|
||||
}
|
||||
if (declaration.includes("inputs[0]")) {
|
||||
return { result: { objectId: "input-object" } } as T;
|
||||
}
|
||||
return { result: { value: true } } as T;
|
||||
}
|
||||
if (method === "Runtime.evaluate") {
|
||||
const expression = (params as { expression?: string }).expression ?? "";
|
||||
if (expression.includes("overlayDetails")) {
|
||||
return { result: { value: { hitIndex: 0 } } } as T;
|
||||
}
|
||||
if (expression.includes("overlayHostPresent")) {
|
||||
return {
|
||||
result: {
|
||||
value: { overlayHostPresent: true, overlayHostConnected: true },
|
||||
},
|
||||
} as T;
|
||||
}
|
||||
if (expression.includes("count:")) {
|
||||
return { result: { value: { count: 1, multiple: false } } } as T;
|
||||
}
|
||||
if (expression.includes("?.inputs[0]")) {
|
||||
return { result: { objectId: "input-object" } } as T;
|
||||
}
|
||||
return { result: { value: true } } as T;
|
||||
}
|
||||
return {} as T;
|
||||
});
|
||||
const cdp = {
|
||||
send,
|
||||
detachSession: vi.fn(async () => {}),
|
||||
ensureNetworkCapture: vi.fn(async () => {}),
|
||||
networkEntriesSince: vi.fn(() => ({
|
||||
tab_id: 7,
|
||||
entries: [],
|
||||
next_since: 0,
|
||||
truncated: false,
|
||||
})),
|
||||
setDeviceMetricsOverride: vi.fn(async () => {}),
|
||||
clearDeviceMetricsOverride: vi.fn(async () => {}),
|
||||
setUserAgentOverride: vi.fn(async () => {}),
|
||||
setTouchEmulationEnabled: vi.fn(async () => {}),
|
||||
};
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions, cdp: cdp as TestDispatcherCdp });
|
||||
dispatcher.start();
|
||||
|
||||
deliver(
|
||||
makeRequest("tool.upload", {
|
||||
session_id: "aa11",
|
||||
ref: "@e1",
|
||||
files: [{ transfer_id: "tr_1", name: "test.png", staged_path: "/stage/test.png" }],
|
||||
}),
|
||||
);
|
||||
await flushMicrotasks();
|
||||
await vi.waitFor(() => expect(sent).toHaveLength(1));
|
||||
|
||||
expect(sent[0]).toMatchObject({ result: { tab_id: 7, file_names: ["test.png"] } });
|
||||
expect(sendMessage).toHaveBeenNthCalledWith(1, 7, {
|
||||
type: "bh-automation-bypass",
|
||||
enabled: true,
|
||||
});
|
||||
expect(sendMessage).toHaveBeenNthCalledWith(2, 7, {
|
||||
type: "bh-automation-bypass",
|
||||
enabled: false,
|
||||
});
|
||||
});
|
||||
|
||||
it("detaches CDP state before stopping a session", async () => {
|
||||
const { transport, sent, deliver } = fakeTransport();
|
||||
const sessions = new SessionManager({
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 4242),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
await sessions.start("aa11");
|
||||
@@ -210,7 +457,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 4242),
|
||||
remove,
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
await sessions.start("aa11");
|
||||
@@ -258,7 +505,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 1),
|
||||
remove: vi.fn(),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions });
|
||||
@@ -278,7 +525,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 1),
|
||||
remove: vi.fn(),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions });
|
||||
@@ -302,7 +549,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 1),
|
||||
remove: vi.fn(),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions });
|
||||
@@ -319,7 +566,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 1),
|
||||
remove: vi.fn(),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions });
|
||||
@@ -336,7 +583,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 1),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
const onSessionsChanged = vi.fn();
|
||||
@@ -353,16 +600,32 @@ describe("ToolDispatcher", () => {
|
||||
});
|
||||
|
||||
it("invokes onBrowserControlResumed for browser-control tools but not passive reads", async () => {
|
||||
vi.stubGlobal("chrome", {
|
||||
tabs: {
|
||||
create: vi.fn(async (props: chrome.tabs.CreateProperties) => ({
|
||||
id: 7,
|
||||
windowId: props.windowId,
|
||||
url: props.url,
|
||||
active: props.active,
|
||||
})),
|
||||
},
|
||||
});
|
||||
const { transport, sent, deliver } = fakeTransport();
|
||||
const sessions = new SessionManager({
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 1),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
const onBrowserControlResumed = vi.fn();
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions, onBrowserControlResumed });
|
||||
const onAgentTabClaimed = vi.fn();
|
||||
const dispatcher = new ToolDispatcher({
|
||||
transport,
|
||||
sessions,
|
||||
onBrowserControlResumed,
|
||||
onAgentTabClaimed,
|
||||
});
|
||||
dispatcher.start();
|
||||
|
||||
deliver(makeRequest("tool.session_start", { session_id: "aa11" }));
|
||||
@@ -379,6 +642,7 @@ describe("ToolDispatcher", () => {
|
||||
});
|
||||
await flushMicrotasks();
|
||||
expect(onBrowserControlResumed).toHaveBeenCalledWith("aa11");
|
||||
expect(onAgentTabClaimed).toHaveBeenCalledWith(7, 1);
|
||||
});
|
||||
|
||||
it("reasserts remembered hover before follow-up work and releases only after actions", async () => {
|
||||
@@ -392,7 +656,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 1),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
const order: string[] = [];
|
||||
@@ -474,7 +738,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 1),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
const cdp = {
|
||||
@@ -524,6 +788,135 @@ describe("ToolDispatcher", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ tab_id: 7, code: "not_found" },
|
||||
{ tab_id: undefined, code: "invalid_params" },
|
||||
])("keeps hover when tab_return is rejected with $code", async ({ tab_id, code }) => {
|
||||
const sendMessage = vi.fn(async () => undefined);
|
||||
vi.stubGlobal("chrome", {
|
||||
tabs: {
|
||||
sendMessage,
|
||||
get: vi.fn(async () => ({ id: 7, windowId: 100, active: true })),
|
||||
query: vi.fn(async () => [{ id: 7, windowId: 100, active: true }]),
|
||||
},
|
||||
});
|
||||
const { transport, sent, deliver } = fakeTransport();
|
||||
const sessions = new SessionManager({
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 100),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
await sessions.start("aa11");
|
||||
const cdp = {
|
||||
send: vi.fn(async <T>() => ({}) as T),
|
||||
detachSession: vi.fn(async () => {}),
|
||||
releaseSessionTab: vi.fn(async () => {}),
|
||||
};
|
||||
const dispatcher = new ToolDispatcher({
|
||||
transport,
|
||||
sessions,
|
||||
cdp: cdp as unknown as TestDispatcherCdp,
|
||||
});
|
||||
const helpers = dispatcher as unknown as {
|
||||
rememberHover: (sessionId: string, result: unknown) => unknown;
|
||||
setHoverBypass: (sessionId: string, tabId: number, enabled: boolean) => Promise<void>;
|
||||
withHoverReassert: <T>(
|
||||
params: { session_id: string; tab_id?: number },
|
||||
work: () => Promise<T>,
|
||||
) => Promise<T>;
|
||||
};
|
||||
helpers.rememberHover("aa11", { tab_id: 7, x: 10, y: 20 });
|
||||
await helpers.setHoverBypass("aa11", 7, true);
|
||||
dispatcher.start();
|
||||
deliver(makeRequest("tool.tab_return", { session_id: "aa11", tab_id }));
|
||||
await vi.waitFor(() => expect(sent).toHaveLength(1));
|
||||
|
||||
expect(sent[0]).toMatchObject({ error: { code } });
|
||||
expect(cdp.releaseSessionTab).not.toHaveBeenCalled();
|
||||
expect(sendMessage).not.toHaveBeenCalledWith(7, expect.objectContaining({ enabled: false }));
|
||||
await helpers.withHoverReassert({ session_id: "aa11", tab_id: 7 }, async () => ({}));
|
||||
expect(cdp.send).toHaveBeenCalledExactlyOnceWith(7, "Input.dispatchMouseEvent", {
|
||||
type: "mouseMoved",
|
||||
x: 10,
|
||||
y: 20,
|
||||
});
|
||||
dispatcher.stop();
|
||||
});
|
||||
|
||||
it("releases the returned tab's hover before moving it and leaves other tabs alone", async () => {
|
||||
const sendMessage = vi.fn(async () => undefined);
|
||||
const move = vi.fn(async () => {
|
||||
expect(sendMessage).toHaveBeenCalledWith(7, expect.objectContaining({ enabled: false }));
|
||||
return { id: 7, windowId: 200, index: 4 };
|
||||
});
|
||||
vi.stubGlobal("chrome", {
|
||||
tabs: { sendMessage, move },
|
||||
windows: { get: vi.fn(async () => ({ id: 200 })) },
|
||||
});
|
||||
const { transport, sent, deliver } = fakeTransport();
|
||||
const sessions = new SessionManager({
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 100),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
const ctx = await sessions.start("aa11");
|
||||
ctx.borrowedTabs.set(7, { tabId: 7, originalWindowId: 200, originalIndex: 4 });
|
||||
const cdp = {
|
||||
send: vi.fn(async <T>() => ({}) as T),
|
||||
detachSession: vi.fn(async () => {}),
|
||||
releaseSessionTab: vi.fn(async () => {}),
|
||||
};
|
||||
const dispatcher = new ToolDispatcher({
|
||||
transport,
|
||||
sessions,
|
||||
cdp: cdp as unknown as TestDispatcherCdp,
|
||||
});
|
||||
const helpers = dispatcher as unknown as {
|
||||
rememberHover: (sessionId: string, result: unknown) => unknown;
|
||||
setHoverBypass: (sessionId: string, tabId: number, enabled: boolean) => Promise<void>;
|
||||
withHoverReassert: <T>(
|
||||
params: { session_id: string; tab_id?: number },
|
||||
work: () => Promise<T>,
|
||||
) => Promise<T>;
|
||||
};
|
||||
for (const [sessionId, tabId] of [
|
||||
["aa11", 7],
|
||||
["aa11", 8],
|
||||
["bb22", 9],
|
||||
] as const) {
|
||||
helpers.rememberHover(sessionId, { tab_id: tabId, x: 10, y: 20 });
|
||||
await helpers.setHoverBypass(sessionId, tabId, true);
|
||||
}
|
||||
dispatcher.start();
|
||||
deliver(makeRequest("tool.tab_return", { session_id: "aa11", tab_id: 7 }));
|
||||
await vi.waitFor(() => expect(sent).toHaveLength(1));
|
||||
|
||||
expect(sent[0]).toMatchObject({ result: { tab_id: 7, returned_to_window_id: 200 } });
|
||||
expect(cdp.releaseSessionTab).toHaveBeenCalledExactlyOnceWith("aa11", 7);
|
||||
await helpers.withHoverReassert({ session_id: "aa11", tab_id: 7 }, async () => ({}));
|
||||
expect(cdp.send).not.toHaveBeenCalled();
|
||||
for (const [sessionId, tabId] of [
|
||||
["aa11", 8],
|
||||
["bb22", 9],
|
||||
] as const) {
|
||||
expect(sendMessage).not.toHaveBeenCalledWith(
|
||||
tabId,
|
||||
expect.objectContaining({ enabled: false }),
|
||||
);
|
||||
await helpers.withHoverReassert({ session_id: sessionId, tab_id: tabId }, async () => ({}));
|
||||
expect(cdp.send).toHaveBeenLastCalledWith(tabId, "Input.dispatchMouseEvent", {
|
||||
type: "mouseMoved",
|
||||
x: 10,
|
||||
y: 20,
|
||||
});
|
||||
}
|
||||
dispatcher.stop();
|
||||
});
|
||||
|
||||
it("transfers hover overlay bypass ownership between sessions", async () => {
|
||||
vi.stubGlobal("chrome", {
|
||||
tabs: {
|
||||
@@ -535,7 +928,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 1),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions });
|
||||
@@ -572,7 +965,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 4242),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
await sessions.start("aa11");
|
||||
@@ -624,7 +1017,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 1),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions });
|
||||
@@ -658,7 +1051,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 4242),
|
||||
remove: vi.fn(),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
transport.send = () => {
|
||||
@@ -680,7 +1073,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 5555),
|
||||
remove,
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
transport.send = () => {
|
||||
@@ -708,7 +1101,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(() => createPromise),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions });
|
||||
@@ -759,7 +1152,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 1),
|
||||
remove: vi.fn(),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions });
|
||||
@@ -780,7 +1173,7 @@ describe("ToolDispatcher", () => {
|
||||
agentWindow: {
|
||||
create: vi.fn(() => createPromise),
|
||||
remove: vi.fn(),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions });
|
||||
|
||||
@@ -20,7 +20,7 @@ function fakeAgentWindow(ids: number[]) {
|
||||
return id;
|
||||
}),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { RpcError } from "@/transport/types";
|
||||
import { cdpError, classifyCdpError } from "../errors";
|
||||
|
||||
const denied = "Cannot access a chrome-extension:// URL of different extension";
|
||||
|
||||
describe("CDP extension access errors", () => {
|
||||
it("classifies both Chrome Error instances and string failures", () => {
|
||||
for (const error of [new Error(denied), denied]) {
|
||||
expect(cdpError(error)).toEqual({
|
||||
code: "cdp_failed",
|
||||
message: denied,
|
||||
data: { reason: "cdp_extension_access_denied" },
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
it("preserves transfer outcome and cleanup instructions when CDP was denied", () => {
|
||||
const error: RpcError = {
|
||||
code: "cdp_failed",
|
||||
message: denied,
|
||||
data: {
|
||||
reason: "transfer_outcome_unknown",
|
||||
effect_state: "unknown",
|
||||
cleanup_state: "failed",
|
||||
},
|
||||
};
|
||||
expect(classifyCdpError(error)).toBe(error);
|
||||
});
|
||||
|
||||
it("leaves other error codes and unrelated debugger failures unchanged", () => {
|
||||
for (const error of [
|
||||
{ code: "permission_denied", message: denied },
|
||||
{ code: "cdp_failed", message: "Another debugger is already attached" },
|
||||
] satisfies RpcError[]) {
|
||||
expect(classifyCdpError(error)).toBe(error);
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -12,7 +12,7 @@ function fakeAgentWindow(ids: number[]) {
|
||||
return id;
|
||||
}),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,988 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { SessionManager } from "@/session-manager/manager";
|
||||
import { type DownloadsApi, handleDownload } from "../download";
|
||||
import { captureBrowserDownload } from "../download-capture";
|
||||
import { uploadThroughFileDrop } from "../file-drop-transaction";
|
||||
import { uploadThroughActivatedFileInput } from "../file-input-transaction";
|
||||
import type { ResolvedActionTarget } from "../interaction";
|
||||
import type { CdpRunner } from "../shared";
|
||||
import { handleUpload } from "../upload";
|
||||
|
||||
function sessions() {
|
||||
return new SessionManager({
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 100),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function tabsApi() {
|
||||
return {
|
||||
get: vi.fn(
|
||||
async (tabId: number) => ({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab,
|
||||
),
|
||||
query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
|
||||
};
|
||||
}
|
||||
|
||||
function fakeEvent<T extends (...args: never[]) => unknown>() {
|
||||
const listeners = new Set<T>();
|
||||
return {
|
||||
addListener: (listener: T) => listeners.add(listener),
|
||||
removeListener: (listener: T) => listeners.delete(listener),
|
||||
emit: (...args: Parameters<T>) => {
|
||||
for (const listener of listeners) listener(...args);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function actionTarget(frameId?: string, sessionId?: string): ResolvedActionTarget {
|
||||
return {
|
||||
tab: { tabId: 4, windowId: 100, active: true },
|
||||
backendNodeId: 123,
|
||||
cdpTarget: { tabId: 4, ...(sessionId ? { sessionId } : {}) },
|
||||
...(frameId ? { frameId } : {}),
|
||||
usedRef: "e3",
|
||||
};
|
||||
}
|
||||
|
||||
function withoutOverlay<T>(operation: () => Promise<T>): Promise<T> {
|
||||
return operation();
|
||||
}
|
||||
|
||||
function uploadCdp(
|
||||
options: {
|
||||
inputCount?: number;
|
||||
multiple?: boolean;
|
||||
chooser?: { frameId?: string; backendNodeId?: number; mode?: string };
|
||||
pendingResolve?: boolean;
|
||||
} = {},
|
||||
) {
|
||||
const calls: Array<{ method: string; params?: object }> = [];
|
||||
let cdpEvent: Parameters<NonNullable<CdpRunner["onEvent"]>>[0] | undefined;
|
||||
const send = vi.fn(async (_tabId: number, method: string, params?: object) => {
|
||||
calls.push({ method, params });
|
||||
if (method === "DOM.scrollIntoViewIfNeeded") return {};
|
||||
if (method === "Page.setInterceptFileChooserDialog") return {};
|
||||
if (method === "Page.getLayoutMetrics")
|
||||
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
|
||||
if (method === "DOM.getContentQuads") return { quads: [[0, 0, 20, 0, 20, 20, 0, 20]] };
|
||||
if (method === "DOM.resolveNode") {
|
||||
if (options.pendingResolve) return new Promise<never>(() => {});
|
||||
return { object: { objectId: "trigger-object" } };
|
||||
}
|
||||
if (method === "Runtime.callFunctionOn") {
|
||||
const declaration = (params as { functionDeclaration?: string }).functionDeclaration ?? "";
|
||||
if (declaration.includes("Object.defineProperty")) return { result: { value: true } };
|
||||
if (declaration.includes("count: state.inputs.length")) {
|
||||
return {
|
||||
result: {
|
||||
value: {
|
||||
count: options.inputCount ?? 1,
|
||||
multiple: options.multiple ?? true,
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
if (declaration.includes("inputs[0]")) {
|
||||
return { result: { objectId: "input-object" } };
|
||||
}
|
||||
return { result: { value: true } };
|
||||
}
|
||||
if (method === "DOM.describeNode") return { node: { backendNodeId: 456 } };
|
||||
if (
|
||||
method === "Input.dispatchMouseEvent" &&
|
||||
(params as { type?: string }).type === "mousePressed" &&
|
||||
options.chooser
|
||||
) {
|
||||
cdpEvent?.({ tabId: 4 }, "Page.fileChooserOpened", options.chooser);
|
||||
}
|
||||
return {};
|
||||
});
|
||||
const cdp: CdpRunner = {
|
||||
send: send as unknown as CdpRunner["send"],
|
||||
onEvent: (handler) => {
|
||||
cdpEvent = handler;
|
||||
return { dispose: vi.fn() };
|
||||
},
|
||||
};
|
||||
return {
|
||||
calls,
|
||||
emitChooser: (event: { frameId?: string; backendNodeId?: number; mode?: string }) =>
|
||||
cdpEvent?.({ tabId: 4 }, "Page.fileChooserOpened", event),
|
||||
cdp,
|
||||
};
|
||||
}
|
||||
|
||||
function dropCdp(
|
||||
options: {
|
||||
targetContainsPoint?: boolean;
|
||||
dispatchError?: (type: string) => Error | Promise<never> | undefined;
|
||||
onCall?: (method: string) => void;
|
||||
} = {},
|
||||
) {
|
||||
const calls: Array<{ method: string; params?: object }> = [];
|
||||
const detach = vi.fn(async () => {});
|
||||
const send = vi.fn(async (_tabId: number, method: string, params?: object) => {
|
||||
calls.push({ method, params });
|
||||
options.onCall?.(method);
|
||||
if (method === "DOM.scrollIntoViewIfNeeded") return {};
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
|
||||
}
|
||||
if (method === "DOM.getContentQuads") {
|
||||
return { quads: [[10, 20, 210, 20, 210, 120, 10, 120]] };
|
||||
}
|
||||
if (method === "DOM.getNodeForLocation") return { backendNodeId: 124 };
|
||||
if (method === "DOM.resolveNode") {
|
||||
const id = (params as { backendNodeId?: number }).backendNodeId;
|
||||
return { object: { objectId: id === 123 ? "drop-target" : "hit-target" } };
|
||||
}
|
||||
if (method === "Runtime.callFunctionOn") {
|
||||
return { result: { value: options.targetContainsPoint ?? true } };
|
||||
}
|
||||
if (method === "Input.dispatchDragEvent") {
|
||||
const type = (params as { type?: string }).type ?? "";
|
||||
const error = options.dispatchError?.(type);
|
||||
if (error instanceof Promise) return error;
|
||||
if (error) throw error;
|
||||
return {};
|
||||
}
|
||||
if (method === "Runtime.releaseObjectGroup") return {};
|
||||
throw new Error(`unexpected drop CDP command ${method}`);
|
||||
});
|
||||
const cdp: CdpRunner = {
|
||||
send: send as unknown as CdpRunner["send"],
|
||||
detach,
|
||||
};
|
||||
return { calls, cdp, detach };
|
||||
}
|
||||
|
||||
describe("file transfer tools", () => {
|
||||
it("captures the file input activated by the requested click and injects only staged paths", async () => {
|
||||
const manager = sessions();
|
||||
const ctx = await manager.start("s1");
|
||||
ctx.refStore.set("e3", 123, { tabId: 4 });
|
||||
const { cdp, calls } = uploadCdp();
|
||||
|
||||
const result = await handleUpload(
|
||||
manager,
|
||||
{
|
||||
session_id: "s1",
|
||||
ref: "@e3",
|
||||
files: [
|
||||
{ transfer_id: "tr_1", name: "one.png", staged_path: "/private/stage/one" },
|
||||
{ transfer_id: "tr_2", name: "two.png", staged_path: "/private/stage/two" },
|
||||
],
|
||||
},
|
||||
{ cdp, tabsApi: tabsApi() },
|
||||
);
|
||||
|
||||
expect(result).toMatchObject({ tab_id: 4, file_names: ["one.png", "two.png"] });
|
||||
expect(calls[0]).toEqual({
|
||||
method: "Page.setInterceptFileChooserDialog",
|
||||
params: { enabled: true },
|
||||
});
|
||||
expect(calls).toContainEqual({
|
||||
method: "Page.setInterceptFileChooserDialog",
|
||||
params: { enabled: false, cancel: true },
|
||||
});
|
||||
expect(calls).toContainEqual({
|
||||
method: "DOM.setFileInputFiles",
|
||||
params: { files: ["/private/stage/one", "/private/stage/two"], backendNodeId: 456 },
|
||||
});
|
||||
});
|
||||
|
||||
it("fails immediately when the trigger does not activate a file input", async () => {
|
||||
const { cdp } = uploadCdp({ inputCount: 0 });
|
||||
const result = await uploadThroughActivatedFileInput({
|
||||
cdp,
|
||||
actionTarget: actionTarget(),
|
||||
files: ["/private/stage/one"],
|
||||
timeoutMs: 100,
|
||||
trigger: async () => ({ tab_id: 4, x: 10, y: 10 }),
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({
|
||||
code: "unsupported",
|
||||
data: { reason: "file_input_not_activated", phase: "resolve_input" },
|
||||
});
|
||||
});
|
||||
|
||||
it("fails before clicking when chooser interception is unavailable", async () => {
|
||||
const trigger = vi.fn(async () => ({ tab_id: 4, x: 10, y: 10 }));
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async (_tabId: number, method: string) => {
|
||||
if (method === "Page.setInterceptFileChooserDialog") {
|
||||
throw new Error("method unavailable");
|
||||
}
|
||||
return {};
|
||||
}) as CdpRunner["send"],
|
||||
};
|
||||
|
||||
const result = await uploadThroughActivatedFileInput({
|
||||
cdp,
|
||||
actionTarget: actionTarget(),
|
||||
files: ["/private/stage/one"],
|
||||
timeoutMs: 100,
|
||||
trigger,
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({
|
||||
code: "cdp_failed",
|
||||
data: { effect_state: "none", phase: "arm_interception" },
|
||||
});
|
||||
expect(trigger).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("uses an exact chooser event as an independent input-location signal", async () => {
|
||||
const { cdp, calls, emitChooser } = uploadCdp({ inputCount: 0 });
|
||||
const result = await uploadThroughActivatedFileInput({
|
||||
cdp,
|
||||
actionTarget: actionTarget("f1"),
|
||||
files: ["/private/stage/one"],
|
||||
timeoutMs: 100,
|
||||
trigger: async () => {
|
||||
emitChooser({ frameId: "f1", backendNodeId: 789, mode: "selectSingle" });
|
||||
return { tab_id: 4, x: 10, y: 10 };
|
||||
},
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({ multiple: false });
|
||||
expect(calls).toContainEqual({
|
||||
method: "DOM.setFileInputFiles",
|
||||
params: { files: ["/private/stage/one"], backendNodeId: 789 },
|
||||
});
|
||||
});
|
||||
|
||||
it("reports a File System Access picker without waiting for a timeout", async () => {
|
||||
const { cdp, emitChooser } = uploadCdp({
|
||||
inputCount: 0,
|
||||
});
|
||||
const result = await uploadThroughActivatedFileInput({
|
||||
cdp,
|
||||
actionTarget: actionTarget("f1"),
|
||||
files: ["/private/stage/one"],
|
||||
timeoutMs: 100,
|
||||
trigger: async () => {
|
||||
emitChooser({ frameId: "f1", mode: "selectSingle" });
|
||||
return { tab_id: 4, x: 10, y: 10 };
|
||||
},
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({
|
||||
code: "unsupported",
|
||||
message: "upload trigger invoked a non-input file picker",
|
||||
data: { reason: "file_input_not_activated", phase: "resolve_input" },
|
||||
});
|
||||
});
|
||||
|
||||
it("bounds a stuck file-input probe", async () => {
|
||||
const { cdp } = uploadCdp({ pendingResolve: true });
|
||||
|
||||
const result = await uploadThroughActivatedFileInput({
|
||||
cdp,
|
||||
actionTarget: actionTarget(),
|
||||
files: ["/private/stage/one"],
|
||||
timeoutMs: 5,
|
||||
trigger: vi.fn(),
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({
|
||||
code: "timeout",
|
||||
data: { reason: "file_input_probe_failed", phase: "arm_input_probe" },
|
||||
});
|
||||
});
|
||||
|
||||
it("marks a timed-out file assignment unknown and detaches browser state", async () => {
|
||||
const fixture = uploadCdp();
|
||||
const originalSend = fixture.cdp.send;
|
||||
const detach = vi.fn(async () => {});
|
||||
fixture.cdp.detach = detach;
|
||||
fixture.cdp.send = vi.fn((tabId: number, method: string, params?: object) => {
|
||||
if (method === "DOM.setFileInputFiles") return new Promise<never>(() => {});
|
||||
return originalSend(tabId, method, params);
|
||||
}) as CdpRunner["send"];
|
||||
|
||||
const result = await uploadThroughActivatedFileInput({
|
||||
cdp: fixture.cdp,
|
||||
actionTarget: actionTarget(),
|
||||
files: ["/private/stage/one"],
|
||||
timeoutMs: 10,
|
||||
trigger: async () => ({ tab_id: 4, x: 10, y: 10 }),
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({
|
||||
code: "timeout",
|
||||
data: { effect_state: "unknown", phase: "set_files" },
|
||||
});
|
||||
expect(detach).toHaveBeenCalledWith(4);
|
||||
});
|
||||
|
||||
it("delivers staged files through one explicit native drop without clicking", async () => {
|
||||
const manager = sessions();
|
||||
const ctx = await manager.start("s1");
|
||||
ctx.refStore.set("e3", 123, { tabId: 4 });
|
||||
const lifecycle: string[] = [];
|
||||
const { cdp, calls } = dropCdp({ onCall: (method) => lifecycle.push(method) });
|
||||
const overlayPhases: string[] = [];
|
||||
|
||||
const result = await handleUpload(
|
||||
manager,
|
||||
{
|
||||
session_id: "s1",
|
||||
ref: "@e3",
|
||||
mode: "drop",
|
||||
files: [{ transfer_id: "tr_1", name: "one.png", staged_path: "/stage/one.png" }],
|
||||
},
|
||||
{
|
||||
cdp,
|
||||
tabsApi: tabsApi(),
|
||||
sendToTab: vi.fn(async (_tabId, message) => {
|
||||
overlayPhases.push(message.phase);
|
||||
lifecycle.push(`overlay:${message.phase}`);
|
||||
}),
|
||||
},
|
||||
);
|
||||
|
||||
expect(result).toMatchObject({ tab_id: 4, used_ref: "e3", file_names: ["one.png"] });
|
||||
expect(calls.filter((call) => call.method === "Input.dispatchDragEvent")).toEqual([
|
||||
{
|
||||
method: "Input.dispatchDragEvent",
|
||||
params: {
|
||||
type: "dragEnter",
|
||||
x: 110,
|
||||
y: 70,
|
||||
data: { items: [], files: ["/stage/one.png"], dragOperationsMask: 1 },
|
||||
},
|
||||
},
|
||||
{
|
||||
method: "Input.dispatchDragEvent",
|
||||
params: {
|
||||
type: "dragOver",
|
||||
x: 110,
|
||||
y: 70,
|
||||
data: { items: [], files: ["/stage/one.png"], dragOperationsMask: 1 },
|
||||
},
|
||||
},
|
||||
{
|
||||
method: "Input.dispatchDragEvent",
|
||||
params: {
|
||||
type: "drop",
|
||||
x: 110,
|
||||
y: 70,
|
||||
data: { items: [], files: ["/stage/one.png"], dragOperationsMask: 1 },
|
||||
},
|
||||
},
|
||||
]);
|
||||
expect(calls.some((call) => call.method === "Input.dispatchMouseEvent")).toBe(false);
|
||||
expect(overlayPhases).toEqual(["begin", "end"]);
|
||||
expect(lifecycle.indexOf("DOM.getContentQuads")).toBeLessThan(
|
||||
lifecycle.indexOf("overlay:begin"),
|
||||
);
|
||||
expect(lifecycle.indexOf("overlay:begin")).toBeLessThan(
|
||||
lifecycle.indexOf("DOM.getNodeForLocation"),
|
||||
);
|
||||
expect(lifecycle.lastIndexOf("Input.dispatchDragEvent")).toBeLessThan(
|
||||
lifecycle.indexOf("overlay:end"),
|
||||
);
|
||||
expect(lifecycle.indexOf("overlay:end")).toBeLessThan(
|
||||
lifecycle.indexOf("Runtime.releaseObjectGroup"),
|
||||
);
|
||||
});
|
||||
|
||||
it("fails closed before dragging when another element owns the action point", async () => {
|
||||
const { cdp, calls } = dropCdp({ targetContainsPoint: false });
|
||||
const result = await uploadThroughFileDrop({
|
||||
cdp,
|
||||
actionTarget: actionTarget(),
|
||||
files: ["/stage/one.png"],
|
||||
timeoutMs: 100,
|
||||
withOverlayHidden: withoutOverlay,
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({
|
||||
code: "permission_denied",
|
||||
data: {
|
||||
reason: "file_drop_target_unavailable",
|
||||
mechanism: "drop",
|
||||
effect_state: "none",
|
||||
phase: "resolve_target",
|
||||
},
|
||||
});
|
||||
expect(calls.some((call) => call.method === "Input.dispatchDragEvent")).toBe(false);
|
||||
});
|
||||
|
||||
it("dispatches an OOPIF file drop in the child target's local coordinates", async () => {
|
||||
const rootCalls: Array<{ method: string; params?: object }> = [];
|
||||
const childCalls: Array<{ method: string; params?: object }> = [];
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async (_tabId, method, params) => {
|
||||
rootCalls.push({ method, params });
|
||||
if (method === "DOM.scrollIntoViewIfNeeded") return {};
|
||||
if (method === "DOM.getBoxModel") {
|
||||
return { model: { content: [100, 100, 300, 100, 300, 200, 100, 200] } };
|
||||
}
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 800, clientHeight: 600 } };
|
||||
}
|
||||
throw new Error(`unexpected root command ${method}`);
|
||||
}) as CdpRunner["send"],
|
||||
sendToTarget: vi.fn(async (_target, method, params) => {
|
||||
childCalls.push({ method, params });
|
||||
if (method === "DOM.scrollIntoViewIfNeeded") return {};
|
||||
if (method === "DOM.getContentQuads") {
|
||||
return { quads: [[10, 20, 210, 20, 210, 120, 10, 120]] };
|
||||
}
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 400, clientHeight: 300 } };
|
||||
}
|
||||
if (method === "DOM.getNodeForLocation") {
|
||||
return { backendNodeId: 124, frameId: "child" };
|
||||
}
|
||||
if (method === "DOM.resolveNode") {
|
||||
const id = (params as { backendNodeId?: number }).backendNodeId;
|
||||
return { object: { objectId: id === 123 ? "drop-target" : "hit-target" } };
|
||||
}
|
||||
if (method === "Runtime.callFunctionOn") return { result: { value: true } };
|
||||
if (method === "Input.dispatchDragEvent" || method === "Runtime.releaseObjectGroup") {
|
||||
return {};
|
||||
}
|
||||
throw new Error(`unexpected child command ${method}`);
|
||||
}) as CdpRunner["sendToTarget"],
|
||||
getFrameGraph: vi.fn(async () => ({
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "child",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 10,
|
||||
target: { tabId: 4, sessionId: "child-session" },
|
||||
},
|
||||
],
|
||||
})),
|
||||
};
|
||||
|
||||
const result = await uploadThroughFileDrop({
|
||||
cdp,
|
||||
actionTarget: actionTarget("child", "child-session"),
|
||||
files: ["/stage/one.png"],
|
||||
timeoutMs: 100,
|
||||
withOverlayHidden: withoutOverlay,
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({ x: 110, y: 70 });
|
||||
expect(
|
||||
childCalls
|
||||
.filter((call) => call.method === "Input.dispatchDragEvent")
|
||||
.map((call) => call.params),
|
||||
).toEqual([
|
||||
expect.objectContaining({ type: "dragEnter", x: 110, y: 70 }),
|
||||
expect.objectContaining({ type: "dragOver", x: 110, y: 70 }),
|
||||
expect.objectContaining({ type: "drop", x: 110, y: 70 }),
|
||||
]);
|
||||
expect(rootCalls.some((call) => call.method === "Input.dispatchDragEvent")).toBe(false);
|
||||
});
|
||||
|
||||
it("reports a missing native drag command as unsupported before commit", async () => {
|
||||
const { cdp } = dropCdp({
|
||||
dispatchError: (type) =>
|
||||
type === "dragEnter" ? new Error("'Input.dispatchDragEvent' wasn't found") : undefined,
|
||||
});
|
||||
const result = await uploadThroughFileDrop({
|
||||
cdp,
|
||||
actionTarget: actionTarget(),
|
||||
files: ["/stage/one.png"],
|
||||
timeoutMs: 100,
|
||||
withOverlayHidden: withoutOverlay,
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({
|
||||
code: "unsupported",
|
||||
data: {
|
||||
reason: "upload_mechanism_unsupported",
|
||||
effect_state: "none",
|
||||
phase: "drag_enter",
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("cancels an unresolved drag before returning a pre-commit timeout", async () => {
|
||||
const { cdp, calls } = dropCdp({
|
||||
dispatchError: (type) => (type === "dragEnter" ? new Promise<never>(() => {}) : undefined),
|
||||
});
|
||||
const result = await uploadThroughFileDrop({
|
||||
cdp,
|
||||
actionTarget: actionTarget(),
|
||||
files: ["/stage/one.png"],
|
||||
timeoutMs: 10,
|
||||
withOverlayHidden: withoutOverlay,
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({
|
||||
code: "timeout",
|
||||
data: { reason: "file_drop_failed", effect_state: "none", phase: "drag_enter" },
|
||||
});
|
||||
expect(calls).toContainEqual({
|
||||
method: "Input.dispatchDragEvent",
|
||||
params: expect.objectContaining({ type: "dragCancel" }),
|
||||
});
|
||||
});
|
||||
|
||||
it("marks an unresolved drop unknown and detaches browser state", async () => {
|
||||
const { cdp, detach } = dropCdp({
|
||||
dispatchError: (type) => (type === "drop" ? new Promise<never>(() => {}) : undefined),
|
||||
});
|
||||
const result = await uploadThroughFileDrop({
|
||||
cdp,
|
||||
actionTarget: actionTarget(),
|
||||
files: ["/stage/one.png"],
|
||||
timeoutMs: 10,
|
||||
withOverlayHidden: withoutOverlay,
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({
|
||||
code: "timeout",
|
||||
data: { reason: "file_drop_failed", effect_state: "unknown", phase: "drop" },
|
||||
});
|
||||
expect(detach).toHaveBeenCalledWith(4);
|
||||
});
|
||||
|
||||
it("routes one exact-target download through a browser-relative capability", async () => {
|
||||
const manager = sessions();
|
||||
const ctx = await manager.start("s1");
|
||||
ctx.refStore.set("e3", 123, { tabId: 4 });
|
||||
const onCreated = fakeEvent<(item: chrome.downloads.DownloadItem) => void>();
|
||||
const onChanged = fakeEvent<(delta: chrome.downloads.DownloadDelta) => void>();
|
||||
const onDeterminingFilename =
|
||||
fakeEvent<
|
||||
(
|
||||
item: chrome.downloads.DownloadItem,
|
||||
suggest: (suggestion?: chrome.downloads.DownloadFilenameSuggestion) => void,
|
||||
) => void | true
|
||||
>();
|
||||
const initial = {
|
||||
id: 9,
|
||||
url: "https://example.test/result.zip",
|
||||
finalUrl: "https://example.test/result.zip",
|
||||
filename: "result.zip",
|
||||
state: "in_progress",
|
||||
fileSize: -1,
|
||||
totalBytes: 12,
|
||||
mime: "application/zip",
|
||||
danger: "safe",
|
||||
} as chrome.downloads.DownloadItem;
|
||||
const completed = {
|
||||
...initial,
|
||||
filename: "/profile/Downloads/BrowserSkill/tr_1/result.zip",
|
||||
state: "complete",
|
||||
fileSize: 12,
|
||||
} as chrome.downloads.DownloadItem;
|
||||
const downloads: DownloadsApi = {
|
||||
onCreated,
|
||||
onChanged,
|
||||
onDeterminingFilename,
|
||||
search: vi.fn(async () => [completed]),
|
||||
cancel: vi.fn(async () => {}),
|
||||
removeFile: vi.fn(async () => {}),
|
||||
};
|
||||
let cdpEvent: Parameters<NonNullable<CdpRunner["onEvent"]>>[0] | undefined;
|
||||
let suggested: chrome.downloads.DownloadFilenameSuggestion | undefined;
|
||||
const send = vi.fn(async (_tabId: number, method: string, params?: object) => {
|
||||
if (method === "Page.getLayoutMetrics")
|
||||
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
|
||||
if (method === "DOM.getContentQuads") return { quads: [[0, 0, 20, 0, 20, 20, 0, 20]] };
|
||||
if (
|
||||
method === "Input.dispatchMouseEvent" &&
|
||||
(params as { type?: string }).type === "mousePressed"
|
||||
) {
|
||||
cdpEvent?.({ tabId: 4 }, "Page.downloadWillBegin", {
|
||||
url: initial.url,
|
||||
suggestedFilename: "result.zip",
|
||||
});
|
||||
await new Promise<void>((resolve) => {
|
||||
onDeterminingFilename.emit(initial, (value) => {
|
||||
suggested = value;
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
onCreated.emit(completed);
|
||||
}
|
||||
return {};
|
||||
});
|
||||
const cdp: CdpRunner = {
|
||||
send: send as unknown as CdpRunner["send"],
|
||||
onEvent: (handler) => {
|
||||
cdpEvent = handler;
|
||||
return { dispose: vi.fn() };
|
||||
},
|
||||
};
|
||||
|
||||
const result = await handleDownload(
|
||||
manager,
|
||||
{ session_id: "s1", ref: "@e3", browser_relative_dir: "BrowserSkill/tr_1" },
|
||||
{ cdp, tabsApi: tabsApi(), downloads },
|
||||
);
|
||||
|
||||
expect(suggested).toEqual({
|
||||
filename: "BrowserSkill/tr_1/result.zip",
|
||||
conflictAction: "overwrite",
|
||||
});
|
||||
expect(result).toMatchObject({
|
||||
tab_id: 4,
|
||||
suggested_filename: "result.zip",
|
||||
byte_size: 12,
|
||||
browser_path: completed.filename,
|
||||
});
|
||||
});
|
||||
|
||||
it("does not claim a download without an intent from the exact target", async () => {
|
||||
const onCreated = fakeEvent<(item: chrome.downloads.DownloadItem) => void>();
|
||||
const onChanged = fakeEvent<(delta: chrome.downloads.DownloadDelta) => void>();
|
||||
const onDeterminingFilename =
|
||||
fakeEvent<
|
||||
(
|
||||
item: chrome.downloads.DownloadItem,
|
||||
suggest: (suggestion?: chrome.downloads.DownloadFilenameSuggestion) => void,
|
||||
) => void | true
|
||||
>();
|
||||
const downloads: DownloadsApi = {
|
||||
onCreated,
|
||||
onChanged,
|
||||
onDeterminingFilename,
|
||||
search: vi.fn(async () => []),
|
||||
cancel: vi.fn(async () => {}),
|
||||
removeFile: vi.fn(async () => {}),
|
||||
};
|
||||
let cdpEvent: Parameters<NonNullable<CdpRunner["onEvent"]>>[0] | undefined;
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async () => ({})) as unknown as CdpRunner["send"],
|
||||
onEvent: (handler) => {
|
||||
cdpEvent = handler;
|
||||
return { dispose: vi.fn() };
|
||||
},
|
||||
};
|
||||
const unrelated = {
|
||||
id: 17,
|
||||
url: "https://example.test/unrelated.zip",
|
||||
finalUrl: "https://example.test/unrelated.zip",
|
||||
filename: "unrelated.zip",
|
||||
state: "in_progress",
|
||||
} as chrome.downloads.DownloadItem;
|
||||
let defaultSuggestionCalled = false;
|
||||
|
||||
const result = await captureBrowserDownload({
|
||||
cdp,
|
||||
target: { tabId: 4, sessionId: "expected-child" },
|
||||
downloads,
|
||||
browserRelativeDir: "BrowserSkill/tr_1",
|
||||
timeoutMs: 5,
|
||||
trigger: async () => {
|
||||
cdpEvent?.({ tabId: 4, sessionId: "other-child" }, "Page.downloadWillBegin", {
|
||||
url: unrelated.url,
|
||||
suggestedFilename: unrelated.filename,
|
||||
});
|
||||
onDeterminingFilename.emit(unrelated, (suggestion) => {
|
||||
defaultSuggestionCalled = suggestion === undefined;
|
||||
});
|
||||
return { tab_id: 4, x: 10, y: 10 };
|
||||
},
|
||||
});
|
||||
|
||||
expect(defaultSuggestionCalled).toBe(true);
|
||||
expect(downloads.cancel).not.toHaveBeenCalled();
|
||||
expect(result).toMatchObject({
|
||||
code: "cdp_failed",
|
||||
data: { reason: "download_capture_failed" },
|
||||
});
|
||||
});
|
||||
|
||||
it("correlates a filename candidate that arrives before the CDP intent", async () => {
|
||||
const onCreated = fakeEvent<(item: chrome.downloads.DownloadItem) => void>();
|
||||
const onChanged = fakeEvent<(delta: chrome.downloads.DownloadDelta) => void>();
|
||||
const onDeterminingFilename =
|
||||
fakeEvent<
|
||||
(
|
||||
item: chrome.downloads.DownloadItem,
|
||||
suggest: (suggestion?: chrome.downloads.DownloadFilenameSuggestion) => void,
|
||||
) => void | true
|
||||
>();
|
||||
const initial = {
|
||||
id: 21,
|
||||
url: "https://example.test/candidate-first.bin",
|
||||
finalUrl: "https://example.test/candidate-first.bin",
|
||||
filename: "candidate-first.bin",
|
||||
state: "in_progress",
|
||||
fileSize: -1,
|
||||
totalBytes: 4,
|
||||
bytesReceived: 0,
|
||||
} as chrome.downloads.DownloadItem;
|
||||
const complete = {
|
||||
...initial,
|
||||
state: "complete",
|
||||
fileSize: 4,
|
||||
} as chrome.downloads.DownloadItem;
|
||||
const downloads: DownloadsApi = {
|
||||
onCreated,
|
||||
onChanged,
|
||||
onDeterminingFilename,
|
||||
search: vi.fn(async () => [complete]),
|
||||
cancel: vi.fn(async () => {}),
|
||||
removeFile: vi.fn(async () => {}),
|
||||
};
|
||||
let cdpEvent: Parameters<NonNullable<CdpRunner["onEvent"]>>[0] | undefined;
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async () => ({})) as CdpRunner["send"],
|
||||
onEvent: (handler) => {
|
||||
cdpEvent = handler;
|
||||
return { dispose: vi.fn() };
|
||||
},
|
||||
};
|
||||
let suggestion: chrome.downloads.DownloadFilenameSuggestion | undefined;
|
||||
|
||||
const result = await captureBrowserDownload({
|
||||
cdp,
|
||||
target: { tabId: 4 },
|
||||
downloads,
|
||||
browserRelativeDir: "BrowserSkill/tr_21",
|
||||
timeoutMs: 1_000,
|
||||
trigger: async () => {
|
||||
const suggested = new Promise<void>((resolve) => {
|
||||
onDeterminingFilename.emit(initial, (value) => {
|
||||
suggestion = value;
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
cdpEvent?.({ tabId: 4 }, "Page.downloadWillBegin", {
|
||||
url: initial.url,
|
||||
suggestedFilename: initial.filename,
|
||||
});
|
||||
await suggested;
|
||||
onCreated.emit(complete);
|
||||
return { tab_id: 4, x: 10, y: 10 };
|
||||
},
|
||||
});
|
||||
|
||||
expect(suggestion).toEqual({
|
||||
filename: "BrowserSkill/tr_21/candidate-first.bin",
|
||||
conflictAction: "overwrite",
|
||||
});
|
||||
expect(result).toMatchObject({ item: { id: 21, state: "complete" } });
|
||||
});
|
||||
|
||||
it("removes a completed download when the final size exceeds the limit", async () => {
|
||||
const onCreated = fakeEvent<(item: chrome.downloads.DownloadItem) => void>();
|
||||
const onChanged = fakeEvent<(delta: chrome.downloads.DownloadDelta) => void>();
|
||||
const onDeterminingFilename =
|
||||
fakeEvent<
|
||||
(
|
||||
item: chrome.downloads.DownloadItem,
|
||||
suggest: (suggestion?: chrome.downloads.DownloadFilenameSuggestion) => void,
|
||||
) => void | true
|
||||
>();
|
||||
const initial = {
|
||||
id: 22,
|
||||
url: "https://example.test/oversized.bin",
|
||||
finalUrl: "https://example.test/oversized.bin",
|
||||
filename: "oversized.bin",
|
||||
state: "in_progress",
|
||||
fileSize: -1,
|
||||
totalBytes: -1,
|
||||
bytesReceived: 0,
|
||||
} as chrome.downloads.DownloadItem;
|
||||
const complete = {
|
||||
...initial,
|
||||
filename: "/profile/Downloads/BrowserSkill/tr_22/oversized.bin",
|
||||
state: "complete",
|
||||
fileSize: 8,
|
||||
totalBytes: 8,
|
||||
bytesReceived: 8,
|
||||
} as chrome.downloads.DownloadItem;
|
||||
const downloads: DownloadsApi = {
|
||||
onCreated,
|
||||
onChanged,
|
||||
onDeterminingFilename,
|
||||
search: vi.fn(async () => [complete]),
|
||||
cancel: vi.fn(async () => {}),
|
||||
removeFile: vi.fn(async () => {}),
|
||||
};
|
||||
let cdpEvent: Parameters<NonNullable<CdpRunner["onEvent"]>>[0] | undefined;
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async () => ({})) as CdpRunner["send"],
|
||||
onEvent: (handler) => {
|
||||
cdpEvent = handler;
|
||||
return { dispose: vi.fn() };
|
||||
},
|
||||
};
|
||||
|
||||
const result = await captureBrowserDownload({
|
||||
cdp,
|
||||
target: { tabId: 4 },
|
||||
downloads,
|
||||
browserRelativeDir: "BrowserSkill/tr_22",
|
||||
maxByteSize: 4,
|
||||
timeoutMs: 1_000,
|
||||
trigger: async () => {
|
||||
cdpEvent?.({ tabId: 4 }, "Page.downloadWillBegin", {
|
||||
url: initial.url,
|
||||
suggestedFilename: initial.filename,
|
||||
});
|
||||
await new Promise<void>((resolve) => {
|
||||
onDeterminingFilename.emit(initial, () => resolve());
|
||||
});
|
||||
onCreated.emit(initial);
|
||||
onChanged.emit({ id: initial.id, state: { current: "complete" } });
|
||||
return { tab_id: 4, x: 10, y: 10 };
|
||||
},
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({
|
||||
code: "cdp_failed",
|
||||
data: { reason: "download_capture_failed", effect_state: "committed" },
|
||||
});
|
||||
expect(downloads.removeFile).toHaveBeenCalledWith(initial.id);
|
||||
expect(downloads.cancel).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("reconciles a download that completes while cancellation is being requested", async () => {
|
||||
const onCreated = fakeEvent<(item: chrome.downloads.DownloadItem) => void>();
|
||||
const onChanged = fakeEvent<(delta: chrome.downloads.DownloadDelta) => void>();
|
||||
const onDeterminingFilename =
|
||||
fakeEvent<
|
||||
(
|
||||
item: chrome.downloads.DownloadItem,
|
||||
suggest: (suggestion?: chrome.downloads.DownloadFilenameSuggestion) => void,
|
||||
) => void | true
|
||||
>();
|
||||
const initial = {
|
||||
id: 23,
|
||||
url: "https://example.test/racing.bin",
|
||||
finalUrl: "https://example.test/racing.bin",
|
||||
filename: "racing.bin",
|
||||
state: "in_progress",
|
||||
fileSize: -1,
|
||||
totalBytes: -1,
|
||||
bytesReceived: 0,
|
||||
} as chrome.downloads.DownloadItem;
|
||||
const complete = {
|
||||
...initial,
|
||||
filename: "/profile/Downloads/BrowserSkill/tr_23/racing.bin",
|
||||
state: "complete",
|
||||
fileSize: 4,
|
||||
totalBytes: 4,
|
||||
bytesReceived: 4,
|
||||
} as chrome.downloads.DownloadItem;
|
||||
const downloads: DownloadsApi = {
|
||||
onCreated,
|
||||
onChanged,
|
||||
onDeterminingFilename,
|
||||
search: vi.fn().mockResolvedValueOnce([initial]).mockResolvedValueOnce([complete]),
|
||||
cancel: vi.fn(async () => {
|
||||
throw new Error("download already complete");
|
||||
}),
|
||||
removeFile: vi.fn(async () => {}),
|
||||
};
|
||||
let cdpEvent: Parameters<NonNullable<CdpRunner["onEvent"]>>[0] | undefined;
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async () => ({})) as CdpRunner["send"],
|
||||
onEvent: (handler) => {
|
||||
cdpEvent = handler;
|
||||
return { dispose: vi.fn() };
|
||||
},
|
||||
};
|
||||
|
||||
const result = await captureBrowserDownload({
|
||||
cdp,
|
||||
target: { tabId: 4 },
|
||||
downloads,
|
||||
browserRelativeDir: "BrowserSkill/tr_23",
|
||||
timeoutMs: 80,
|
||||
trigger: async () => {
|
||||
cdpEvent?.({ tabId: 4 }, "Page.downloadWillBegin", {
|
||||
url: initial.url,
|
||||
suggestedFilename: initial.filename,
|
||||
});
|
||||
await new Promise<void>((resolve) => {
|
||||
onDeterminingFilename.emit(initial, () => resolve());
|
||||
});
|
||||
onCreated.emit(initial);
|
||||
return { tab_id: 4, x: 10, y: 10 };
|
||||
},
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({
|
||||
code: "cdp_failed",
|
||||
data: { reason: "download_capture_failed", effect_state: "committed" },
|
||||
});
|
||||
expect(downloads.cancel).toHaveBeenCalledWith(initial.id);
|
||||
expect(downloads.removeFile).toHaveBeenCalledWith(initial.id);
|
||||
expect(result).not.toMatchObject({ data: { cleanup_state: "failed" } });
|
||||
});
|
||||
|
||||
it("rejects ambiguous attribution without cancelling either unclaimed download", async () => {
|
||||
const onCreated = fakeEvent<(item: chrome.downloads.DownloadItem) => void>();
|
||||
const onChanged = fakeEvent<(delta: chrome.downloads.DownloadDelta) => void>();
|
||||
const onDeterminingFilename =
|
||||
fakeEvent<
|
||||
(
|
||||
item: chrome.downloads.DownloadItem,
|
||||
suggest: (suggestion?: chrome.downloads.DownloadFilenameSuggestion) => void,
|
||||
) => void | true
|
||||
>();
|
||||
const downloads: DownloadsApi = {
|
||||
onCreated,
|
||||
onChanged,
|
||||
onDeterminingFilename,
|
||||
search: vi.fn(async () => []),
|
||||
cancel: vi.fn(async () => {}),
|
||||
removeFile: vi.fn(async () => {}),
|
||||
};
|
||||
let cdpEvent: Parameters<NonNullable<CdpRunner["onEvent"]>>[0] | undefined;
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async () => ({})) as CdpRunner["send"],
|
||||
onEvent: (handler) => {
|
||||
cdpEvent = handler;
|
||||
return { dispose: vi.fn() };
|
||||
},
|
||||
};
|
||||
const defaults: number[] = [];
|
||||
const candidate = (id: number) =>
|
||||
({
|
||||
id,
|
||||
url: "https://example.test/same.bin",
|
||||
finalUrl: "https://example.test/same.bin",
|
||||
filename: "same.bin",
|
||||
state: "in_progress",
|
||||
}) as chrome.downloads.DownloadItem;
|
||||
|
||||
const result = await captureBrowserDownload({
|
||||
cdp,
|
||||
target: { tabId: 4 },
|
||||
downloads,
|
||||
browserRelativeDir: "BrowserSkill/tr_ambiguous",
|
||||
timeoutMs: 100,
|
||||
trigger: async () => {
|
||||
cdpEvent?.({ tabId: 4 }, "Page.downloadWillBegin", {
|
||||
url: "https://example.test/same.bin",
|
||||
suggestedFilename: "same.bin",
|
||||
});
|
||||
for (const id of [31, 32]) {
|
||||
onDeterminingFilename.emit(candidate(id), (value) => {
|
||||
if (value === undefined) defaults.push(id);
|
||||
});
|
||||
}
|
||||
return { tab_id: 4, x: 10, y: 10 };
|
||||
},
|
||||
});
|
||||
|
||||
expect(defaults.sort()).toEqual([31, 32]);
|
||||
expect(downloads.cancel).not.toHaveBeenCalled();
|
||||
expect(result).toMatchObject({
|
||||
code: "cdp_failed",
|
||||
data: { effect_state: "unknown", phase: "attribution" },
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,573 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { SessionManager } from "@/session-manager/manager";
|
||||
import type { CdpRunner } from "@/tools/shared";
|
||||
import { handleFill } from "../interaction";
|
||||
|
||||
interface ScriptParams {
|
||||
functionDeclaration: string;
|
||||
arguments?: Array<{ value: unknown }>;
|
||||
}
|
||||
|
||||
async function setup(markup = '<input value="old">') {
|
||||
document.body.innerHTML = markup;
|
||||
const element = document.body.firstElementChild as HTMLInputElement;
|
||||
const manager = new SessionManager({
|
||||
agentWindow: {
|
||||
create: async () => 100,
|
||||
remove: async () => {},
|
||||
ensureActiveTab: async () => 4,
|
||||
},
|
||||
});
|
||||
const ctx = await manager.start("aa11");
|
||||
ctx.refStore.set("e1", 12, { tabId: 4 });
|
||||
const script = vi.fn(async (params: ScriptParams): Promise<unknown> => {
|
||||
try {
|
||||
const fn = new Function(`return (${params.functionDeclaration})`)();
|
||||
return {
|
||||
result: {
|
||||
value: await fn.apply(
|
||||
element,
|
||||
(params.arguments ?? []).map((arg) => arg.value),
|
||||
),
|
||||
},
|
||||
};
|
||||
} catch {
|
||||
return { exceptionDetails: { text: "test page script failed" } };
|
||||
}
|
||||
});
|
||||
// Number/email controls have a browser caret even though their selection
|
||||
// getters return null. Model that caret instead of assuming it is at the end.
|
||||
const hiddenCaret = { start: 0, end: 0 };
|
||||
const moveToEnd = vi.fn(async () => {
|
||||
if (typeof element.selectionStart === "number") {
|
||||
element.setSelectionRange(element.value.length, element.value.length);
|
||||
} else {
|
||||
hiddenCaret.start = hiddenCaret.end = element.value.length;
|
||||
}
|
||||
});
|
||||
const insert = vi.fn(async (text: string) => {
|
||||
const focused = document.activeElement as HTMLInputElement;
|
||||
if (focused.readOnly || focused.disabled) return;
|
||||
const start = focused.selectionStart ?? hiddenCaret.start;
|
||||
const end = focused.selectionEnd ?? hiddenCaret.end;
|
||||
const value = focused.value.slice(0, start) + text + focused.value.slice(end);
|
||||
focused.value = focused.maxLength >= 0 ? value.slice(0, focused.maxLength) : value;
|
||||
hiddenCaret.start = hiddenCaret.end = Math.min(start + text.length, focused.value.length);
|
||||
focused.dispatchEvent(new Event("input", { bubbles: true }));
|
||||
});
|
||||
const release = vi.fn(async () => ({}));
|
||||
const send = vi.fn(async (_tabId: number, method: string, params?: object) => {
|
||||
switch (method) {
|
||||
case "DOM.describeNode":
|
||||
return {
|
||||
node: {
|
||||
nodeName: element.tagName,
|
||||
attributes: Array.from(element.attributes).flatMap((attr) => [attr.name, attr.value]),
|
||||
},
|
||||
};
|
||||
case "DOM.scrollIntoViewIfNeeded":
|
||||
return {};
|
||||
case "DOM.focus":
|
||||
element.focus();
|
||||
return {};
|
||||
case "DOM.resolveNode":
|
||||
return { object: { objectId: "fill-target" } };
|
||||
case "Runtime.callFunctionOn":
|
||||
return script(params as ScriptParams);
|
||||
case "Input.insertText":
|
||||
await insert((params as { text: string }).text);
|
||||
return {};
|
||||
case "Input.dispatchKeyEvent":
|
||||
if ((params as { commands?: string[] }).commands?.includes("moveToEndOfDocument")) {
|
||||
await moveToEnd();
|
||||
}
|
||||
return {};
|
||||
case "Runtime.releaseObject":
|
||||
return release();
|
||||
default:
|
||||
throw new Error(`unexpected ${method}`);
|
||||
}
|
||||
});
|
||||
return {
|
||||
element,
|
||||
hiddenCaret,
|
||||
moveToEnd,
|
||||
script,
|
||||
insert,
|
||||
release,
|
||||
send,
|
||||
fill: (value = "hello", clearBefore = true, signal?: AbortSignal) =>
|
||||
handleFill(
|
||||
manager,
|
||||
{ session_id: "aa11", ref: "e1", value, clear_before: clearBefore },
|
||||
{
|
||||
cdp: { send: send as CdpRunner["send"] },
|
||||
tabsApi: {
|
||||
get: async (id) => ({ id, windowId: 100, active: true }) as chrome.tabs.Tab,
|
||||
query: async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab],
|
||||
},
|
||||
signal,
|
||||
},
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
document.body.innerHTML = "";
|
||||
});
|
||||
|
||||
describe("fill result verification", () => {
|
||||
it.each([
|
||||
"readonly",
|
||||
"disabled",
|
||||
])("preserves a %s field before rejecting it", async (attribute) => {
|
||||
const h = await setup(`<input ${attribute} value="old">`);
|
||||
expect(await h.fill()).toMatchObject({
|
||||
code: "invalid_params",
|
||||
data: { reason: "target_not_fillable" },
|
||||
});
|
||||
expect(h.element.value).toBe("old");
|
||||
expect(h.insert).not.toHaveBeenCalled();
|
||||
expect(h.release).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it.each([
|
||||
"checkbox",
|
||||
"radio",
|
||||
"range",
|
||||
"date",
|
||||
"file",
|
||||
])("rejects an unsupported %s input before clearing", async (type) => {
|
||||
const h = await setup(`<input type="${type}">`);
|
||||
const before = h.element.value;
|
||||
expect(await h.fill()).toMatchObject({
|
||||
code: "invalid_params",
|
||||
data: { reason: "target_not_fillable" },
|
||||
});
|
||||
expect(h.element.value).toBe(before);
|
||||
expect(h.insert).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("rejects an overlong value before clearing", async () => {
|
||||
const h = await setup('<input maxlength="3" value="old">');
|
||||
expect(await h.fill()).toMatchObject({
|
||||
code: "invalid_params",
|
||||
message: /maxlength/,
|
||||
data: { reason: "fill_value_invalid" },
|
||||
});
|
||||
expect(h.element.value).toBe("old");
|
||||
expect(h.insert).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each([
|
||||
true,
|
||||
false,
|
||||
])("classifies a DOM.focus rejection on a disabled=%s target", async (disabled) => {
|
||||
const h = await setup(`<input ${disabled ? "disabled" : ""} value="old">`);
|
||||
const send = h.send.getMockImplementation()!;
|
||||
h.send.mockImplementation(async (tabId, method, params) => {
|
||||
if (method === "DOM.focus") throw new Error("Element is not focusable");
|
||||
return send(tabId, method, params);
|
||||
});
|
||||
expect(await h.fill()).toMatchObject({
|
||||
code: disabled ? "invalid_params" : "cdp_failed",
|
||||
data: { reason: disabled ? "target_not_fillable" : "fill_failed" },
|
||||
});
|
||||
expect(h.element.value).toBe("old");
|
||||
expect(h.insert).not.toHaveBeenCalled();
|
||||
expect(h.release).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("fills a background document without requiring foreground browser focus", async () => {
|
||||
const h = await setup();
|
||||
vi.spyOn(document, "hasFocus").mockReturnValue(false);
|
||||
expect(await h.fill()).toMatchObject({ value_length: 5 });
|
||||
expect(h.element.value).toBe("hello");
|
||||
});
|
||||
|
||||
it("restores focus once after clearing without typing into another field", async () => {
|
||||
const h = await setup();
|
||||
const other = document.createElement("input");
|
||||
document.body.append(other);
|
||||
h.element.addEventListener("input", () => other.focus());
|
||||
expect(await h.fill()).toMatchObject({ value_length: 5 });
|
||||
expect(h.element.value).toBe("hello");
|
||||
expect(other.value).toBe("");
|
||||
expect(h.send.mock.calls.filter(([, method]) => method === "DOM.focus")).toHaveLength(2);
|
||||
});
|
||||
|
||||
it.each([
|
||||
"focus",
|
||||
"value",
|
||||
])("rechecks %s after restoring focus, without retrying again", async (action) => {
|
||||
const h = await setup();
|
||||
const other = document.createElement("input");
|
||||
document.body.append(other);
|
||||
h.element.addEventListener(
|
||||
"input",
|
||||
() => {
|
||||
other.focus();
|
||||
h.element.addEventListener("focus", () => {
|
||||
if (action === "focus") queueMicrotask(() => other.focus());
|
||||
else h.element.value = "changed";
|
||||
});
|
||||
},
|
||||
{ once: true },
|
||||
);
|
||||
expect(await h.fill()).toMatchObject({
|
||||
code: "cdp_failed",
|
||||
data: { reason: action === "focus" ? "fill_focus_lost" : "fill_target_changed" },
|
||||
});
|
||||
expect(other.value).toBe("");
|
||||
expect(h.insert).not.toHaveBeenCalled();
|
||||
expect(h.send.mock.calls.filter(([, method]) => method === "DOM.focus")).toHaveLength(2);
|
||||
});
|
||||
|
||||
it.each([
|
||||
"remove",
|
||||
"readonly",
|
||||
"restore",
|
||||
"microtask-restore",
|
||||
])("stops when clearing causes the page to %s the target", async (action) => {
|
||||
const h = await setup();
|
||||
h.element.addEventListener("input", () => {
|
||||
if (action === "remove") h.element.remove();
|
||||
if (action === "readonly") h.element.readOnly = true;
|
||||
if (action === "restore") h.element.value = "restored";
|
||||
if (action === "microtask-restore")
|
||||
queueMicrotask(() => {
|
||||
h.element.value = "restored";
|
||||
});
|
||||
});
|
||||
expect(await h.fill()).toMatchObject({
|
||||
code: "cdp_failed",
|
||||
data: { reason: "fill_target_changed" },
|
||||
});
|
||||
expect(h.insert).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("fails when insertText is acknowledged without changing the value", async () => {
|
||||
const h = await setup();
|
||||
h.insert.mockImplementation(async () => {});
|
||||
expect(await h.fill()).toMatchObject({
|
||||
code: "cdp_failed",
|
||||
data: { reason: "fill_value_mismatch" },
|
||||
});
|
||||
expect(h.release).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("detects truncation when maxlength changes after preparation", async () => {
|
||||
const h = await setup();
|
||||
h.element.addEventListener(
|
||||
"input",
|
||||
() => {
|
||||
h.element.maxLength = 3;
|
||||
},
|
||||
{ once: true },
|
||||
);
|
||||
expect(await h.fill()).toMatchObject({
|
||||
code: "cdp_failed",
|
||||
data: { reason: "fill_value_mismatch" },
|
||||
});
|
||||
expect(h.element.value).toBe("hel");
|
||||
});
|
||||
|
||||
it.each([
|
||||
"input",
|
||||
"change",
|
||||
"microtask-change",
|
||||
])("detects a value reverted by the %s handler", async (event) => {
|
||||
const h = await setup();
|
||||
h.element.addEventListener(event === "microtask-change" ? "change" : event, () => {
|
||||
if (event === "microtask-change")
|
||||
queueMicrotask(() => {
|
||||
h.element.value = "";
|
||||
});
|
||||
else h.element.value = "";
|
||||
});
|
||||
expect(await h.fill()).toMatchObject({ code: "cdp_failed", message: /expected value/ });
|
||||
});
|
||||
|
||||
it("compares content rather than length without including field values in the error", async () => {
|
||||
const h = await setup('<input type="password">');
|
||||
h.element.addEventListener("change", () => {
|
||||
h.element.value = "other-secret";
|
||||
});
|
||||
const result = await h.fill("given-secret");
|
||||
expect(result).toMatchObject({ code: "cdp_failed", message: /expected value/ });
|
||||
expect(JSON.stringify(result)).not.toMatch(/given-secret|other-secret/);
|
||||
});
|
||||
|
||||
it("does not report success for a target replaced by a change handler", async () => {
|
||||
const h = await setup();
|
||||
h.element.addEventListener("change", () => h.element.replaceWith(h.element.cloneNode()));
|
||||
expect(await h.fill()).toMatchObject({
|
||||
code: "cdp_failed",
|
||||
data: { reason: "fill_target_changed" },
|
||||
});
|
||||
});
|
||||
|
||||
it.each([
|
||||
1, 2, 3, 4, 5,
|
||||
])("rejects script exceptions at phase %s and releases the target", async (phase) => {
|
||||
const h = await setup();
|
||||
vi.spyOn(document, "hasFocus").mockReturnValue(false);
|
||||
const run = h.script.getMockImplementation()!;
|
||||
let count = 0;
|
||||
h.script.mockImplementation(async (params) =>
|
||||
++count === phase ? { exceptionDetails: { text: "page secret" } } : run(params),
|
||||
);
|
||||
const result = await h.fill();
|
||||
expect(result).toMatchObject({ code: "cdp_failed", data: { reason: "fill_failed" } });
|
||||
expect(JSON.stringify(result)).not.toContain("page secret");
|
||||
if (phase <= 3) expect(h.insert).not.toHaveBeenCalled();
|
||||
expect(h.release).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it.each([1, 2, 3, 5])("rejects missing results at phase %s", async (phase) => {
|
||||
const h = await setup();
|
||||
vi.spyOn(document, "hasFocus").mockReturnValue(false);
|
||||
const run = h.script.getMockImplementation()!;
|
||||
let count = 0;
|
||||
h.script.mockImplementation(async (params) =>
|
||||
++count === phase ? { result: { type: "undefined" } } : run(params),
|
||||
);
|
||||
expect(await h.fill()).toMatchObject({ code: "cdp_failed" });
|
||||
if (phase <= 3) expect(h.insert).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each(["input", "change"])("checks state after nested microtasks from %s", async (event) => {
|
||||
const h = await setup();
|
||||
h.element.addEventListener(event, () => {
|
||||
queueMicrotask(() =>
|
||||
queueMicrotask(() => {
|
||||
h.element.value = "reverted";
|
||||
}),
|
||||
);
|
||||
});
|
||||
expect(await h.fill()).toMatchObject({ code: "cdp_failed" });
|
||||
if (event === "input") expect(h.insert).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each([
|
||||
"<input>",
|
||||
'<input type="password">',
|
||||
"<textarea></textarea>",
|
||||
])("verifies successful text in %s", async (markup) => {
|
||||
const h = await setup(markup);
|
||||
expect(await h.fill("你好🙂")).toMatchObject({ value_length: 4 });
|
||||
expect(h.element.value).toBe("你好🙂");
|
||||
expect(h.release).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("verifies append against the old value plus the requested text", async () => {
|
||||
const h = await setup();
|
||||
h.element.focus();
|
||||
h.element.setSelectionRange(3, 3);
|
||||
expect(await h.fill("!", false)).toMatchObject({ value_length: 4 });
|
||||
expect(h.element.value).toBe("old!");
|
||||
});
|
||||
|
||||
it.each([
|
||||
["<textarea></textarea>", "a\r\nb", "a\nb"],
|
||||
["<input>", "a\nb", "a b"],
|
||||
["<input>", "a\r\nb", "a b"],
|
||||
[
|
||||
'<input type="email" multiple>',
|
||||
" a@example.com , b@example.com ",
|
||||
"a@example.com,b@example.com",
|
||||
],
|
||||
])("accepts native normalization for %s", async (markup, requested, normalized) => {
|
||||
if (markup.startsWith("<textarea")) {
|
||||
// Happy DOM does not implement Chrome's textarea line-ending normalization.
|
||||
const setter = Object.getOwnPropertyDescriptor(HTMLTextAreaElement.prototype, "value")!.set!;
|
||||
vi.spyOn(HTMLTextAreaElement.prototype, "value", "set").mockImplementation(function (
|
||||
this: HTMLTextAreaElement,
|
||||
value,
|
||||
) {
|
||||
setter.call(this, value.replace(/\r\n?/g, "\n"));
|
||||
});
|
||||
}
|
||||
const h = await setup(markup);
|
||||
h.insert.mockImplementation(async () => {
|
||||
h.element.value = normalized;
|
||||
});
|
||||
expect(await h.fill(requested)).toMatchObject({ value_length: normalized.length });
|
||||
});
|
||||
|
||||
it("does not count an editable padding break as an extra typed character", async () => {
|
||||
const h = await setup('<div contenteditable="true" tabindex="0"></div>');
|
||||
// These two layout/editability properties are missing in Happy DOM.
|
||||
let innerText = "";
|
||||
Object.defineProperty(h.element, "isContentEditable", { get: () => true });
|
||||
Object.defineProperty(h.element, "innerText", { get: () => innerText });
|
||||
h.insert.mockImplementation(async () => {
|
||||
h.element.innerHTML = "one<div><br></div>";
|
||||
innerText = "one\n\n";
|
||||
});
|
||||
expect(await h.fill("one\n")).toMatchObject({ value_length: 4 });
|
||||
});
|
||||
|
||||
it.each([
|
||||
[0, 0],
|
||||
[1, 1],
|
||||
[0, 3],
|
||||
])("appends after moving selection %s:%s to the end", async (start, end) => {
|
||||
const h = await setup();
|
||||
h.element.focus();
|
||||
h.element.setSelectionRange(start, end);
|
||||
expect(await h.fill("🙂", false)).toMatchObject({ value_length: 5 });
|
||||
expect(h.element.value).toBe("old🙂");
|
||||
});
|
||||
|
||||
it.each(
|
||||
["number", "email"].flatMap((type) =>
|
||||
[
|
||||
[0, 0],
|
||||
[1, 1],
|
||||
[0, 2],
|
||||
].map(([start, end]) => ({ type, start, end })),
|
||||
),
|
||||
)("appends to $type from selection $start:$end without using its unsupported API", async ({
|
||||
type,
|
||||
start,
|
||||
end,
|
||||
}) => {
|
||||
const h = await setup(`<input type="${type}" value="12">`);
|
||||
Object.assign(h.hiddenCaret, { start, end });
|
||||
vi.spyOn(h.element, "selectionStart", "get").mockReturnValue(null);
|
||||
vi.spyOn(h.element, "selectionEnd", "get").mockReturnValue(null);
|
||||
const selection = vi.spyOn(h.element, "setSelectionRange").mockImplementation(() => {
|
||||
throw new DOMException("unsupported", "InvalidStateError");
|
||||
});
|
||||
expect(await h.fill("3", false)).toMatchObject({ value_length: 3 });
|
||||
expect(h.element.value).toBe("123");
|
||||
expect(selection).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each([
|
||||
"number",
|
||||
"email",
|
||||
])("does not report successful %s append when the caret command is ignored", async (type) => {
|
||||
const h = await setup(`<input type="${type}" value="12">`);
|
||||
vi.spyOn(h.element, "selectionStart", "get").mockReturnValue(null);
|
||||
vi.spyOn(h.element, "selectionEnd", "get").mockReturnValue(null);
|
||||
h.moveToEnd.mockResolvedValueOnce(undefined);
|
||||
expect(await h.fill("3", false)).toMatchObject({
|
||||
code: "cdp_failed",
|
||||
data: { reason: "fill_value_mismatch" },
|
||||
});
|
||||
expect(h.element.value).toBe("312");
|
||||
expect(h.insert).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("rejects invalid number text before clearing", async () => {
|
||||
const h = await setup('<input type="number" value="12">');
|
||||
expect(await h.fill("abc")).toMatchObject({
|
||||
code: "invalid_params",
|
||||
data: { reason: "fill_value_invalid" },
|
||||
});
|
||||
expect(h.element.value).toBe("12");
|
||||
expect(h.insert).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("reports page formatting as unconfirmed without retrying the write", async () => {
|
||||
const h = await setup();
|
||||
h.element.addEventListener("change", () => {
|
||||
h.element.value = h.element.value.toUpperCase();
|
||||
});
|
||||
expect(await h.fill()).toMatchObject({
|
||||
code: "cdp_failed",
|
||||
data: { reason: "fill_value_mismatch" },
|
||||
});
|
||||
expect(h.element.value).toBe("HELLO");
|
||||
expect(h.insert).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it.each([
|
||||
["focus", false],
|
||||
["focus", true],
|
||||
["value", false],
|
||||
["value", true],
|
||||
["cancel", false],
|
||||
["cancel", true],
|
||||
["throw", false],
|
||||
["throw", true],
|
||||
] as const)("stops when positioning triggers %s (clear=%s) and releases the key", async (action, clearBefore) => {
|
||||
const h = await setup();
|
||||
vi.spyOn(document, "hasFocus").mockReturnValue(false);
|
||||
const controller = new AbortController();
|
||||
const other = document.createElement("input");
|
||||
document.body.append(other);
|
||||
const send = h.send.getMockImplementation()!;
|
||||
h.send.mockImplementation(async (tabId, method, params) => {
|
||||
const reply = await send(tabId, method, params);
|
||||
if (
|
||||
method === "Input.dispatchKeyEvent" &&
|
||||
(params as { type: string }).type === "rawKeyDown"
|
||||
) {
|
||||
if (action === "focus") other.focus();
|
||||
if (action === "value") h.element.value = "changed";
|
||||
if (action === "cancel") controller.abort();
|
||||
if (action === "throw") throw new Error("dispatch failed");
|
||||
}
|
||||
return reply;
|
||||
});
|
||||
const result = await h.fill("!", clearBefore, controller.signal);
|
||||
expect(result).toMatchObject({ code: action === "cancel" ? "cancelled" : "cdp_failed" });
|
||||
expect(h.insert).not.toHaveBeenCalled();
|
||||
expect(other.value).toBe("");
|
||||
expect(h.send).toHaveBeenCalledWith(
|
||||
4,
|
||||
"Input.dispatchKeyEvent",
|
||||
expect.objectContaining({ type: "keyUp" }),
|
||||
);
|
||||
expect(h.release).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it.each([true, false])("verifies an empty request with clear_before=%s", async (clearBefore) => {
|
||||
const h = await setup();
|
||||
expect(await h.fill("", clearBefore)).toMatchObject({ value_length: clearBefore ? 0 : 3 });
|
||||
expect(h.element.value).toBe(clearBefore ? "" : "old");
|
||||
expect(h.insert).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("cancels after insertion without sending change events", async () => {
|
||||
const h = await setup();
|
||||
const controller = new AbortController();
|
||||
h.insert.mockImplementation(async () => controller.abort());
|
||||
expect(await h.fill("hello", true, controller.signal)).toMatchObject({ code: "cancelled" });
|
||||
expect(h.script).toHaveBeenCalledTimes(2);
|
||||
expect(h.release).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("cancels after resolving the remote object without clearing the field", async () => {
|
||||
const h = await setup();
|
||||
const controller = new AbortController();
|
||||
const send = h.send.getMockImplementation()!;
|
||||
h.send.mockImplementation(async (tabId, method, params) => {
|
||||
const result = await send(tabId, method, params);
|
||||
if (method === "DOM.resolveNode") controller.abort();
|
||||
return result;
|
||||
});
|
||||
expect(await h.fill("hello", true, controller.signal)).toMatchObject({ code: "cancelled" });
|
||||
expect(h.element.value).toBe("old");
|
||||
expect(h.script).not.toHaveBeenCalled();
|
||||
expect(h.release).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("honors cancellation during verification", async () => {
|
||||
const h = await setup();
|
||||
const controller = new AbortController();
|
||||
h.element.addEventListener("change", () => controller.abort());
|
||||
expect(await h.fill("hello", true, controller.signal)).toMatchObject({ code: "cancelled" });
|
||||
expect(h.release).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("keeps the verified outcome when releasing the remote object fails", async () => {
|
||||
const h = await setup();
|
||||
h.release.mockRejectedValueOnce(new Error("target closed"));
|
||||
expect(await h.fill()).toMatchObject({ value_length: 5 });
|
||||
});
|
||||
});
|
||||
@@ -169,6 +169,7 @@ describe("frame geometry projection", () => {
|
||||
expect(geometry).toMatchObject({
|
||||
topBounds: { x: 100, y: 100, width: 200, height: 100 },
|
||||
actionPoint: { x: 200, y: 150 },
|
||||
targetActionPoint: { x: 100, y: 50 },
|
||||
});
|
||||
});
|
||||
|
||||
@@ -261,6 +262,7 @@ describe("frame geometry projection", () => {
|
||||
expect(geometry).toMatchObject({
|
||||
topBounds: { x: 100, y: 100, width: 200, height: 100 },
|
||||
actionPoint: { x: 200, y: 150 },
|
||||
targetActionPoint: { x: 200, y: 150 },
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -21,7 +21,7 @@ function fakeAgentWindow(ids: number[]) {
|
||||
return id;
|
||||
}),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -540,6 +540,21 @@ describe("handleHover", () => {
|
||||
});
|
||||
});
|
||||
|
||||
function successfulFillScript(params: unknown) {
|
||||
const script = params as { arguments?: Array<{ value: unknown }>; functionDeclaration: string };
|
||||
const args = script.arguments ?? [];
|
||||
return {
|
||||
result: {
|
||||
value:
|
||||
args.length === 2
|
||||
? { before: "", expected: args[0].value }
|
||||
: script.functionDeclaration.startsWith("function(expected)")
|
||||
? { connected: true, matches: true, valueLength: String(args[0].value).length }
|
||||
: "ready",
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe("handleFill", () => {
|
||||
it("returns not_found for unknown ref", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
@@ -586,7 +601,8 @@ describe("handleFill", () => {
|
||||
"DOM.scrollIntoViewIfNeeded": () => ({}),
|
||||
"DOM.focus": () => ({}),
|
||||
"DOM.resolveNode": () => ({ object: { objectId: "obj-1" } }),
|
||||
"Runtime.callFunctionOn": () => ({ result: { type: "undefined" } }),
|
||||
"Runtime.callFunctionOn": successfulFillScript,
|
||||
"Runtime.releaseObject": () => ({}),
|
||||
"Input.insertText": () => ({}),
|
||||
});
|
||||
const res = await handleFill(
|
||||
@@ -600,13 +616,12 @@ describe("handleFill", () => {
|
||||
expect(res.used_ref).toBe("e1");
|
||||
const insert = fake.sent.find((c) => c.method === "Input.insertText");
|
||||
expect(insert?.params).toEqual({ text: "hello" });
|
||||
// clear_before defaults to true → callFunctionOn invoked once to
|
||||
// clear, once to fire input/change after typing.
|
||||
// Foreground replacement needs no extra caret-positioning round trip.
|
||||
const callFns = fake.sent.filter((c) => c.method === "Runtime.callFunctionOn");
|
||||
expect(callFns.length).toBeGreaterThanOrEqual(2);
|
||||
expect(callFns).toHaveLength(4);
|
||||
});
|
||||
|
||||
it("clear_before=false skips the wipe call", async () => {
|
||||
it("passes clear_before=false to preparation and verifies the result", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.refStore.set("e1", 1, { tabId: 4 });
|
||||
@@ -618,12 +633,12 @@ describe("handleFill", () => {
|
||||
"DOM.focus": () => ({}),
|
||||
"DOM.resolveNode": () => ({ object: { objectId: "obj-2" } }),
|
||||
"Runtime.callFunctionOn": (p) => {
|
||||
const fn = (p as { functionDeclaration?: string }).functionDeclaration ?? "";
|
||||
// No "this.value = ''" clearing on a no-clear path; only the
|
||||
// post-input dispatchEvent.
|
||||
expect(fn).not.toMatch(/this\.value\s*=\s*''/);
|
||||
return { result: { type: "undefined" } };
|
||||
const args = (p as { arguments?: Array<{ value: unknown }> }).arguments ?? [];
|
||||
if (args.length === 2) expect(args[1].value).toBe(false);
|
||||
return successfulFillScript(p);
|
||||
},
|
||||
"Input.dispatchKeyEvent": () => ({}),
|
||||
"Runtime.releaseObject": () => ({}),
|
||||
"Input.insertText": () => ({}),
|
||||
});
|
||||
const res = await handleFill(
|
||||
@@ -650,7 +665,8 @@ describe("handleFill", () => {
|
||||
"DOM.scrollIntoViewIfNeeded": () => ({}),
|
||||
"DOM.focus": () => ({}),
|
||||
"DOM.resolveNode": () => ({ object: { objectId: "obj-3" } }),
|
||||
"Runtime.callFunctionOn": () => ({ result: { type: "undefined" } }),
|
||||
"Runtime.callFunctionOn": successfulFillScript,
|
||||
"Runtime.releaseObject": () => ({}),
|
||||
"Input.insertText": () => ({}),
|
||||
});
|
||||
const res = await handleFill(
|
||||
|
||||
@@ -0,0 +1,266 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { SessionManager } from "@/session-manager/manager";
|
||||
import type { BrowserNavigationApi } from "../browser-navigation";
|
||||
import { handleNavigate, handleReload } from "../navigation";
|
||||
import type { CdpRunner } from "../shared";
|
||||
|
||||
const denied = "Cannot access a chrome-extension:// URL of different extension";
|
||||
const url = "https://example.test/recovered";
|
||||
|
||||
function event<T extends (...args: never[]) => void>() {
|
||||
const listeners = new Set<T>();
|
||||
return {
|
||||
listeners,
|
||||
addListener: (listener: T) => listeners.add(listener),
|
||||
removeListener: (listener: T) => listeners.delete(listener),
|
||||
fire: (...args: Parameters<T>) => {
|
||||
for (const listener of [...listeners]) listener(...args);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function fixture() {
|
||||
const manager = new SessionManager({
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 100),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 4),
|
||||
},
|
||||
});
|
||||
await manager.start("test");
|
||||
type Listener = Parameters<BrowserNavigationApi["onCommitted"]["addListener"]>[0];
|
||||
const events = {
|
||||
onCommitted: event<Listener>(),
|
||||
onDOMContentLoaded: event<Listener>(),
|
||||
onCompleted: event<Listener>(),
|
||||
onErrorOccurred: event<Listener>(),
|
||||
};
|
||||
const cdpEvents = event<Parameters<NonNullable<CdpRunner["onEvent"]>>[0]>();
|
||||
const state = { blocked: true, destinationBlocked: false, emitIdle: true };
|
||||
const send = vi.fn(async (_tabId: number, method: string) => {
|
||||
if (state.blocked) throw new Error(denied);
|
||||
if (method === "Page.getFrameTree") {
|
||||
return { frameTree: { frame: { id: "main", loaderId: "new-loader" } } };
|
||||
}
|
||||
if (method === "Runtime.evaluate") return { result: { value: "complete" } };
|
||||
if (method === "Page.setLifecycleEventsEnabled" && state.emitIdle) {
|
||||
cdpEvents.fire({ tabId: 4 }, "Page.lifecycleEvent", {
|
||||
frameId: "main",
|
||||
loaderId: "new-loader",
|
||||
name: "networkIdle",
|
||||
});
|
||||
}
|
||||
if (method === "Page.navigate") return { frameId: "main", loaderId: "new-loader" };
|
||||
return {};
|
||||
});
|
||||
const cdp: CdpRunner = {
|
||||
send: send as CdpRunner["send"],
|
||||
onEvent: (listener) => {
|
||||
cdpEvents.addListener(listener);
|
||||
return { dispose: () => cdpEvents.removeListener(listener) };
|
||||
},
|
||||
};
|
||||
const main = { tabId: 4, frameId: 0, documentId: "new-document" };
|
||||
const navigate = async () => {
|
||||
state.blocked = state.destinationBlocked;
|
||||
events.onCommitted.fire(main);
|
||||
events.onDOMContentLoaded.fire(main);
|
||||
events.onCompleted.fire(main);
|
||||
};
|
||||
const browserNavigation = { ...events, update: vi.fn(navigate), reload: vi.fn(navigate) };
|
||||
const tab = { id: 4, windowId: 100, active: true, url } as chrome.tabs.Tab;
|
||||
const tabsApi = { get: vi.fn(async () => tab), query: vi.fn(async () => [tab]) };
|
||||
const deps = { cdp, tabsApi, browserNavigation, defaultTimeoutMs: 1000 };
|
||||
const expectCleanedUp = () => {
|
||||
for (const entry of Object.values(events)) expect(entry.listeners.size).toBe(0);
|
||||
expect(cdpEvents.listeners.size).toBe(0);
|
||||
};
|
||||
return { manager, deps, state, send, events, main, cdpEvents, expectCleanedUp };
|
||||
}
|
||||
|
||||
describe("navigation after Chrome denies extension-frame access", () => {
|
||||
it.each([
|
||||
"load",
|
||||
"commit",
|
||||
"domcontentloaded",
|
||||
"networkidle",
|
||||
] as const)("recovers the same tab with the requested %s checkpoint", async (waitUntil) => {
|
||||
const f = await fixture();
|
||||
const result = await handleNavigate(
|
||||
f.manager,
|
||||
{
|
||||
session_id: "test",
|
||||
url,
|
||||
wait_until: waitUntil,
|
||||
},
|
||||
f.deps,
|
||||
);
|
||||
|
||||
expect(result).toMatchObject({ tab_id: 4, final_url: url, reached: waitUntil });
|
||||
expect(f.deps.browserNavigation.update).toHaveBeenCalledExactlyOnceWith(4, { url });
|
||||
expect(f.send.mock.calls.some(([, method]) => method === "Page.navigate")).toBe(false);
|
||||
expect(f.send).toHaveBeenCalledWith(4, "Page.enable", {});
|
||||
f.expectCleanedUp();
|
||||
});
|
||||
|
||||
it("recovers reload through tabs.reload and preserves hard reload", async () => {
|
||||
const f = await fixture();
|
||||
const result = await handleReload(f.manager, { session_id: "test", hard: true }, f.deps);
|
||||
expect(result).toMatchObject({ tab_id: 4, reached: "load" });
|
||||
expect(f.deps.browserNavigation.reload).toHaveBeenCalledExactlyOnceWith(4, {
|
||||
bypassCache: true,
|
||||
});
|
||||
expect(f.deps.browserNavigation.update).not.toHaveBeenCalled();
|
||||
f.expectCleanedUp();
|
||||
});
|
||||
|
||||
it("reports the restriction when reload leaves the conflicting frame present", async () => {
|
||||
const f = await fixture();
|
||||
f.state.destinationBlocked = true;
|
||||
const result = await handleReload(f.manager, { session_id: "test" }, f.deps);
|
||||
expect(result).toMatchObject({
|
||||
code: "cdp_failed",
|
||||
data: { reason: "cdp_extension_access_denied" },
|
||||
});
|
||||
expect(f.deps.browserNavigation.reload).toHaveBeenCalledOnce();
|
||||
f.expectCleanedUp();
|
||||
});
|
||||
|
||||
it("does not fall back for unrelated debugger failures", async () => {
|
||||
const f = await fixture();
|
||||
f.send.mockRejectedValue(new Error("Another debugger is already attached"));
|
||||
const result = await handleNavigate(f.manager, { session_id: "test", url }, f.deps);
|
||||
expect(result).toMatchObject({
|
||||
code: "cdp_failed",
|
||||
message: "Another debugger is already attached",
|
||||
});
|
||||
expect(f.deps.browserNavigation.update).not.toHaveBeenCalled();
|
||||
f.expectCleanedUp();
|
||||
});
|
||||
|
||||
it("does not replay navigation when a dispatched CDP command fails", async () => {
|
||||
const f = await fixture();
|
||||
f.state.blocked = false;
|
||||
const send = f.send.getMockImplementation()!;
|
||||
f.send.mockImplementation(async (tabId, method) => {
|
||||
if (method === "Page.navigate") throw new Error(denied);
|
||||
return send(tabId, method);
|
||||
});
|
||||
const result = await handleNavigate(f.manager, { session_id: "test", url }, f.deps);
|
||||
expect(result).toMatchObject({
|
||||
code: "cdp_failed",
|
||||
data: { reason: "cdp_extension_access_denied" },
|
||||
});
|
||||
expect(f.deps.browserNavigation.update).not.toHaveBeenCalled();
|
||||
f.expectCleanedUp();
|
||||
});
|
||||
|
||||
it("keeps the Agent Window guard in front of the recovery", async () => {
|
||||
const f = await fixture();
|
||||
f.deps.tabsApi.get.mockResolvedValue({
|
||||
id: 4,
|
||||
windowId: 200,
|
||||
active: true,
|
||||
url,
|
||||
} as chrome.tabs.Tab);
|
||||
const result = await handleNavigate(f.manager, { session_id: "test", tab_id: 4, url }, f.deps);
|
||||
expect(result).toMatchObject({ code: "permission_denied" });
|
||||
expect(f.send).not.toHaveBeenCalled();
|
||||
expect(f.deps.browserNavigation.update).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("ignores old-document, subframe, and other-tab completion events", async () => {
|
||||
const f = await fixture();
|
||||
f.deps.browserNavigation.update.mockImplementation(async () => {});
|
||||
let settled = false;
|
||||
const work = handleNavigate(f.manager, { session_id: "test", url }, f.deps).then((result) => {
|
||||
settled = true;
|
||||
return result;
|
||||
});
|
||||
await vi.waitFor(() => expect(f.deps.browserNavigation.update).toHaveBeenCalledOnce());
|
||||
f.events.onCompleted.fire(f.main);
|
||||
f.events.onCommitted.fire({ ...f.main, tabId: 9 });
|
||||
f.events.onCommitted.fire({ ...f.main, frameId: 1 });
|
||||
f.events.onCompleted.fire(f.main);
|
||||
await Promise.resolve();
|
||||
expect(settled).toBe(false);
|
||||
f.events.onCommitted.fire(f.main);
|
||||
f.events.onCompleted.fire({ ...f.main, documentId: "old-document" });
|
||||
await Promise.resolve();
|
||||
expect(settled).toBe(false);
|
||||
f.state.blocked = false;
|
||||
f.events.onCompleted.fire(f.main);
|
||||
expect(await work).toMatchObject({ reached: "load" });
|
||||
f.expectCleanedUp();
|
||||
});
|
||||
|
||||
it("does not infer network idle from a browser load event", async () => {
|
||||
const f = await fixture();
|
||||
f.state.emitIdle = false;
|
||||
const result = await handleNavigate(
|
||||
f.manager,
|
||||
{
|
||||
session_id: "test",
|
||||
url,
|
||||
wait_until: "networkidle",
|
||||
timeout_ms: 10,
|
||||
},
|
||||
f.deps,
|
||||
);
|
||||
expect(result).toMatchObject({ reached: "timeout" });
|
||||
f.expectCleanedUp();
|
||||
});
|
||||
|
||||
it("cleans up browser listeners on timeout", async () => {
|
||||
const f = await fixture();
|
||||
f.deps.browserNavigation.update.mockImplementation(async () => {});
|
||||
const result = await handleNavigate(
|
||||
f.manager,
|
||||
{ session_id: "test", url, timeout_ms: 10 },
|
||||
f.deps,
|
||||
);
|
||||
expect(result).toMatchObject({ reached: "timeout" });
|
||||
f.expectCleanedUp();
|
||||
});
|
||||
|
||||
it("does not navigate an already-cancelled request", async () => {
|
||||
const f = await fixture();
|
||||
const result = await handleNavigate(
|
||||
f.manager,
|
||||
{ session_id: "test", url },
|
||||
{
|
||||
...f.deps,
|
||||
signal: AbortSignal.abort(),
|
||||
},
|
||||
);
|
||||
expect(result).toMatchObject({ code: "cancelled" });
|
||||
expect(f.deps.browserNavigation.update).not.toHaveBeenCalled();
|
||||
f.expectCleanedUp();
|
||||
});
|
||||
|
||||
it("cleans up browser listeners when cancelled after dispatch", async () => {
|
||||
const f = await fixture();
|
||||
const ac = new AbortController();
|
||||
f.deps.browserNavigation.update.mockImplementation(async () => {
|
||||
ac.abort();
|
||||
});
|
||||
const result = await handleNavigate(
|
||||
f.manager,
|
||||
{ session_id: "test", url },
|
||||
{ ...f.deps, signal: ac.signal },
|
||||
);
|
||||
expect(result).toMatchObject({ code: "cancelled" });
|
||||
expect(f.deps.browserNavigation.update).toHaveBeenCalledOnce();
|
||||
f.expectCleanedUp();
|
||||
});
|
||||
|
||||
it("reports browser navigation errors and releases listeners", async () => {
|
||||
const f = await fixture();
|
||||
f.deps.browserNavigation.update.mockImplementation(async () => {
|
||||
f.events.onErrorOccurred.fire({ ...f.main, error: "net::ERR_NAME_NOT_RESOLVED" });
|
||||
});
|
||||
const result = await handleNavigate(f.manager, { session_id: "test", url }, f.deps);
|
||||
expect(result).toMatchObject({ code: "cdp_failed", message: "net::ERR_NAME_NOT_RESOLVED" });
|
||||
f.expectCleanedUp();
|
||||
});
|
||||
});
|
||||
@@ -21,7 +21,7 @@ function fakeAgentWindow(ids: number[]) {
|
||||
return id;
|
||||
}),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -12,7 +12,7 @@ function fakeAgentWindow(ids: number[]) {
|
||||
return id;
|
||||
}),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -28,7 +28,7 @@ function fakeAgentWindow(ids: number[]) {
|
||||
return id;
|
||||
}),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1656,7 +1656,7 @@ describe("buildVomScene", () => {
|
||||
|
||||
const scene = buildVomScene(axNodes, captured);
|
||||
expect(scene.nodes.find((n) => n.id === 20)).toEqual(
|
||||
expect.objectContaining({ id: 20, role: "generic", cursor: "pointer" }),
|
||||
expect.objectContaining({ id: 20, role: "button", name: "close", cursor: "pointer" }),
|
||||
);
|
||||
const rendered = renderVom(scene);
|
||||
expect(rendered.text).toContain('@e1 button "close"');
|
||||
@@ -1732,7 +1732,7 @@ describe("buildVomScene", () => {
|
||||
};
|
||||
|
||||
const scene = buildVomScene(axNodes, captured);
|
||||
expect(scene.nodes.find((n) => n.id === 20)?.role).toBe("generic");
|
||||
expect(scene.nodes.find((n) => n.id === 20)).toBeUndefined();
|
||||
expect(scene.nodes.find((n) => n.id === 30)?.role).toBe("link");
|
||||
});
|
||||
|
||||
@@ -1801,9 +1801,9 @@ describe("buildVomScene", () => {
|
||||
|
||||
const scene = buildVomScene(axNodes, captured);
|
||||
expect(scene.nodes.find((n) => n.id === 20)).toEqual(
|
||||
expect.objectContaining({ id: 20, role: "generic", attrs: { "aria-label": "收藏" } }),
|
||||
expect.objectContaining({ id: 20, role: "button", attrs: { "aria-label": "收藏" } }),
|
||||
);
|
||||
expect(scene.nodes.find((n) => n.id === 30)?.role).toBe("generic");
|
||||
expect(scene.nodes.find((n) => n.id === 30)).toBeUndefined();
|
||||
const rendered = renderVom(scene);
|
||||
expect(rendered.text).toContain('@e1 button "收藏"');
|
||||
expect(rendered.refs.map(({ ref, backendNodeId }) => ({ ref, backendNodeId }))).toEqual([
|
||||
@@ -1904,36 +1904,41 @@ describe("buildVomScene", () => {
|
||||
backendDOMNodeId: 20,
|
||||
},
|
||||
];
|
||||
const scene = buildVomScene(axNodes, {
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
surfaceProbes: [
|
||||
{ triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["Shoes", "Bags"] },
|
||||
],
|
||||
nodes: [
|
||||
{
|
||||
backendNodeId: 10,
|
||||
parentBackendNodeId: null,
|
||||
tag: "body",
|
||||
attrs: {},
|
||||
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 20,
|
||||
parentBackendNodeId: 10,
|
||||
tag: "button",
|
||||
attrs: {},
|
||||
rect: { x: 20, y: 20, w: 120, h: 40 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
],
|
||||
});
|
||||
const scene = buildVomScene(
|
||||
axNodes,
|
||||
{
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
nodes: [
|
||||
{
|
||||
backendNodeId: 10,
|
||||
parentBackendNodeId: null,
|
||||
tag: "body",
|
||||
attrs: {},
|
||||
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 20,
|
||||
parentBackendNodeId: 10,
|
||||
tag: "button",
|
||||
attrs: {},
|
||||
rect: { x: 20, y: 20, w: 120, h: 40 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
surfaceProbes: [
|
||||
{ triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["Shoes", "Bags"] },
|
||||
],
|
||||
},
|
||||
);
|
||||
|
||||
expect(scene.surfaces).toEqual([
|
||||
{ triggerId: 20, triggerAction: "hover", subItems: ["Shoes", "Bags"] },
|
||||
@@ -1957,46 +1962,51 @@ describe("buildVomScene", () => {
|
||||
backendDOMNodeId: 21,
|
||||
},
|
||||
];
|
||||
const scene = buildVomScene(axNodes, {
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
surfaceProbes: [
|
||||
{ triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["My profile"] },
|
||||
],
|
||||
nodes: [
|
||||
{
|
||||
backendNodeId: 10,
|
||||
parentBackendNodeId: null,
|
||||
tag: "body",
|
||||
attrs: {},
|
||||
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 20,
|
||||
parentBackendNodeId: 10,
|
||||
tag: "div",
|
||||
attrs: { class: "tg-avatar" },
|
||||
rect: { x: 900, y: 10, w: 30, h: 30 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 21,
|
||||
parentBackendNodeId: 20,
|
||||
tag: "div",
|
||||
attrs: { class: "tg-avatar__inner" },
|
||||
rect: { x: 902, y: 12, w: 26, h: 26 },
|
||||
paintOrder: 2,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
],
|
||||
});
|
||||
const scene = buildVomScene(
|
||||
axNodes,
|
||||
{
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
nodes: [
|
||||
{
|
||||
backendNodeId: 10,
|
||||
parentBackendNodeId: null,
|
||||
tag: "body",
|
||||
attrs: {},
|
||||
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 20,
|
||||
parentBackendNodeId: 10,
|
||||
tag: "div",
|
||||
attrs: { class: "tg-avatar" },
|
||||
rect: { x: 900, y: 10, w: 30, h: 30 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 21,
|
||||
parentBackendNodeId: 20,
|
||||
tag: "div",
|
||||
attrs: { class: "tg-avatar__inner" },
|
||||
rect: { x: 902, y: 12, w: 26, h: 26 },
|
||||
paintOrder: 2,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
surfaceProbes: [
|
||||
{ triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["My profile"] },
|
||||
],
|
||||
},
|
||||
);
|
||||
|
||||
expect(renderVom(scene).text).toContain('@e1 button "image" [hover first: My profile]');
|
||||
});
|
||||
@@ -2017,48 +2027,53 @@ describe("buildVomScene", () => {
|
||||
backendDOMNodeId: 21,
|
||||
},
|
||||
];
|
||||
const scene = buildVomScene(axNodes, {
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
surfaceProbes: [
|
||||
{ triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["My profile"] },
|
||||
{ triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["Sign out"] },
|
||||
{ triggerBackendNodeId: 21, triggerAction: "hover", subItems: ["Settings"] },
|
||||
],
|
||||
nodes: [
|
||||
{
|
||||
backendNodeId: 10,
|
||||
parentBackendNodeId: null,
|
||||
tag: "body",
|
||||
attrs: {},
|
||||
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 20,
|
||||
parentBackendNodeId: 10,
|
||||
tag: "div",
|
||||
attrs: { class: "tg-avatar" },
|
||||
rect: { x: 900, y: 10, w: 30, h: 30 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 21,
|
||||
parentBackendNodeId: 20,
|
||||
tag: "div",
|
||||
attrs: { class: "tg-avatar__inner" },
|
||||
rect: { x: 902, y: 12, w: 26, h: 26 },
|
||||
paintOrder: 2,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
],
|
||||
});
|
||||
const scene = buildVomScene(
|
||||
axNodes,
|
||||
{
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
nodes: [
|
||||
{
|
||||
backendNodeId: 10,
|
||||
parentBackendNodeId: null,
|
||||
tag: "body",
|
||||
attrs: {},
|
||||
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 20,
|
||||
parentBackendNodeId: 10,
|
||||
tag: "div",
|
||||
attrs: { class: "tg-avatar" },
|
||||
rect: { x: 900, y: 10, w: 30, h: 30 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 21,
|
||||
parentBackendNodeId: 20,
|
||||
tag: "div",
|
||||
attrs: { class: "tg-avatar__inner" },
|
||||
rect: { x: 902, y: 12, w: 26, h: 26 },
|
||||
paintOrder: 2,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
surfaceProbes: [
|
||||
{ triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["My profile"] },
|
||||
{ triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["Sign out"] },
|
||||
{ triggerBackendNodeId: 21, triggerAction: "hover", subItems: ["Settings"] },
|
||||
],
|
||||
},
|
||||
);
|
||||
|
||||
expect(scene.surfaces).toEqual([
|
||||
{ triggerId: 21, triggerAction: "hover", subItems: ["My profile"] },
|
||||
@@ -2081,57 +2096,62 @@ describe("buildVomScene", () => {
|
||||
backendDOMNodeId: 21,
|
||||
},
|
||||
];
|
||||
const scene = buildVomScene(axNodes, {
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
surfaceProbes: [
|
||||
{
|
||||
triggerBackendNodeId: 20,
|
||||
triggerPoint: { x: 915, y: 25 },
|
||||
triggerAction: "hover",
|
||||
subItems: ["My profile", "Sign out"],
|
||||
},
|
||||
],
|
||||
nodes: [
|
||||
{
|
||||
backendNodeId: 10,
|
||||
parentBackendNodeId: null,
|
||||
tag: "body",
|
||||
attrs: {},
|
||||
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
cursor: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 20,
|
||||
parentBackendNodeId: 10,
|
||||
tag: "div",
|
||||
attrs: { class: "tg-avatar" },
|
||||
rect: { x: 900, y: 10, w: 30, h: 30 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
cursor: "pointer",
|
||||
},
|
||||
{
|
||||
backendNodeId: 21,
|
||||
parentBackendNodeId: 20,
|
||||
tag: "div",
|
||||
attrs: { class: "tg-avatar__inner" },
|
||||
rect: { x: 902, y: 12, w: 26, h: 26 },
|
||||
paintOrder: 2,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
cursor: "pointer",
|
||||
},
|
||||
],
|
||||
});
|
||||
const scene = buildVomScene(
|
||||
axNodes,
|
||||
{
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
nodes: [
|
||||
{
|
||||
backendNodeId: 10,
|
||||
parentBackendNodeId: null,
|
||||
tag: "body",
|
||||
attrs: {},
|
||||
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
cursor: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 20,
|
||||
parentBackendNodeId: 10,
|
||||
tag: "div",
|
||||
attrs: { class: "tg-avatar" },
|
||||
rect: { x: 900, y: 10, w: 30, h: 30 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
cursor: "pointer",
|
||||
},
|
||||
{
|
||||
backendNodeId: 21,
|
||||
parentBackendNodeId: 20,
|
||||
tag: "div",
|
||||
attrs: { class: "tg-avatar__inner" },
|
||||
rect: { x: 902, y: 12, w: 26, h: 26 },
|
||||
paintOrder: 2,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
cursor: "pointer",
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
surfaceProbes: [
|
||||
{
|
||||
triggerBackendNodeId: 20,
|
||||
triggerPoint: { x: 915, y: 25 },
|
||||
triggerAction: "hover",
|
||||
subItems: ["My profile", "Sign out"],
|
||||
},
|
||||
],
|
||||
},
|
||||
);
|
||||
|
||||
expect(scene.surfaces).toEqual([
|
||||
{ triggerId: 21, triggerAction: "hover", subItems: ["My profile", "Sign out"] },
|
||||
{ triggerId: 20, triggerAction: "hover", subItems: ["My profile", "Sign out"] },
|
||||
]);
|
||||
expect(renderVom(scene).text).toContain(
|
||||
'@e1 button "image" [hover first: My profile | Sign out]',
|
||||
@@ -2154,41 +2174,46 @@ describe("buildVomScene", () => {
|
||||
backendDOMNodeId: 42,
|
||||
},
|
||||
];
|
||||
const scene = buildVomScene(axNodes, {
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
surfaceProbes: [
|
||||
{
|
||||
triggerBackendNodeId: 999,
|
||||
triggerPoint: { x: 900, y: 20 },
|
||||
triggerAction: "hover",
|
||||
subItems: ["My profile"],
|
||||
},
|
||||
],
|
||||
nodes: [
|
||||
{
|
||||
backendNodeId: 10,
|
||||
parentBackendNodeId: null,
|
||||
tag: "body",
|
||||
attrs: {},
|
||||
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 42,
|
||||
parentBackendNodeId: 10,
|
||||
tag: "a",
|
||||
attrs: {},
|
||||
rect: { x: 860, y: 120, w: 40, h: 30 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
],
|
||||
});
|
||||
const scene = buildVomScene(
|
||||
axNodes,
|
||||
{
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
nodes: [
|
||||
{
|
||||
backendNodeId: 10,
|
||||
parentBackendNodeId: null,
|
||||
tag: "body",
|
||||
attrs: {},
|
||||
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 42,
|
||||
parentBackendNodeId: 10,
|
||||
tag: "a",
|
||||
attrs: {},
|
||||
rect: { x: 860, y: 120, w: 40, h: 30 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
surfaceProbes: [
|
||||
{
|
||||
triggerBackendNodeId: 999,
|
||||
triggerPoint: { x: 900, y: 20 },
|
||||
triggerAction: "hover",
|
||||
subItems: ["My profile"],
|
||||
},
|
||||
],
|
||||
},
|
||||
);
|
||||
|
||||
expect(scene.surfaces).toBeUndefined();
|
||||
expect(renderVom(scene).text).not.toContain("[hover first:");
|
||||
@@ -2606,7 +2631,106 @@ describe("handleSnapshot", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("enables conditional surface probing for semantic observe", async () => {
|
||||
it("hovers the page only after the accessibility tree has been captured", async () => {
|
||||
// Hovering can open menus and reflow the page. Probing between the DOM and
|
||||
// AX captures would leave the two halves of one observation describing the
|
||||
// page on either side of that change.
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
await sm.start("aa11");
|
||||
const root: CdpAxNode = {
|
||||
nodeId: "1",
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
name: { type: "computedString", value: "Example" },
|
||||
backendDOMNodeId: 100,
|
||||
childIds: ["2"],
|
||||
};
|
||||
const button: CdpAxNode = {
|
||||
nodeId: "2",
|
||||
parentId: "1",
|
||||
role: { type: "role", value: "button" },
|
||||
name: { type: "computedString", value: "Products" },
|
||||
backendDOMNodeId: 200,
|
||||
};
|
||||
const strings = ["body", "button", "position", "static", "pointer-events", "auto", "cursor"];
|
||||
const i = (s: string) => strings.indexOf(s);
|
||||
const methodOrder: string[] = [];
|
||||
const send = vi.fn(async (_tabId: number, method: string, params?: object) => {
|
||||
methodOrder.push(method);
|
||||
if (method === "Accessibility.enable") return {};
|
||||
if (method === "Accessibility.getFullAXTree") return { nodes: [root, button] };
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 } };
|
||||
}
|
||||
if (method === "DOMSnapshot.enable") return {};
|
||||
if (method === "DOMSnapshot.captureSnapshot") {
|
||||
return {
|
||||
strings,
|
||||
documents: [
|
||||
{
|
||||
nodes: {
|
||||
parentIndex: [-1, 0],
|
||||
nodeName: [i("body"), i("button")],
|
||||
backendNodeId: [100, 200],
|
||||
attributes: [[], []],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [0, 1],
|
||||
styles: [
|
||||
[i("static"), i("auto"), i("auto")],
|
||||
[i("static"), i("auto"), i("auto")],
|
||||
],
|
||||
bounds: [
|
||||
[0, 0, 1000, 800],
|
||||
[20, 20, 120, 40],
|
||||
],
|
||||
paintOrders: [0, 1],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
if (method === "Input.dispatchMouseEvent") return {};
|
||||
if (method === "Runtime.evaluate") {
|
||||
const expression = (params as { expression?: string } | undefined)?.expression ?? "";
|
||||
// Report the button as a `:hover` rule target so a real hover fires.
|
||||
if (expression.includes("document.styleSheets")) {
|
||||
return { result: { value: [{ x: 80, y: 40 }] } };
|
||||
}
|
||||
return { result: { value: [] } };
|
||||
}
|
||||
throw new Error(`unexpected CDP method ${method}`);
|
||||
});
|
||||
|
||||
const res = await handleObserve(
|
||||
sm,
|
||||
{ session_id: "aa11", probe_hover: true },
|
||||
{
|
||||
cdp: {
|
||||
send: send as unknown as <T = unknown>(
|
||||
tabId: number,
|
||||
method: string,
|
||||
params?: object,
|
||||
) => Promise<T>,
|
||||
trackSessionTab: vi.fn(),
|
||||
},
|
||||
tabsApi: {
|
||||
get: vi.fn(
|
||||
async (tabId: number) =>
|
||||
({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab,
|
||||
),
|
||||
query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
const firstHover = methodOrder.indexOf("Input.dispatchMouseEvent");
|
||||
expect(firstHover).toBeGreaterThan(-1);
|
||||
expect(methodOrder.lastIndexOf("DOMSnapshot.captureSnapshot")).toBeLessThan(firstHover);
|
||||
expect(methodOrder.lastIndexOf("Accessibility.getFullAXTree")).toBeLessThan(firstHover);
|
||||
});
|
||||
|
||||
it("runs conditional surface probing only when observe opts in", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
await sm.start("aa11");
|
||||
const root: CdpAxNode = {
|
||||
@@ -2662,30 +2786,46 @@ describe("handleSnapshot", () => {
|
||||
if (method === "Runtime.evaluate") return { result: { value: [] } };
|
||||
throw new Error(`unexpected CDP method ${method}`);
|
||||
});
|
||||
const res = await handleObserve(
|
||||
const deps = {
|
||||
cdp: {
|
||||
send: send as unknown as <T = unknown>(
|
||||
tabId: number,
|
||||
method: string,
|
||||
params?: object,
|
||||
) => Promise<T>,
|
||||
trackSessionTab: vi.fn(),
|
||||
},
|
||||
tabsApi: {
|
||||
get: vi.fn(
|
||||
async (tabId: number) => ({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab,
|
||||
),
|
||||
query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
|
||||
},
|
||||
};
|
||||
|
||||
const withoutProbe = await handleObserve(
|
||||
sm,
|
||||
{ session_id: "aa11", debug_surfaces: true },
|
||||
{
|
||||
cdp: {
|
||||
send: send as unknown as <T = unknown>(
|
||||
tabId: number,
|
||||
method: string,
|
||||
params?: object,
|
||||
) => Promise<T>,
|
||||
trackSessionTab: vi.fn(),
|
||||
},
|
||||
tabsApi: {
|
||||
get: vi.fn(
|
||||
async (tabId: number) =>
|
||||
({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab,
|
||||
),
|
||||
query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
|
||||
},
|
||||
},
|
||||
deps,
|
||||
);
|
||||
if ("code" in withoutProbe) {
|
||||
throw new Error(`unexpected error: ${JSON.stringify(withoutProbe)}`);
|
||||
}
|
||||
expect(withoutProbe.hover_probe).toBeUndefined();
|
||||
expect(send).not.toHaveBeenCalledWith(
|
||||
4,
|
||||
"Runtime.evaluate",
|
||||
expect.objectContaining({ returnByValue: true }),
|
||||
);
|
||||
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
expect(res.debug).toEqual({ surface_probes: [] });
|
||||
const withProbe = await handleObserve(
|
||||
sm,
|
||||
{ session_id: "aa11", debug_surfaces: true, probe_hover: true },
|
||||
deps,
|
||||
);
|
||||
if ("code" in withProbe) throw new Error(`unexpected error: ${JSON.stringify(withProbe)}`);
|
||||
expect(withProbe.debug).toEqual({ surface_probes: [] });
|
||||
expect(withProbe.hover_probe).toEqual({ performed: true, revealed_content: false });
|
||||
expect(send).toHaveBeenCalledWith(
|
||||
4,
|
||||
"Runtime.evaluate",
|
||||
@@ -3155,6 +3295,98 @@ describe("handleSnapshot", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("decorates live observations with active controlled content", async () => {
|
||||
const strings = [
|
||||
"html",
|
||||
"body",
|
||||
"button",
|
||||
"div",
|
||||
"#text",
|
||||
"role",
|
||||
"tab",
|
||||
"aria-selected",
|
||||
"true",
|
||||
"aria-controls",
|
||||
"reviews-panel",
|
||||
"id",
|
||||
"static",
|
||||
"auto",
|
||||
"visible",
|
||||
"1",
|
||||
"A detailed review",
|
||||
];
|
||||
const index = (value: string) => strings.indexOf(value);
|
||||
const style = [index("static"), index("auto"), index("auto"), index("visible"), index("1")];
|
||||
const snapshot = {
|
||||
strings,
|
||||
documents: [
|
||||
{
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1, 1, 3],
|
||||
nodeName: [index("html"), index("body"), index("button"), index("div"), index("#text")],
|
||||
backendNodeId: [10, 11, 12, 13, 14],
|
||||
attributes: [
|
||||
[],
|
||||
[],
|
||||
[
|
||||
index("role"),
|
||||
index("tab"),
|
||||
index("aria-selected"),
|
||||
index("true"),
|
||||
index("aria-controls"),
|
||||
index("reviews-panel"),
|
||||
],
|
||||
[index("id"), index("reviews-panel")],
|
||||
[],
|
||||
],
|
||||
nodeValue: [-1, -1, -1, -1, index("A detailed review")],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [0, 1, 2, 3, 4],
|
||||
styles: [style, style, style, style, style],
|
||||
bounds: [
|
||||
[0, 0, 1000, 800],
|
||||
[0, 0, 1000, 800],
|
||||
[20, 20, 120, 40],
|
||||
[20, 80, 500, 200],
|
||||
[20, 80, 200, 20],
|
||||
],
|
||||
paintOrders: [0, 0, 1, 1, 1],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
const ax: CdpAxNode[] = [
|
||||
{
|
||||
nodeId: "root",
|
||||
backendDOMNodeId: 11,
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
childIds: ["reviews"],
|
||||
},
|
||||
{
|
||||
nodeId: "reviews",
|
||||
parentId: "root",
|
||||
backendDOMNodeId: 12,
|
||||
role: { type: "role", value: "tab" },
|
||||
name: { type: "computedString", value: "Reviews" },
|
||||
properties: [
|
||||
{ name: "selected", value: { value: true } },
|
||||
{ name: "controls", value: { value: "reviews-panel" } },
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
const result = await captureVomObservation(
|
||||
makeOverlayDeps(ax, snapshot, VP_METRICS).cdp,
|
||||
4,
|
||||
"https://example.com",
|
||||
);
|
||||
|
||||
expect(result.text).toContain('@e1 tab "Reviews"');
|
||||
expect(result.text).toContain("[§ active: Reviews]");
|
||||
expect(result.text).toContain("A detailed review");
|
||||
});
|
||||
|
||||
it("does not leak form secrets through the record-safe observation payload", async () => {
|
||||
const secrets = {
|
||||
email: "user@example.com",
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,8 +1,26 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { SessionManager } from "@/session-manager/manager";
|
||||
import { handleSessionStop } from "../session";
|
||||
import type { ChromeTabsApi } from "../shared";
|
||||
import { type AgentOverlayResetApi, type ChromeWindowsApi, type TabMutationApi } from "../tabs";
|
||||
|
||||
/** Build a read-only query api over a FakeState. */
|
||||
function makeQuery(state: FakeState): ChromeTabsApi {
|
||||
return {
|
||||
get: vi.fn(async (id) => {
|
||||
const t = state.tabs.get(id);
|
||||
if (!t) throw new Error(`tab ${id} not found`);
|
||||
return t;
|
||||
}),
|
||||
query: vi.fn(async (q: chrome.tabs.QueryInfo) => {
|
||||
const w = q.windowId;
|
||||
return Array.from(state.tabs.values()).filter(
|
||||
(t) => typeof w !== "number" || t.windowId === w,
|
||||
);
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
function fakeAgentWindow(ids: number[]) {
|
||||
let i = 0;
|
||||
const create = vi.fn(async () => {
|
||||
@@ -11,7 +29,7 @@ function fakeAgentWindow(ids: number[]) {
|
||||
return id;
|
||||
});
|
||||
const remove = vi.fn(async () => {});
|
||||
const ensureActiveTab = vi.fn(async () => {});
|
||||
const ensureActiveTab = vi.fn(async () => 0);
|
||||
return { create, remove, ensureActiveTab };
|
||||
}
|
||||
|
||||
@@ -30,7 +48,9 @@ function makeApis(
|
||||
} {
|
||||
const tabs: TabMutationApi = {
|
||||
create: vi.fn(),
|
||||
remove: vi.fn(async () => {}),
|
||||
remove: vi.fn(async (id: number) => {
|
||||
state.tabs.delete(id);
|
||||
}),
|
||||
update: vi.fn(async (_id, _p) => undefined),
|
||||
get: vi.fn(async (id) => {
|
||||
const t = state.tabs.get(id);
|
||||
@@ -237,11 +257,15 @@ describe("handleSessionStop with auto-return", () => {
|
||||
moves: [],
|
||||
};
|
||||
const { tabs, windows } = makeApis(state, { moveThrowsFor: new Set([1]) });
|
||||
const cdp = {
|
||||
detachSession: vi.fn(async () => {}),
|
||||
releaseSessionTab: vi.fn(async () => {}),
|
||||
};
|
||||
|
||||
const res = await handleSessionStop(
|
||||
sm,
|
||||
{ session_id: "aa11" },
|
||||
{ tabManagement: { tabs, windows } },
|
||||
{ cdp, tabManagement: { tabs, windows } },
|
||||
);
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
expect(res.return_failures?.map((f) => f.tab_id)).toEqual([1]);
|
||||
@@ -249,6 +273,8 @@ describe("handleSessionStop with auto-return", () => {
|
||||
expect(sm.has("aa11")).toBe(true);
|
||||
expect(ctx.borrowedTabs.has(1)).toBe(true);
|
||||
expect(ctx.borrowedTabs.has(2)).toBe(false);
|
||||
expect(cdp.releaseSessionTab).toHaveBeenCalledExactlyOnceWith("aa11", 2);
|
||||
expect(cdp.detachSession).not.toHaveBeenCalled();
|
||||
expect(aw.remove).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
@@ -267,4 +293,275 @@ describe("handleSessionStop with auto-return", () => {
|
||||
await handleSessionStop(sm, { session_id: "aa11" }, { tabManagement: { tabs, windows } });
|
||||
expect(ctx.refStore.isEmpty()).toBe(true);
|
||||
});
|
||||
|
||||
it.each([
|
||||
"before teardown",
|
||||
"during its return",
|
||||
"after its return failed",
|
||||
"before its turn in the return loop",
|
||||
])("finishes stopping when a borrowed tab closes %s", async (timing) => {
|
||||
const aw = fakeAgentWindow([100]);
|
||||
const sm = new SessionManager({ agentWindow: aw });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.borrowedTabs.set(1, { tabId: 1, originalWindowId: 200, originalIndex: 0 });
|
||||
ctx.borrowedTabs.set(2, { tabId: 2, originalWindowId: 200, originalIndex: 1 });
|
||||
const state: FakeState = {
|
||||
tabs: new Map([
|
||||
[1, { id: 1, windowId: 100 } as chrome.tabs.Tab],
|
||||
[2, { id: 2, windowId: 100 } as chrome.tabs.Tab],
|
||||
]),
|
||||
windowsClosed: new Set(),
|
||||
moves: [],
|
||||
};
|
||||
const { tabs, windows } = makeApis(state);
|
||||
const cdp = { detachSession: vi.fn(async () => {}) };
|
||||
const closeTab = (tabId: number) => {
|
||||
state.tabs.delete(tabId);
|
||||
sm.forgetClosedTab(tabId);
|
||||
};
|
||||
const baseMove = tabs.move;
|
||||
tabs.move = vi.fn(async (tabId, props) => {
|
||||
if (tabId === 1 && timing === "during its return") closeTab(1);
|
||||
if (tabId === 1 && timing === "after its return failed") {
|
||||
throw new Error("simulated move failure");
|
||||
}
|
||||
if (tabId === 2 && timing === "after its return failed") closeTab(1);
|
||||
if (tabId === 1 && timing === "before its turn in the return loop") closeTab(2);
|
||||
if (!state.tabs.has(tabId)) throw new Error(`tab ${tabId} not found`);
|
||||
return baseMove(tabId, props);
|
||||
});
|
||||
if (timing === "before teardown") closeTab(1);
|
||||
|
||||
const res = await handleSessionStop(
|
||||
sm,
|
||||
{ session_id: "aa11" },
|
||||
{
|
||||
cdp,
|
||||
tabManagement: {
|
||||
tabs,
|
||||
windows,
|
||||
agentOverlayReset: { resetAgentOverlays: vi.fn(async () => {}) },
|
||||
},
|
||||
tabsQuery: makeQuery(state),
|
||||
},
|
||||
);
|
||||
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
const closedTabId = timing === "before its turn in the return loop" ? 2 : 1;
|
||||
expect(res.returned_tab_ids).toEqual([closedTabId === 1 ? 2 : 1]);
|
||||
expect(res.return_failures).toBeUndefined();
|
||||
expect(ctx.borrowedTabs.size).toBe(0);
|
||||
expect(sm.has("aa11")).toBe(false);
|
||||
expect(cdp.detachSession).toHaveBeenCalledWith("aa11");
|
||||
expect(aw.remove).toHaveBeenCalledWith(100);
|
||||
if (timing === "before teardown" || timing === "before its turn in the return loop") {
|
||||
expect(tabs.move).not.toHaveBeenCalledWith(closedTabId, expect.anything());
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("handleSessionStop window release (issue #57)", () => {
|
||||
const agentWindowId = 100;
|
||||
|
||||
it("releases the window when user-created tabs remain", async () => {
|
||||
const aw = fakeAgentWindow([agentWindowId]);
|
||||
// ensureActiveTab returns the home tab id (10). We override the fake to
|
||||
// return a known id.
|
||||
aw.ensureActiveTab = vi.fn(async () => 10);
|
||||
const sm = new SessionManager({ agentWindow: aw });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.agentCreatedTabs.add(11);
|
||||
ctx.agentCreatedTabs.add(12);
|
||||
// user tab 99 (created via Chrome UI, NOT in agentCreatedTabs)
|
||||
const state: FakeState = {
|
||||
tabs: new Map([
|
||||
[10, { id: 10, windowId: agentWindowId } as chrome.tabs.Tab],
|
||||
[11, { id: 11, windowId: agentWindowId } as chrome.tabs.Tab],
|
||||
[12, { id: 12, windowId: agentWindowId } as chrome.tabs.Tab],
|
||||
[99, { id: 99, windowId: agentWindowId } as chrome.tabs.Tab],
|
||||
]),
|
||||
windowsClosed: new Set(),
|
||||
moves: [],
|
||||
};
|
||||
const { tabs, windows } = makeApis(state);
|
||||
const query = makeQuery(state);
|
||||
|
||||
const res = await handleSessionStop(
|
||||
sm,
|
||||
{ session_id: "aa11" },
|
||||
{ tabManagement: { tabs, windows }, tabsQuery: query },
|
||||
);
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
expect(res.window_released).toBe(true);
|
||||
// agent tabs + home removed, user tab 99 kept
|
||||
expect((tabs.remove as ReturnType<typeof vi.fn>).mock.calls.map((c) => c[0]).sort()).toEqual([
|
||||
10, 11, 12,
|
||||
]);
|
||||
expect(state.tabs.has(99)).toBe(true);
|
||||
// window released (dropOnly), not closed
|
||||
expect(aw.remove).not.toHaveBeenCalled();
|
||||
expect(sm.has("aa11")).toBe(false);
|
||||
});
|
||||
|
||||
it("closes the window (not release) when no user tabs remain", async () => {
|
||||
const aw = fakeAgentWindow([agentWindowId]);
|
||||
aw.ensureActiveTab = vi.fn(async () => 10);
|
||||
const sm = new SessionManager({ agentWindow: aw });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.agentCreatedTabs.add(11);
|
||||
const state: FakeState = {
|
||||
tabs: new Map([
|
||||
[10, { id: 10, windowId: agentWindowId } as chrome.tabs.Tab],
|
||||
[11, { id: 11, windowId: agentWindowId } as chrome.tabs.Tab],
|
||||
]),
|
||||
windowsClosed: new Set(),
|
||||
moves: [],
|
||||
};
|
||||
const { tabs, windows } = makeApis(state);
|
||||
const query = makeQuery(state);
|
||||
|
||||
const res = await handleSessionStop(
|
||||
sm,
|
||||
{ session_id: "aa11" },
|
||||
{ tabManagement: { tabs, windows }, tabsQuery: query },
|
||||
);
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
expect(res.window_released).toBeFalsy();
|
||||
expect(aw.remove).toHaveBeenCalledWith(agentWindowId);
|
||||
expect(sm.has("aa11")).toBe(false);
|
||||
});
|
||||
|
||||
it("is non-fatal when an agent tab is already gone", async () => {
|
||||
const aw = fakeAgentWindow([agentWindowId]);
|
||||
aw.ensureActiveTab = vi.fn(async () => 10);
|
||||
const sm = new SessionManager({ agentWindow: aw });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.agentCreatedTabs.add(11); // no longer in state → remove throws
|
||||
const state: FakeState = {
|
||||
tabs: new Map([[10, { id: 10, windowId: agentWindowId } as chrome.tabs.Tab]]),
|
||||
windowsClosed: new Set(),
|
||||
moves: [],
|
||||
};
|
||||
const { tabs, windows } = makeApis(state);
|
||||
(tabs.remove as ReturnType<typeof vi.fn>).mockImplementation(async (id: number) => {
|
||||
if (id === 11) throw new Error("tab 11 already closed");
|
||||
state.tabs.delete(id); // home (10) removed normally
|
||||
});
|
||||
const query = makeQuery(state);
|
||||
|
||||
const res = await handleSessionStop(
|
||||
sm,
|
||||
{ session_id: "aa11" },
|
||||
{ tabManagement: { tabs, windows }, tabsQuery: query },
|
||||
);
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
// user tab 10? no — 10 is home, removed. window would be empty → close.
|
||||
expect(res.window_released).toBeFalsy();
|
||||
expect(aw.remove).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("degrades to close-window when tabsQuery is missing", async () => {
|
||||
const aw = fakeAgentWindow([agentWindowId]);
|
||||
aw.ensureActiveTab = vi.fn(async () => 10);
|
||||
const sm = new SessionManager({ agentWindow: aw });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.agentCreatedTabs.add(11);
|
||||
const state: FakeState = {
|
||||
tabs: new Map([
|
||||
[10, { id: 10, windowId: agentWindowId } as chrome.tabs.Tab],
|
||||
[11, { id: 11, windowId: agentWindowId } as chrome.tabs.Tab],
|
||||
]),
|
||||
windowsClosed: new Set(),
|
||||
moves: [],
|
||||
};
|
||||
const { tabs, windows } = makeApis(state);
|
||||
|
||||
const res = await handleSessionStop(
|
||||
sm,
|
||||
{ session_id: "aa11" },
|
||||
{ tabManagement: { tabs, windows } }, // no tabsQuery
|
||||
);
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
// No query available → conservatively close the window.
|
||||
expect(res.window_released).toBeFalsy();
|
||||
expect(aw.remove).toHaveBeenCalledWith(agentWindowId);
|
||||
});
|
||||
|
||||
it("degrades to close-window when tabsQuery.query throws", async () => {
|
||||
const aw = fakeAgentWindow([agentWindowId]);
|
||||
aw.ensureActiveTab = vi.fn(async () => 10);
|
||||
const sm = new SessionManager({ agentWindow: aw });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.agentCreatedTabs.add(11);
|
||||
const state: FakeState = {
|
||||
tabs: new Map([[11, { id: 11, windowId: agentWindowId } as chrome.tabs.Tab]]),
|
||||
windowsClosed: new Set(),
|
||||
moves: [],
|
||||
};
|
||||
const { tabs, windows } = makeApis(state);
|
||||
const query = makeQuery(state);
|
||||
(query.query as ReturnType<typeof vi.fn>).mockImplementation(async () => {
|
||||
throw new Error("query failed");
|
||||
});
|
||||
|
||||
const res = await handleSessionStop(
|
||||
sm,
|
||||
{ session_id: "aa11" },
|
||||
{ tabManagement: { tabs, windows }, tabsQuery: query },
|
||||
);
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
expect(res.window_released).toBeFalsy();
|
||||
expect(aw.remove).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("tracks agentCreatedTabs across tab_create / tab_close", async () => {
|
||||
const aw = fakeAgentWindow([agentWindowId]);
|
||||
aw.ensureActiveTab = vi.fn(async () => 10);
|
||||
const sm = new SessionManager({ agentWindow: aw });
|
||||
const ctx = await sm.start("aa11");
|
||||
// Simulate handleTabCreate adding, then handleTabClose removing.
|
||||
ctx.agentCreatedTabs.add(11);
|
||||
expect(ctx.agentCreatedTabs.has(11)).toBe(true);
|
||||
ctx.agentCreatedTabs.delete(11); // as handleTabClose does on success
|
||||
expect(ctx.agentCreatedTabs.has(11)).toBe(false);
|
||||
// Unknown id delete is a safe no-op (user-closed tab).
|
||||
expect(() => ctx.agentCreatedTabs.delete(999)).not.toThrow();
|
||||
});
|
||||
|
||||
it("closes the window (not release) when an agent tab fails to close", async () => {
|
||||
// Regression: a leaked agent tab (still in agentCreatedTabs, but
|
||||
// still present because Step 4 remove() threw) must NOT be mistaken
|
||||
// for a user tab. Otherwise the window would be released (dropOnly)
|
||||
// and the agent tab would leak (issue #57 regression).
|
||||
const aw = fakeAgentWindow([agentWindowId]);
|
||||
aw.ensureActiveTab = vi.fn(async () => 10);
|
||||
const sm = new SessionManager({ agentWindow: aw });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.agentCreatedTabs.add(11); // Step 4 remove() will throw for this tab
|
||||
const state: FakeState = {
|
||||
tabs: new Map([
|
||||
[10, { id: 10, windowId: agentWindowId } as chrome.tabs.Tab],
|
||||
[11, { id: 11, windowId: agentWindowId } as chrome.tabs.Tab],
|
||||
]),
|
||||
windowsClosed: new Set(),
|
||||
moves: [],
|
||||
};
|
||||
const { tabs, windows } = makeApis(state);
|
||||
(tabs.remove as ReturnType<typeof vi.fn>).mockImplementation(async (id: number) => {
|
||||
if (id === 11) throw new Error("tab 11 failed to close");
|
||||
state.tabs.delete(id); // home (10) removed normally; 11 lingers
|
||||
});
|
||||
const query = makeQuery(state);
|
||||
|
||||
const res = await handleSessionStop(
|
||||
sm,
|
||||
{ session_id: "aa11" },
|
||||
{ tabManagement: { tabs, windows }, tabsQuery: query },
|
||||
);
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
// The leaked agent tab must NOT keep the window open.
|
||||
expect(res.window_released).toBeFalsy();
|
||||
expect(aw.remove).toHaveBeenCalledWith(agentWindowId);
|
||||
expect(sm.has("aa11")).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -12,7 +12,7 @@ function fakeAgentWindow() {
|
||||
return {
|
||||
create: vi.fn(async () => 100),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -11,7 +11,7 @@ function fakeAgentWindow(ids: number[]) {
|
||||
return id;
|
||||
},
|
||||
remove: async () => {},
|
||||
ensureActiveTab: async () => {},
|
||||
ensureActiveTab: async () => 1,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { type CdpDebuggerApi, ChromiumCdp } from "@/browser-driver/chromium-cdp";
|
||||
import { SessionManager } from "@/session-manager/manager";
|
||||
import { handleHover } from "../interaction";
|
||||
import {
|
||||
type AgentOverlayResetApi,
|
||||
type ChromeWindowsApi,
|
||||
@@ -23,7 +25,7 @@ function fakeAgentWindow(ids: number[]) {
|
||||
return id;
|
||||
}),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -221,6 +223,7 @@ describe("handleTabCreate", () => {
|
||||
active: true,
|
||||
});
|
||||
expect(res).toMatchObject({ tab_id: 10, window_id: 100 });
|
||||
expect(sm.get("aa11")?.agentCreatedTabs.has(10)).toBe(true);
|
||||
});
|
||||
|
||||
it("forwards explicit url + index + active=false", async () => {
|
||||
@@ -256,6 +259,41 @@ describe("handleTabCreate", () => {
|
||||
const res = await handleTabCreate(sm, { session_id: "aa11", index: -1 }, { tabs: api });
|
||||
expect(res).toMatchObject({ code: "invalid_params" });
|
||||
});
|
||||
|
||||
it("does not claim any tab when chrome.tabs.create fails", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
await sm.start("aa11");
|
||||
const state: FakeTabState = { tabs: new Map(), nextTabId: 10, windowsClosed: new Set() };
|
||||
const { api, spies } = makeTabMutationApi(state);
|
||||
spies.create.mockRejectedValueOnce(new Error("chrome.tabs.create failed"));
|
||||
|
||||
const res = await handleTabCreate(sm, { session_id: "aa11" }, { tabs: api });
|
||||
expect(res).toMatchObject({ code: "protocol_error" });
|
||||
|
||||
expect(sm.get("aa11")?.agentCreatedTabs).toEqual(new Set([1]));
|
||||
});
|
||||
|
||||
it("claims the returned id even when a user tab appears during tab creation", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
const state: FakeTabState = { tabs: new Map(), nextTabId: 10, windowsClosed: new Set() };
|
||||
const { api, spies } = makeTabMutationApi(state);
|
||||
let resolveCreate!: (tab: chrome.tabs.Tab) => void;
|
||||
const createResult = new Promise<chrome.tabs.Tab>((resolve) => {
|
||||
resolveCreate = resolve;
|
||||
});
|
||||
spies.create.mockImplementationOnce(() => createResult);
|
||||
|
||||
const pending = handleTabCreate(sm, { session_id: "aa11" }, { tabs: api });
|
||||
// A user can create an unrelated same-window tab while Chrome is still
|
||||
// resolving the agent request. There is no pending slot for it to steal.
|
||||
expect(ctx.agentCreatedTabs.has(99)).toBe(false);
|
||||
resolveCreate({ id: 10, windowId: 100, active: true } as chrome.tabs.Tab);
|
||||
|
||||
await expect(pending).resolves.toMatchObject({ tab_id: 10 });
|
||||
expect(ctx.agentCreatedTabs.has(10)).toBe(true);
|
||||
expect(ctx.agentCreatedTabs.has(99)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("handleTabClose", () => {
|
||||
@@ -511,6 +549,148 @@ describe("handleTabBorrow", () => {
|
||||
});
|
||||
|
||||
describe("handleTabReturn", () => {
|
||||
it.each([
|
||||
"not_found",
|
||||
"cdp_failed",
|
||||
])("detaches after selector resolution fails with %s", async (code) => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.borrowedTabs.set(7, { tabId: 7, originalWindowId: 200, originalIndex: 4 });
|
||||
const state: FakeTabState = {
|
||||
tabs: new Map([[7, { id: 7, windowId: 100 } as chrome.tabs.Tab]]),
|
||||
nextTabId: 50,
|
||||
windowsClosed: new Set(),
|
||||
};
|
||||
const { api } = makeTabMutationApi(state);
|
||||
const { api: windows } = makeWindowsApi(state);
|
||||
const debuggerApi = {
|
||||
attach: vi.fn(async () => {}),
|
||||
detach: vi.fn(async () => {}),
|
||||
sendCommand: vi.fn(async (_target: unknown, method: string) => {
|
||||
if (method === "DOM.getDocument") return { root: { nodeId: 1 } };
|
||||
if (method === "DOM.querySelector") {
|
||||
if (code === "cdp_failed") throw new Error("selector query failed");
|
||||
return { nodeId: 0 };
|
||||
}
|
||||
return {};
|
||||
}),
|
||||
onEvent: { addListener: vi.fn(), removeListener: vi.fn() },
|
||||
onDetach: { addListener: vi.fn(), removeListener: vi.fn() },
|
||||
};
|
||||
const cdp = new ChromiumCdp(debuggerApi as unknown as CdpDebuggerApi);
|
||||
|
||||
const hover = await handleHover(
|
||||
sm,
|
||||
{ session_id: "aa11", tab_id: 7, selector: "#missing" },
|
||||
{
|
||||
cdp,
|
||||
tabsApi: { get: api.get, query: vi.fn(async () => []) },
|
||||
},
|
||||
);
|
||||
expect(hover).toMatchObject({ code });
|
||||
expect(cdp.isAttached(7)).toBe(true);
|
||||
|
||||
const returned = await handleTabReturn(
|
||||
sm,
|
||||
{ session_id: "aa11", tab_id: 7 },
|
||||
{
|
||||
tabs: api,
|
||||
windows,
|
||||
cdp,
|
||||
agentOverlayReset: { resetAgentOverlays: vi.fn(async () => {}) },
|
||||
},
|
||||
);
|
||||
expect(returned).toMatchObject({ returned_to_window_id: 200 });
|
||||
expect(ctx.borrowedTabs.has(7)).toBe(false);
|
||||
expect(cdp.isAttached(7)).toBe(false);
|
||||
expect(debuggerApi.detach).toHaveBeenCalledExactlyOnceWith({ tabId: 7 });
|
||||
cdp.dispose();
|
||||
});
|
||||
|
||||
it("honours cancellation while preparing a validated return", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.borrowedTabs.set(7, { tabId: 7, originalWindowId: 200, originalIndex: 4 });
|
||||
const state: FakeTabState = {
|
||||
tabs: new Map([[7, { id: 7, windowId: 100 } as chrome.tabs.Tab]]),
|
||||
nextTabId: 50,
|
||||
windowsClosed: new Set(),
|
||||
};
|
||||
const { api, spies } = makeTabMutationApi(state);
|
||||
const { api: windows } = makeWindowsApi(state);
|
||||
const controller = new AbortController();
|
||||
const beforeReturn = vi.fn(async () => {
|
||||
controller.abort();
|
||||
});
|
||||
const result = await handleTabReturn(
|
||||
sm,
|
||||
{ session_id: "aa11", tab_id: 7 },
|
||||
{
|
||||
tabs: api,
|
||||
windows,
|
||||
signal: controller.signal,
|
||||
beforeReturn,
|
||||
},
|
||||
);
|
||||
|
||||
expect(result).toMatchObject({ code: "cancelled" });
|
||||
expect(beforeReturn).toHaveBeenCalledExactlyOnceWith("aa11", 7);
|
||||
expect(spies.move).not.toHaveBeenCalled();
|
||||
expect(ctx.borrowedTabs.has(7)).toBe(true);
|
||||
});
|
||||
|
||||
it.each([
|
||||
"original",
|
||||
"fallback",
|
||||
"cleanup_failed",
|
||||
"failed",
|
||||
"cancelled",
|
||||
])("releases CDP only after a successful return (%s)", async (mode) => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.borrowedTabs.set(7, { tabId: 7, originalWindowId: 200, originalIndex: 4 });
|
||||
const state: FakeTabState = {
|
||||
tabs: new Map([[7, { id: 7, windowId: 100 } as chrome.tabs.Tab]]),
|
||||
nextTabId: 50,
|
||||
windowsClosed: new Set(),
|
||||
};
|
||||
const { api, spies } = makeTabMutationApi(state);
|
||||
const { api: windowsApi } = makeWindowsApi(state, { lastFocused: 500 });
|
||||
const success = mode !== "failed" && mode !== "cancelled";
|
||||
if (mode === "fallback") spies.move.mockRejectedValueOnce(new Error("original move failed"));
|
||||
if (mode === "failed") spies.move.mockRejectedValue(new Error("move failed"));
|
||||
let windowAtRelease: number | undefined;
|
||||
const cdp = {
|
||||
releaseSessionTab: vi.fn(async () => {
|
||||
windowAtRelease = state.tabs.get(7)?.windowId;
|
||||
if (mode === "cleanup_failed") throw new Error("debugger release failed");
|
||||
}),
|
||||
};
|
||||
const controller = new AbortController();
|
||||
if (mode === "cancelled") controller.abort();
|
||||
|
||||
const result = await handleTabReturn(
|
||||
sm,
|
||||
{ session_id: "aa11", tab_id: 7 },
|
||||
{
|
||||
tabs: api,
|
||||
windows: windowsApi,
|
||||
cdp,
|
||||
signal: controller.signal,
|
||||
agentOverlayReset: { resetAgentOverlays: vi.fn(async () => {}) },
|
||||
},
|
||||
);
|
||||
|
||||
expect("code" in result).toBe(!success);
|
||||
expect(ctx.borrowedTabs.has(7)).toBe(!success);
|
||||
expect(cdp.releaseSessionTab).toHaveBeenCalledTimes(success ? 1 : 0);
|
||||
if (success) {
|
||||
expect(cdp.releaseSessionTab).toHaveBeenCalledWith("aa11", 7);
|
||||
expect(windowAtRelease).toBe(mode === "fallback" ? 500 : 200);
|
||||
expect(spies.move).toHaveBeenCalledTimes(mode === "fallback" ? 2 : 1);
|
||||
}
|
||||
});
|
||||
|
||||
it("returns the tab to its original window/index and clears borrowedTabs", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
|
||||
@@ -12,7 +12,7 @@ function fakeAgentWindow(ids: number[]) {
|
||||
return id;
|
||||
}),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => 1),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -11,7 +11,7 @@ function fakeAgentWindow(ids: number[]) {
|
||||
return id;
|
||||
});
|
||||
const remove = vi.fn(async () => {});
|
||||
const ensureActiveTab = vi.fn(async () => {});
|
||||
const ensureActiveTab = vi.fn(async () => 1);
|
||||
return { create, remove, ensureActiveTab };
|
||||
}
|
||||
|
||||
|
||||
@@ -24,7 +24,7 @@ export interface BorrowResponseMessage {
|
||||
allowed: boolean;
|
||||
}
|
||||
|
||||
export const CONFIRMATION_TIMEOUT_MS = 5000;
|
||||
export const CONFIRMATION_TIMEOUT_MS = 60_000;
|
||||
const EXIT_ANIMATION_MS = 150;
|
||||
/** Matches BorrowConfirmationOverlay progress ring/bar transition (duration-1000). */
|
||||
export const PROGRESS_TRANSITION_MS = 1000;
|
||||
@@ -480,6 +480,13 @@ export async function requestBorrowConfirmation(
|
||||
settle(false);
|
||||
}, BACKGROUND_TIMEOUT_MS);
|
||||
|
||||
// Bring the user window to the front so the overlay is visible even
|
||||
// when the Agent Window has stolen focus. This is fire-and-forget: if
|
||||
// it fails the overlay / OS notification still give the user a path.
|
||||
void windowsApi.update(notificationAnchor.windowId, { focused: true }).catch((err) => {
|
||||
console.debug("[bsk borrow] proactive focus of user window failed", err);
|
||||
});
|
||||
|
||||
// Surface the OS notification *before* messaging any candidate so the
|
||||
// user has a parallel signal even if every content script is missing.
|
||||
// The Allow / Deny buttons let the user authorize explicitly when the
|
||||
|
||||
@@ -0,0 +1,110 @@
|
||||
import type { RpcError, WaitUntil } from "@/transport/types";
|
||||
import { cdpError } from "./errors";
|
||||
|
||||
interface NavigationEvent {
|
||||
tabId: number;
|
||||
frameId: number;
|
||||
documentId?: string;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
interface NavigationEvents {
|
||||
addListener(listener: (details: NavigationEvent) => void): void;
|
||||
removeListener(listener: (details: NavigationEvent) => void): void;
|
||||
}
|
||||
|
||||
/** Normal browser navigation remains available when chrome.debugger is denied. */
|
||||
export interface BrowserNavigationApi {
|
||||
update(tabId: number, props: { url: string }): Promise<unknown>;
|
||||
reload(tabId: number, props: { bypassCache: boolean }): Promise<void>;
|
||||
onCommitted: NavigationEvents;
|
||||
onDOMContentLoaded: NavigationEvents;
|
||||
onCompleted: NavigationEvents;
|
||||
onErrorOccurred: NavigationEvents;
|
||||
}
|
||||
|
||||
export const chromeBrowserNavigationApi: BrowserNavigationApi = {
|
||||
update: (tabId, props) => chrome.tabs.update(tabId, props),
|
||||
reload: (tabId, props) => chrome.tabs.reload(tabId, props),
|
||||
get onCommitted() {
|
||||
return chrome.webNavigation.onCommitted;
|
||||
},
|
||||
get onDOMContentLoaded() {
|
||||
return chrome.webNavigation.onDOMContentLoaded;
|
||||
},
|
||||
get onCompleted() {
|
||||
return chrome.webNavigation.onCompleted;
|
||||
},
|
||||
get onErrorOccurred() {
|
||||
return chrome.webNavigation.onErrorOccurred;
|
||||
},
|
||||
};
|
||||
|
||||
export type NavigationOutcome =
|
||||
| { reached: "match" | "timeout" | "cancelled"; lastLifecycle?: string }
|
||||
| { reached: "failed"; error: RpcError };
|
||||
|
||||
/** Subscribe before dispatch and ignore the departing document's load events. */
|
||||
export async function navigateWithBrowserApi(
|
||||
api: BrowserNavigationApi,
|
||||
tabId: number,
|
||||
action: () => Promise<unknown>,
|
||||
waitUntil: Exclude<WaitUntil, "networkidle">,
|
||||
timeoutMs: number,
|
||||
signal?: AbortSignal,
|
||||
): Promise<NavigationOutcome> {
|
||||
if (signal?.aborted) return { reached: "cancelled" };
|
||||
if (timeoutMs <= 0) return { reached: "timeout" };
|
||||
let committed = false;
|
||||
let documentId: string | undefined;
|
||||
let lastLifecycle: string | undefined;
|
||||
let finish!: (outcome: NavigationOutcome) => void;
|
||||
const outcome = new Promise<NavigationOutcome>((resolve) => {
|
||||
finish = resolve;
|
||||
});
|
||||
const isMainFrame = (details: NavigationEvent) =>
|
||||
details.tabId === tabId && details.frameId === 0;
|
||||
const onCommitted = (details: NavigationEvent) => {
|
||||
if (!isMainFrame(details)) return;
|
||||
committed = true;
|
||||
documentId = details.documentId;
|
||||
lastLifecycle = "commit";
|
||||
if (waitUntil === "commit") finish({ reached: "match", lastLifecycle });
|
||||
};
|
||||
const onDOMContentLoaded = (details: NavigationEvent) => {
|
||||
if (!isMainFrame(details) || !committed || (documentId && details.documentId !== documentId))
|
||||
return;
|
||||
lastLifecycle = "DOMContentLoaded";
|
||||
if (waitUntil === "domcontentloaded") finish({ reached: "match", lastLifecycle });
|
||||
};
|
||||
const onCompleted = (details: NavigationEvent) => {
|
||||
if (!isMainFrame(details) || !committed || (documentId && details.documentId !== documentId))
|
||||
return;
|
||||
finish({ reached: "match", lastLifecycle: "load" });
|
||||
};
|
||||
const onError = (details: NavigationEvent) => {
|
||||
if (!isMainFrame(details)) return;
|
||||
finish({ reached: "failed", error: cdpError(details.error ?? "browser navigation failed") });
|
||||
};
|
||||
const onAbort = () => finish({ reached: "cancelled", lastLifecycle });
|
||||
const subscriptions = [
|
||||
[api.onCommitted, onCommitted],
|
||||
[api.onDOMContentLoaded, onDOMContentLoaded],
|
||||
[api.onCompleted, onCompleted],
|
||||
[api.onErrorOccurred, onError],
|
||||
] as const;
|
||||
for (const [event, listener] of subscriptions) event.addListener(listener);
|
||||
signal?.addEventListener("abort", onAbort, { once: true });
|
||||
const timer = setTimeout(() => finish({ reached: "timeout", lastLifecycle }), timeoutMs);
|
||||
try {
|
||||
if (signal?.aborted) return { reached: "cancelled" };
|
||||
await action();
|
||||
return await outcome;
|
||||
} catch (error) {
|
||||
return { reached: "failed", error: cdpError(error) };
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
signal?.removeEventListener("abort", onAbort);
|
||||
for (const [event, listener] of subscriptions) event.removeListener(listener);
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,7 @@ import type { Transport } from "@/transport/transport";
|
||||
import type {
|
||||
ClickParams,
|
||||
ConsoleParams,
|
||||
DownloadParams,
|
||||
EmulateParams,
|
||||
EvaluateParams,
|
||||
FillParams,
|
||||
@@ -28,11 +29,14 @@ import type {
|
||||
ScreenshotParams,
|
||||
SelectParams,
|
||||
SnapshotParams,
|
||||
UploadParams,
|
||||
WaitForNavigationParams,
|
||||
} from "@/transport/types";
|
||||
import { isRequestFrame } from "@/transport/types";
|
||||
import { handleConsole } from "./console";
|
||||
import { handleDownload } from "./download";
|
||||
import { type EmulateCdpRunner, handleEmulate } from "./emulate";
|
||||
import { classifyCdpError } from "./errors";
|
||||
import { handleEvaluate } from "./evaluate";
|
||||
import { handleRequestHelp } from "./human-loop";
|
||||
import { handleClick, handleFill, handleHover, handlePress, handleSelect } from "./interaction";
|
||||
@@ -51,7 +55,12 @@ import {
|
||||
handleScreenshot,
|
||||
handleSnapshot,
|
||||
} from "./observation";
|
||||
import { handleRecordAwait, handleRecordStart, handleRecordStop } from "./record";
|
||||
import {
|
||||
handleRecordAwait,
|
||||
handleRecordStart,
|
||||
handleRecordStop,
|
||||
type RecordRuntimeDeps,
|
||||
} from "./record";
|
||||
import {
|
||||
handleSessionStart,
|
||||
handleSessionStop,
|
||||
@@ -61,6 +70,7 @@ import {
|
||||
import { chromeTabsApi, lookupSession, resolveTargetTab } from "./shared";
|
||||
import {
|
||||
type BorrowConfirmationApprover,
|
||||
chromeTabMutationApi,
|
||||
handleTabBorrow,
|
||||
handleTabClose,
|
||||
handleTabCreate,
|
||||
@@ -74,6 +84,7 @@ import {
|
||||
type TabReturnParams,
|
||||
type TabSelectParams,
|
||||
} from "./tabs";
|
||||
import { handleUpload } from "./upload";
|
||||
import { handleWaitForNavigation } from "./waits";
|
||||
import { handleWindowResize, type WindowResizeParams } from "./window";
|
||||
|
||||
@@ -99,6 +110,7 @@ export interface DispatcherDeps {
|
||||
transport: Transport;
|
||||
sessions: SessionManager;
|
||||
cdp?: DispatcherCdpRunner;
|
||||
recording?: RecordRuntimeDeps;
|
||||
/**
|
||||
* Invoked whenever a dispatched RPC may have changed the live
|
||||
* session set (currently `tool.session_start` and
|
||||
@@ -108,6 +120,8 @@ export interface DispatcherDeps {
|
||||
onSessionsChanged?: () => void;
|
||||
/** Invoked before a tool that dispatches page input or mutates browser state is forwarded. */
|
||||
onBrowserControlResumed?: (sessionId: string) => void;
|
||||
/** Invoked after a tab is explicitly claimed so its overlay can be refreshed immediately. */
|
||||
onAgentTabClaimed?: (tabId: number, windowId: number) => void;
|
||||
/** User approval for `tool.tab_borrow` (overlay in content script). */
|
||||
approveBorrow?: BorrowConfirmationApprover;
|
||||
/** i18n notification copy for `tool.request_help` (resolved per-call). */
|
||||
@@ -134,8 +148,10 @@ export class ToolDispatcher {
|
||||
private readonly transport: Transport;
|
||||
private readonly sessions: SessionManager;
|
||||
private readonly cdp?: DispatcherCdpRunner;
|
||||
private readonly recording?: RecordRuntimeDeps;
|
||||
private readonly onSessionsChanged?: () => void;
|
||||
private readonly onBrowserControlResumed?: (sessionId: string) => void;
|
||||
private readonly onAgentTabClaimed?: (tabId: number, windowId: number) => void;
|
||||
private readonly approveBorrow?: BorrowConfirmationApprover;
|
||||
private readonly helpNotificationCopy?: () => { title: string; body: string };
|
||||
private subscription: { dispose(): void } | null = null;
|
||||
@@ -153,8 +169,10 @@ export class ToolDispatcher {
|
||||
this.transport = deps.transport;
|
||||
this.sessions = deps.sessions;
|
||||
this.cdp = deps.cdp;
|
||||
this.recording = deps.recording;
|
||||
this.onSessionsChanged = deps.onSessionsChanged;
|
||||
this.onBrowserControlResumed = deps.onBrowserControlResumed;
|
||||
this.onAgentTabClaimed = deps.onAgentTabClaimed;
|
||||
this.approveBorrow = deps.approveBorrow;
|
||||
this.helpNotificationCopy = deps.helpNotificationCopy;
|
||||
}
|
||||
@@ -222,7 +240,7 @@ export class ToolDispatcher {
|
||||
if (sessionId) this.onBrowserControlResumed?.(sessionId);
|
||||
const result = await this.invoke(req, ac.signal);
|
||||
if (isRpcError(result)) {
|
||||
body = { id: req.id, error: result };
|
||||
body = { id: req.id, error: classifyCdpError(result) };
|
||||
} else {
|
||||
body = { id: req.id, result };
|
||||
if (req.method === "tool.session_start") {
|
||||
@@ -290,13 +308,25 @@ export class ToolDispatcher {
|
||||
await this.releaseHoverLatch((req.params as SessionStopParams).session_id);
|
||||
return handleSessionStop(this.sessions, req.params as SessionStopParams, {
|
||||
cdp: this.cdp,
|
||||
// Must be wired in production: the agent-tab cleanup and the
|
||||
// window-release decision (issue #57) read these deps directly
|
||||
// and silently no-op when they are absent.
|
||||
tabManagement: { tabs: chromeTabMutationApi },
|
||||
tabsQuery: chromeTabsApi,
|
||||
signal,
|
||||
});
|
||||
}
|
||||
case "tool.tab_list":
|
||||
return handleTabList(this.sessions, req.params as TabListParams, chromeTabsApi, signal);
|
||||
case "tool.tab_create":
|
||||
return handleTabCreate(this.sessions, req.params as TabCreateParams, { signal });
|
||||
case "tool.tab_create": {
|
||||
const result = await handleTabCreate(this.sessions, req.params as TabCreateParams, {
|
||||
signal,
|
||||
});
|
||||
if (!isRpcError(result)) {
|
||||
this.onAgentTabClaimed?.(result.tab_id, result.window_id);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
case "tool.tab_close":
|
||||
return this.withHoverReleaseForRequest(
|
||||
req.params as TabCloseParams,
|
||||
@@ -305,13 +335,22 @@ export class ToolDispatcher {
|
||||
);
|
||||
case "tool.tab_select":
|
||||
return handleTabSelect(this.sessions, req.params as TabSelectParams, { signal });
|
||||
case "tool.tab_borrow":
|
||||
return handleTabBorrow(this.sessions, req.params as TabBorrowParams, {
|
||||
case "tool.tab_borrow": {
|
||||
const result = await handleTabBorrow(this.sessions, req.params as TabBorrowParams, {
|
||||
signal,
|
||||
approveBorrow: this.approveBorrow,
|
||||
});
|
||||
if (!isRpcError(result)) {
|
||||
this.onAgentTabClaimed?.(result.tab_id, result.agent_window_id);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
case "tool.tab_return":
|
||||
return handleTabReturn(this.sessions, req.params as TabReturnParams, { signal });
|
||||
return handleTabReturn(this.sessions, req.params as TabReturnParams, {
|
||||
signal,
|
||||
cdp: this.cdp,
|
||||
beforeReturn: (sessionId, tabId) => this.releaseHoverLatch(sessionId, tabId),
|
||||
});
|
||||
case "tool.window_resize":
|
||||
return handleWindowResize(
|
||||
this.sessions,
|
||||
@@ -375,7 +414,11 @@ export class ToolDispatcher {
|
||||
? {
|
||||
cdp: this.cdp,
|
||||
tabsApi: chromeTabsCaptureApi,
|
||||
conditionalSurfaceProbe: !this.hasHoverLatchForScope(hoverScope),
|
||||
// Active hover probing is opt-in. A held hover latch still
|
||||
// suppresses it, because probing would move the cursor off
|
||||
// the element the caller is deliberately holding.
|
||||
conditionalSurfaceProbe:
|
||||
params.probe_hover === true && !this.hasHoverLatchForScope(hoverScope),
|
||||
hoverProbeBypassOverlay: bypassOverlay,
|
||||
}
|
||||
: undefined,
|
||||
@@ -505,6 +548,40 @@ export class ToolDispatcher {
|
||||
),
|
||||
signal,
|
||||
);
|
||||
case "tool.upload":
|
||||
return this.withHoverReleaseForRequest(
|
||||
req.params as UploadParams,
|
||||
() =>
|
||||
this.cdp
|
||||
? handleUpload(this.sessions, req.params as UploadParams, {
|
||||
cdp: this.cdp,
|
||||
tabsApi: chromeTabsApi,
|
||||
signal,
|
||||
bypassOverlay,
|
||||
})
|
||||
: Promise.resolve({
|
||||
code: "unsupported",
|
||||
message: "upload requires CDP",
|
||||
} satisfies RpcError),
|
||||
signal,
|
||||
);
|
||||
case "tool.download":
|
||||
return this.withHoverReleaseForRequest(
|
||||
req.params as DownloadParams,
|
||||
() =>
|
||||
this.cdp
|
||||
? handleDownload(this.sessions, req.params as DownloadParams, {
|
||||
cdp: this.cdp,
|
||||
tabsApi: chromeTabsApi,
|
||||
signal,
|
||||
bypassOverlay,
|
||||
})
|
||||
: Promise.resolve({
|
||||
code: "unsupported",
|
||||
message: "download requires CDP",
|
||||
} satisfies RpcError),
|
||||
signal,
|
||||
);
|
||||
case "tool.evaluate":
|
||||
return handleEvaluate(
|
||||
this.sessions,
|
||||
@@ -531,44 +608,26 @@ export class ToolDispatcher {
|
||||
signal,
|
||||
});
|
||||
case "tool.record_start":
|
||||
return handleRecordStart(this.sessions, req.params as RecordStartParams, {
|
||||
tabsApi: chromeTabsApi,
|
||||
sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
|
||||
bypassOverlay: async (tabId, enabled) => {
|
||||
try {
|
||||
await chrome.tabs.sendMessage(tabId, {
|
||||
type: OVERLAY_AUTOMATION_BYPASS,
|
||||
enabled,
|
||||
});
|
||||
} catch {
|
||||
// Content script may be unavailable on restricted pages.
|
||||
}
|
||||
},
|
||||
...(this.cdp ? { cdp: this.cdp } : {}),
|
||||
signal,
|
||||
});
|
||||
return this.recording
|
||||
? handleRecordStart(this.sessions, req.params as RecordStartParams, {
|
||||
...this.recording,
|
||||
signal,
|
||||
})
|
||||
: recordingRuntimeUnavailable();
|
||||
case "tool.record_stop":
|
||||
return handleRecordStop(this.sessions, req.params as RecordStopParams, {
|
||||
tabsApi: chromeTabsApi,
|
||||
sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
|
||||
bypassOverlay: async (tabId, enabled) => {
|
||||
try {
|
||||
await chrome.tabs.sendMessage(tabId, {
|
||||
type: OVERLAY_AUTOMATION_BYPASS,
|
||||
enabled,
|
||||
});
|
||||
} catch {
|
||||
// Content script may be unavailable on restricted pages.
|
||||
}
|
||||
},
|
||||
signal,
|
||||
});
|
||||
return this.recording
|
||||
? handleRecordStop(this.sessions, req.params as RecordStopParams, {
|
||||
...this.recording,
|
||||
signal,
|
||||
})
|
||||
: recordingRuntimeUnavailable();
|
||||
case "tool.record_await":
|
||||
return handleRecordAwait(this.sessions, req.params as RecordAwaitParams, {
|
||||
tabsApi: chromeTabsApi,
|
||||
sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
|
||||
signal,
|
||||
});
|
||||
return this.recording
|
||||
? handleRecordAwait(this.sessions, req.params as RecordAwaitParams, {
|
||||
...this.recording,
|
||||
signal,
|
||||
})
|
||||
: recordingRuntimeUnavailable();
|
||||
default:
|
||||
return {
|
||||
code: "unknown_method",
|
||||
@@ -704,6 +763,13 @@ function isRpcError(v: unknown): v is RpcError {
|
||||
);
|
||||
}
|
||||
|
||||
function recordingRuntimeUnavailable(): RpcError {
|
||||
return {
|
||||
code: "protocol_error",
|
||||
message: "recording runtime is unavailable",
|
||||
};
|
||||
}
|
||||
|
||||
function sessionIdForBrowserControlMethod(req: RequestFrame): string | null {
|
||||
switch (req.method) {
|
||||
case "tool.tab_create":
|
||||
@@ -722,6 +788,8 @@ function sessionIdForBrowserControlMethod(req: RequestFrame): string | null {
|
||||
case "tool.fill":
|
||||
case "tool.press":
|
||||
case "tool.select":
|
||||
case "tool.upload":
|
||||
case "tool.download":
|
||||
case "tool.evaluate":
|
||||
case "tool.observe":
|
||||
case "tool.request_help":
|
||||
|
||||
@@ -0,0 +1,364 @@
|
||||
// Order-independent coordinator for one browser download. CDP supplies the
|
||||
// exact target/frame intent while chrome.downloads supplies the download id
|
||||
// and filename routing hook; neither event is assumed to arrive first.
|
||||
|
||||
import type { CdpTarget } from "@/browser-driver/frame-graph";
|
||||
import type { ClickResult, RpcError, TransferEffectState } from "@/transport/types";
|
||||
import { transferError } from "./errors";
|
||||
import { type CdpRunner, isRpcError } from "./shared";
|
||||
|
||||
const CORRELATION_GRACE_MS = 750;
|
||||
const UNIQUE_SETTLE_MS = 50;
|
||||
const SIZE_POLL_MS = 250;
|
||||
|
||||
type DeterminingFilenameListener = (
|
||||
item: chrome.downloads.DownloadItem,
|
||||
suggest: (suggestion?: chrome.downloads.DownloadFilenameSuggestion) => void,
|
||||
) => void | true;
|
||||
|
||||
interface ListenerEvent<T> {
|
||||
addListener(listener: T): void;
|
||||
removeListener(listener: T): void;
|
||||
}
|
||||
|
||||
export interface DownloadsApi {
|
||||
onCreated: ListenerEvent<(item: chrome.downloads.DownloadItem) => void>;
|
||||
onChanged: ListenerEvent<(delta: chrome.downloads.DownloadDelta) => void>;
|
||||
onDeterminingFilename: ListenerEvent<DeterminingFilenameListener>;
|
||||
search(query: chrome.downloads.DownloadQuery): Promise<chrome.downloads.DownloadItem[]>;
|
||||
cancel(downloadId: number): Promise<void>;
|
||||
removeFile(downloadId: number): Promise<void>;
|
||||
}
|
||||
|
||||
export const chromeDownloadsApi: DownloadsApi = {
|
||||
get onCreated() {
|
||||
return chrome.downloads.onCreated;
|
||||
},
|
||||
get onChanged() {
|
||||
return chrome.downloads.onChanged;
|
||||
},
|
||||
get onDeterminingFilename() {
|
||||
return chrome.downloads.onDeterminingFilename;
|
||||
},
|
||||
search: (query) => chrome.downloads.search(query),
|
||||
cancel: (id) => chrome.downloads.cancel(id),
|
||||
removeFile: (id) => chrome.downloads.removeFile(id),
|
||||
};
|
||||
|
||||
export interface DownloadCaptureOptions {
|
||||
cdp: CdpRunner;
|
||||
target: CdpTarget;
|
||||
expectedFrameId?: string;
|
||||
downloads: DownloadsApi;
|
||||
browserRelativeDir: string;
|
||||
maxByteSize?: number;
|
||||
timeoutMs: number;
|
||||
signal?: AbortSignal;
|
||||
trigger(): Promise<ClickResult | RpcError>;
|
||||
}
|
||||
|
||||
export interface DownloadCaptureResult {
|
||||
click: ClickResult;
|
||||
item: chrome.downloads.DownloadItem;
|
||||
}
|
||||
|
||||
interface DownloadIntent {
|
||||
url: string;
|
||||
suggestedFilename: string;
|
||||
frameId?: string;
|
||||
}
|
||||
|
||||
interface DownloadCandidate {
|
||||
item: chrome.downloads.DownloadItem;
|
||||
suggest: (suggestion?: chrome.downloads.DownloadFilenameSuggestion) => void;
|
||||
suggested: boolean;
|
||||
graceTimer: ReturnType<typeof setTimeout>;
|
||||
}
|
||||
|
||||
function safeBasename(filename: string): string {
|
||||
const basename = filename.split(/[\\/]/).pop()?.trim();
|
||||
return basename && basename !== "." && basename !== ".." ? basename : "download";
|
||||
}
|
||||
|
||||
function sameTarget(source: { tabId?: number; sessionId?: string }, target: CdpTarget): boolean {
|
||||
return source.tabId === target.tabId && source.sessionId === target.sessionId;
|
||||
}
|
||||
|
||||
function matchesIntent(item: chrome.downloads.DownloadItem, intent: DownloadIntent): boolean {
|
||||
const urlMatches = item.url === intent.url || item.finalUrl === intent.url;
|
||||
return urlMatches && safeBasename(item.filename) === safeBasename(intent.suggestedFilename);
|
||||
}
|
||||
|
||||
function knownSize(item: chrome.downloads.DownloadItem): number | undefined {
|
||||
if (item.fileSize >= 0) return item.fileSize;
|
||||
if (item.totalBytes >= 0) return item.totalBytes;
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function captureError(
|
||||
message: string,
|
||||
effectState: TransferEffectState,
|
||||
phase: string,
|
||||
cleanupFailed = false,
|
||||
): RpcError {
|
||||
return transferError("cdp_failed", "download_capture_failed", message, {
|
||||
effectState,
|
||||
phase,
|
||||
...(cleanupFailed ? { cleanupState: "failed" } : {}),
|
||||
});
|
||||
}
|
||||
|
||||
async function cleanupClaimedDownload(downloads: DownloadsApi, downloadId: number): Promise<void> {
|
||||
const lookup = async () => (await downloads.search({ id: downloadId }))[0];
|
||||
const item = await lookup();
|
||||
if (!item || item.state === "interrupted") return;
|
||||
if (item.state === "complete") {
|
||||
await downloads.removeFile(downloadId);
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
await downloads.cancel(downloadId);
|
||||
} catch (cancelError) {
|
||||
// Completion can win the race after the lookup but before cancellation.
|
||||
// Reconcile against Chrome's authoritative state before declaring cleanup
|
||||
// failed so every terminal state has one explicit cleanup path.
|
||||
const reconciled = await lookup();
|
||||
if (!reconciled || reconciled.state === "interrupted") return;
|
||||
if (reconciled.state === "complete") {
|
||||
await downloads.removeFile(downloadId);
|
||||
return;
|
||||
}
|
||||
throw cancelError;
|
||||
}
|
||||
}
|
||||
|
||||
export async function captureBrowserDownload(
|
||||
options: DownloadCaptureOptions,
|
||||
): Promise<DownloadCaptureResult | RpcError> {
|
||||
let click: ClickResult | undefined;
|
||||
let intent: DownloadIntent | undefined;
|
||||
let capturedId: number | undefined;
|
||||
let settled = false;
|
||||
let succeeded = false;
|
||||
let failureResult: RpcError | undefined;
|
||||
let uniquenessTimer: ReturnType<typeof setTimeout> | undefined;
|
||||
let operationTimer: ReturnType<typeof setTimeout> | undefined;
|
||||
let sizePoll: ReturnType<typeof setInterval> | undefined;
|
||||
const candidates = new Map<number, DownloadCandidate>();
|
||||
const createdItems = new Map<number, chrome.downloads.DownloadItem>();
|
||||
|
||||
let resolveCompletion!: (item: chrome.downloads.DownloadItem) => void;
|
||||
let rejectCompletion!: (error: Error) => void;
|
||||
const completion = new Promise<chrome.downloads.DownloadItem>((resolve, reject) => {
|
||||
resolveCompletion = resolve;
|
||||
rejectCompletion = reject;
|
||||
});
|
||||
const fail = (error: Error) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
rejectCompletion(error);
|
||||
};
|
||||
const complete = (item: chrome.downloads.DownloadItem) => {
|
||||
if (settled) return;
|
||||
const size = knownSize(item);
|
||||
if (size !== undefined && options.maxByteSize !== undefined && size > options.maxByteSize) {
|
||||
fail(new Error(`download exceeds transfer limit ${options.maxByteSize}`));
|
||||
return;
|
||||
}
|
||||
settled = true;
|
||||
resolveCompletion(item);
|
||||
};
|
||||
const suggestDefault = (candidate: DownloadCandidate) => {
|
||||
if (candidate.suggested) return;
|
||||
candidate.suggested = true;
|
||||
candidate.suggest();
|
||||
};
|
||||
const matchingCandidates = (): DownloadCandidate[] => {
|
||||
const currentIntent = intent;
|
||||
return currentIntent
|
||||
? [...candidates.values()].filter(
|
||||
(candidate) => !candidate.suggested && matchesIntent(candidate.item, currentIntent),
|
||||
)
|
||||
: [];
|
||||
};
|
||||
|
||||
const claimUnique = () => {
|
||||
uniquenessTimer = undefined;
|
||||
if (settled || capturedId !== undefined || !intent) return;
|
||||
const matches = matchingCandidates();
|
||||
if (matches.length !== 1) {
|
||||
if (matches.length > 1) {
|
||||
for (const candidate of matches) suggestDefault(candidate);
|
||||
fail(new Error("download attribution is ambiguous"));
|
||||
}
|
||||
return;
|
||||
}
|
||||
const candidate = matches[0];
|
||||
candidate.suggested = true;
|
||||
clearTimeout(candidate.graceTimer);
|
||||
capturedId = candidate.item.id;
|
||||
candidate.suggest({
|
||||
filename: `${options.browserRelativeDir}/${safeBasename(intent.suggestedFilename)}`,
|
||||
conflictAction: "overwrite",
|
||||
});
|
||||
const size = knownSize(candidate.item);
|
||||
if (size !== undefined && options.maxByteSize !== undefined && size > options.maxByteSize) {
|
||||
fail(new Error(`download exceeds transfer limit ${options.maxByteSize}`));
|
||||
return;
|
||||
}
|
||||
const created = createdItems.get(candidate.item.id);
|
||||
if (created?.state === "interrupted") {
|
||||
fail(new Error(created.error ?? "download interrupted"));
|
||||
} else if (created?.state === "complete") {
|
||||
complete(created);
|
||||
}
|
||||
};
|
||||
const reconcile = () => {
|
||||
if (settled || capturedId !== undefined || !intent) return;
|
||||
const matches = matchingCandidates();
|
||||
if (matches.length > 1) {
|
||||
for (const candidate of matches) suggestDefault(candidate);
|
||||
fail(new Error("download attribution is ambiguous"));
|
||||
return;
|
||||
}
|
||||
if (matches.length === 1 && !uniquenessTimer) {
|
||||
uniquenessTimer = setTimeout(claimUnique, UNIQUE_SETTLE_MS);
|
||||
}
|
||||
};
|
||||
|
||||
const determiningListener: DeterminingFilenameListener = (item, suggest) => {
|
||||
const candidate: DownloadCandidate = {
|
||||
item,
|
||||
suggest,
|
||||
suggested: false,
|
||||
graceTimer: setTimeout(() => {
|
||||
suggestDefault(candidate);
|
||||
candidates.delete(item.id);
|
||||
if (intent && matchesIntent(item, intent) && capturedId === undefined) {
|
||||
fail(new Error("download correlation grace elapsed before unique attribution"));
|
||||
}
|
||||
}, CORRELATION_GRACE_MS),
|
||||
};
|
||||
candidates.set(item.id, candidate);
|
||||
reconcile();
|
||||
return true;
|
||||
};
|
||||
const createdListener = (item: chrome.downloads.DownloadItem) => {
|
||||
createdItems.set(item.id, item);
|
||||
if (capturedId !== item.id) return;
|
||||
if (item.state === "interrupted") {
|
||||
fail(new Error(item.error ?? "download interrupted"));
|
||||
} else if (item.state === "complete") {
|
||||
complete(item);
|
||||
}
|
||||
};
|
||||
const changedListener = async (delta: chrome.downloads.DownloadDelta) => {
|
||||
if (capturedId === undefined || delta.id !== capturedId || settled) return;
|
||||
if (delta.state?.current === "interrupted" || delta.error?.current) {
|
||||
fail(new Error(delta.error?.current ?? "download interrupted"));
|
||||
return;
|
||||
}
|
||||
if (delta.state?.current === "complete") {
|
||||
try {
|
||||
const [item] = await options.downloads.search({ id: delta.id });
|
||||
if (item) complete(item);
|
||||
else fail(new Error("completed download disappeared"));
|
||||
} catch (err) {
|
||||
fail(err instanceof Error ? err : new Error(String(err)));
|
||||
}
|
||||
}
|
||||
};
|
||||
const onAbort = () => fail(new DOMException("aborted", "AbortError"));
|
||||
const cdpSubscription = options.cdp.onEvent?.((source, method, raw) => {
|
||||
if (method !== "Page.downloadWillBegin" || !sameTarget(source, options.target)) return;
|
||||
const event = raw as { url?: unknown; suggestedFilename?: unknown; frameId?: unknown };
|
||||
if (typeof event.url !== "string" || typeof event.suggestedFilename !== "string") return;
|
||||
if (options.expectedFrameId && event.frameId !== options.expectedFrameId) {
|
||||
fail(new Error("download originated from a different frame"));
|
||||
return;
|
||||
}
|
||||
if (intent) {
|
||||
fail(new Error("download trigger produced more than one browser download intent"));
|
||||
return;
|
||||
}
|
||||
intent = {
|
||||
url: event.url,
|
||||
suggestedFilename: event.suggestedFilename,
|
||||
...(typeof event.frameId === "string" ? { frameId: event.frameId } : {}),
|
||||
};
|
||||
reconcile();
|
||||
});
|
||||
if (!cdpSubscription) {
|
||||
return captureError("CDP download intent subscription unavailable", "none", "arm");
|
||||
}
|
||||
|
||||
options.downloads.onDeterminingFilename.addListener(determiningListener);
|
||||
options.downloads.onCreated.addListener(createdListener);
|
||||
options.downloads.onChanged.addListener(changedListener);
|
||||
options.signal?.addEventListener("abort", onAbort, { once: true });
|
||||
operationTimer = setTimeout(
|
||||
() => fail(new Error("download did not complete before timeout")),
|
||||
options.timeoutMs,
|
||||
);
|
||||
sizePoll = setInterval(() => {
|
||||
if (capturedId === undefined || settled || options.maxByteSize === undefined) return;
|
||||
void options.downloads
|
||||
.search({ id: capturedId })
|
||||
.then(([item]) => {
|
||||
if (!item || settled) return;
|
||||
if (item.bytesReceived > (options.maxByteSize as number)) {
|
||||
fail(new Error(`download exceeds transfer limit ${options.maxByteSize}`));
|
||||
}
|
||||
})
|
||||
.catch((err) => fail(err instanceof Error ? err : new Error(String(err))));
|
||||
}, SIZE_POLL_MS);
|
||||
|
||||
try {
|
||||
const triggered = await options.trigger();
|
||||
if (isRpcError(triggered)) {
|
||||
void completion.catch(() => undefined);
|
||||
const effect: TransferEffectState =
|
||||
capturedId !== undefined ? "committed" : intent ? "unknown" : "none";
|
||||
failureResult = {
|
||||
...triggered,
|
||||
data: { ...triggered.data, effect_state: effect, phase: "trigger" },
|
||||
};
|
||||
return failureResult;
|
||||
}
|
||||
click = triggered;
|
||||
const item = await completion;
|
||||
succeeded = true;
|
||||
return { click, item };
|
||||
} catch (err) {
|
||||
const effect: TransferEffectState =
|
||||
capturedId !== undefined ? "committed" : click ? "unknown" : "none";
|
||||
failureResult = captureError(
|
||||
err instanceof Error ? err.message : String(err),
|
||||
effect,
|
||||
capturedId !== undefined ? "download" : "attribution",
|
||||
);
|
||||
return failureResult;
|
||||
} finally {
|
||||
settled = true;
|
||||
if (operationTimer) clearTimeout(operationTimer);
|
||||
if (uniquenessTimer) clearTimeout(uniquenessTimer);
|
||||
if (sizePoll) clearInterval(sizePoll);
|
||||
options.signal?.removeEventListener("abort", onAbort);
|
||||
options.downloads.onDeterminingFilename.removeListener(determiningListener);
|
||||
options.downloads.onCreated.removeListener(createdListener);
|
||||
options.downloads.onChanged.removeListener(changedListener);
|
||||
cdpSubscription.dispose();
|
||||
for (const candidate of candidates.values()) {
|
||||
clearTimeout(candidate.graceTimer);
|
||||
if (candidate.item.id !== capturedId) suggestDefault(candidate);
|
||||
}
|
||||
if (!succeeded && capturedId !== undefined) {
|
||||
try {
|
||||
await cleanupClaimedDownload(options.downloads, capturedId);
|
||||
} catch {
|
||||
if (failureResult?.data) failureResult.data.cleanup_state = "failed";
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
// Download orchestration: validate session/tab ownership, then delegate one
|
||||
// browser-global chrome.downloads transaction to download-capture.ts.
|
||||
|
||||
import type { SessionManager } from "@/session-manager/manager";
|
||||
import type { DownloadParams, DownloadResult, RpcError } from "@/transport/types";
|
||||
import { captureBrowserDownload, chromeDownloadsApi, type DownloadsApi } from "./download-capture";
|
||||
import { clickResolvedTarget, type InteractionDeps, resolveActionTarget } from "./interaction";
|
||||
import { enforceAgentWindow, isRpcError, lookupSession, resolveTargetTab } from "./shared";
|
||||
|
||||
let downloadActive = false;
|
||||
|
||||
export type { DownloadsApi } from "./download-capture";
|
||||
|
||||
export interface DownloadDeps extends InteractionDeps {
|
||||
downloads?: DownloadsApi;
|
||||
}
|
||||
|
||||
export async function handleDownload(
|
||||
manager: SessionManager,
|
||||
params: DownloadParams,
|
||||
deps: DownloadDeps,
|
||||
): Promise<DownloadResult | RpcError> {
|
||||
if (downloadActive) return { code: "invalid_params", message: "another bsk download is active" };
|
||||
downloadActive = true;
|
||||
try {
|
||||
const ctx = lookupSession(manager, params, "download");
|
||||
if (isRpcError(ctx)) return ctx;
|
||||
const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi);
|
||||
if (isRpcError(target)) return target;
|
||||
const denied = enforceAgentWindow(ctx, target, "download");
|
||||
if (denied) return denied;
|
||||
if (!params.browser_relative_dir) {
|
||||
return { code: "invalid_params", message: "download requires a daemon capability directory" };
|
||||
}
|
||||
const address = await resolveActionTarget(deps.cdp, ctx, target, params, "download");
|
||||
if (isRpcError(address)) return address;
|
||||
|
||||
const capture = await captureBrowserDownload({
|
||||
cdp: deps.cdp,
|
||||
target: address.cdpTarget,
|
||||
downloads: deps.downloads ?? chromeDownloadsApi,
|
||||
browserRelativeDir: params.browser_relative_dir,
|
||||
maxByteSize: params.max_byte_size,
|
||||
timeoutMs: params.timeout_ms ?? 120_000,
|
||||
signal: deps.signal,
|
||||
expectedFrameId: address.frameId,
|
||||
trigger: () => clickResolvedTarget(ctx, address, {}, deps),
|
||||
});
|
||||
if (isRpcError(capture)) return capture;
|
||||
const { click, item } = capture;
|
||||
return {
|
||||
tab_id: target.tabId,
|
||||
used_ref: click.used_ref,
|
||||
used_selector: click.used_selector,
|
||||
suggested_filename: item.filename.split(/[\\/]/).pop() ?? "download",
|
||||
byte_size: item.fileSize >= 0 ? item.fileSize : item.totalBytes,
|
||||
mime: item.mime || undefined,
|
||||
danger: item.danger,
|
||||
browser_path: item.filename,
|
||||
};
|
||||
} finally {
|
||||
downloadActive = false;
|
||||
}
|
||||
}
|
||||
@@ -2,10 +2,51 @@
|
||||
// rendering. Extension handlers attach reasons here; human-facing copy
|
||||
// lives in bsk-cli `render_error.rs`.
|
||||
|
||||
import type { ErrorCode, RpcError, RpcErrorData, RpcErrorReason } from "@/transport/types";
|
||||
import type {
|
||||
ErrorCode,
|
||||
RpcError,
|
||||
RpcErrorData,
|
||||
RpcErrorReason,
|
||||
TransferCleanupState,
|
||||
TransferEffectState,
|
||||
} from "@/transport/types";
|
||||
|
||||
export type { RpcErrorData, RpcErrorReason };
|
||||
|
||||
/** Chrome checks the whole tab's frame tree, including other extensions' frames. */
|
||||
export function isCdpExtensionAccessDenied(error: unknown): boolean {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
return message.includes("Cannot access a chrome-extension:// URL of different extension");
|
||||
}
|
||||
|
||||
/** Preserve more specific reasons, especially file-transfer outcome/cleanup details. */
|
||||
export function classifyCdpError(error: RpcError): RpcError {
|
||||
if (
|
||||
error.code !== "cdp_failed" ||
|
||||
error.data?.reason ||
|
||||
!isCdpExtensionAccessDenied(error.message)
|
||||
) {
|
||||
return error;
|
||||
}
|
||||
return {
|
||||
...error,
|
||||
data: { ...error.data, reason: "cdp_extension_access_denied" },
|
||||
};
|
||||
}
|
||||
|
||||
export function cdpError(error: unknown): RpcError {
|
||||
return classifyCdpError({
|
||||
code: "cdp_failed",
|
||||
message: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
}
|
||||
|
||||
export interface TransferErrorOptions {
|
||||
effectState: TransferEffectState;
|
||||
phase: string;
|
||||
cleanupState?: TransferCleanupState;
|
||||
}
|
||||
|
||||
export function rpcError(
|
||||
code: ErrorCode,
|
||||
reason: RpcErrorReason,
|
||||
@@ -15,3 +56,16 @@ export function rpcError(
|
||||
const data: RpcErrorData = { reason, ...extra };
|
||||
return { code, message, data };
|
||||
}
|
||||
|
||||
export function transferError(
|
||||
code: ErrorCode,
|
||||
reason: RpcErrorReason,
|
||||
message: string,
|
||||
options: TransferErrorOptions,
|
||||
): RpcError {
|
||||
return rpcError(code, reason, message, {
|
||||
effect_state: options.effectState,
|
||||
phase: options.phase,
|
||||
...(options.cleanupState ? { cleanup_state: options.cleanupState } : {}),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -0,0 +1,319 @@
|
||||
// One native file-drop transaction bound to an explicit page target. CDP drag
|
||||
// events are sent to the target that owns the node: root targets use top-level
|
||||
// coordinates, while OOPIF targets require coordinates local to their session.
|
||||
|
||||
import type { RpcError, TransferEffectState } from "@/transport/types";
|
||||
import { transferError } from "./errors";
|
||||
import { resolveNodeGeometry } from "./frame-geometry";
|
||||
import type { ResolvedActionTarget } from "./interaction";
|
||||
import { type CdpRunner, isRpcError, sendToCdpTarget } from "./shared";
|
||||
import { BoundedWaitError, waitBounded } from "./transfer-transaction";
|
||||
|
||||
const CLEANUP_TIMEOUT_MS = 1_000;
|
||||
|
||||
type FileDropPhase = "resolve_target" | "drag_enter" | "drag_over" | "drop" | "cleanup";
|
||||
|
||||
interface RuntimeReply {
|
||||
result?: { value?: unknown; objectId?: string };
|
||||
exceptionDetails?: { text?: string; exception?: { description?: string } };
|
||||
}
|
||||
|
||||
interface DragData {
|
||||
items: [];
|
||||
files: string[];
|
||||
dragOperationsMask: number;
|
||||
}
|
||||
|
||||
export interface FileDropTransactionOptions {
|
||||
cdp: CdpRunner;
|
||||
actionTarget: ResolvedActionTarget;
|
||||
files: string[];
|
||||
timeoutMs: number;
|
||||
signal?: AbortSignal;
|
||||
withOverlayHidden: <T>(operation: () => Promise<T>) => Promise<T>;
|
||||
}
|
||||
|
||||
export interface FileDropTransactionResult {
|
||||
x: number;
|
||||
y: number;
|
||||
}
|
||||
|
||||
function failure(
|
||||
code: RpcError["code"],
|
||||
reason: "upload_mechanism_unsupported" | "file_drop_target_unavailable" | "file_drop_failed",
|
||||
message: string,
|
||||
effectState: TransferEffectState,
|
||||
phase: FileDropPhase,
|
||||
): RpcError {
|
||||
const error = transferError(code, reason, message, { effectState, phase });
|
||||
error.data = { ...error.data, mechanism: "drop" };
|
||||
return error;
|
||||
}
|
||||
|
||||
function enrichFailure(
|
||||
error: RpcError,
|
||||
effectState: TransferEffectState,
|
||||
phase: FileDropPhase,
|
||||
): RpcError {
|
||||
return {
|
||||
...error,
|
||||
data: {
|
||||
...error.data,
|
||||
mechanism: "drop",
|
||||
effect_state: effectState,
|
||||
phase,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function messageOf(error: unknown): string {
|
||||
return error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
|
||||
function methodUnavailable(error: unknown): boolean {
|
||||
return /(?:method|command).*(?:not found|wasn't found|unsupported)|wasn't found|-32601/i.test(
|
||||
messageOf(error),
|
||||
);
|
||||
}
|
||||
|
||||
function runtimeError(reply: RuntimeReply, fallback: string): Error | null {
|
||||
if (!reply.exceptionDetails) return null;
|
||||
return new Error(
|
||||
reply.exceptionDetails.exception?.description ?? reply.exceptionDetails.text ?? fallback,
|
||||
);
|
||||
}
|
||||
|
||||
async function pointBelongsToTarget(
|
||||
options: FileDropTransactionOptions,
|
||||
x: number,
|
||||
y: number,
|
||||
objectGroup: string,
|
||||
): Promise<boolean> {
|
||||
const target = options.actionTarget.cdpTarget;
|
||||
const hit = await sendToCdpTarget<{ backendNodeId?: number; frameId?: string }>(
|
||||
options.cdp,
|
||||
target,
|
||||
"DOM.getNodeForLocation",
|
||||
{ x: Math.round(x), y: Math.round(y), includeUserAgentShadowDOM: true },
|
||||
);
|
||||
if (typeof hit.backendNodeId !== "number") return false;
|
||||
if (options.actionTarget.frameId && hit.frameId !== options.actionTarget.frameId) return false;
|
||||
|
||||
const [resolvedTarget, resolvedHit] = await Promise.all([
|
||||
sendToCdpTarget<{ object?: { objectId?: string } }>(options.cdp, target, "DOM.resolveNode", {
|
||||
backendNodeId: options.actionTarget.backendNodeId,
|
||||
objectGroup,
|
||||
}),
|
||||
sendToCdpTarget<{ object?: { objectId?: string } }>(options.cdp, target, "DOM.resolveNode", {
|
||||
backendNodeId: hit.backendNodeId,
|
||||
objectGroup,
|
||||
}),
|
||||
]);
|
||||
const targetObjectId = resolvedTarget.object?.objectId;
|
||||
const hitObjectId = resolvedHit.object?.objectId;
|
||||
if (!targetObjectId || !hitObjectId) return false;
|
||||
|
||||
const reply = await sendToCdpTarget<RuntimeReply>(options.cdp, target, "Runtime.callFunctionOn", {
|
||||
objectId: targetObjectId,
|
||||
functionDeclaration: `function(hit) {
|
||||
for (let node = hit; node; ) {
|
||||
if (node === this) return true;
|
||||
if (this.contains?.(node)) return true;
|
||||
const root = node.getRootNode?.();
|
||||
node = node.parentNode || (root && root !== node ? root.host : null);
|
||||
}
|
||||
return false;
|
||||
}`,
|
||||
arguments: [{ objectId: hitObjectId }],
|
||||
returnByValue: true,
|
||||
});
|
||||
const error = runtimeError(reply, "failed to verify file-drop target");
|
||||
if (error) throw error;
|
||||
return reply.result?.value === true;
|
||||
}
|
||||
|
||||
export async function uploadThroughFileDrop(
|
||||
options: FileDropTransactionOptions,
|
||||
): Promise<FileDropTransactionResult | RpcError> {
|
||||
const deadline = Date.now() + options.timeoutMs;
|
||||
const target = options.actionTarget.cdpTarget;
|
||||
const objectGroup = `bsk-file-drop-${crypto.randomUUID()}`;
|
||||
const dragData: DragData = {
|
||||
items: [],
|
||||
files: options.files,
|
||||
dragOperationsMask: 1,
|
||||
};
|
||||
let dragStarted = false;
|
||||
let dropSent = false;
|
||||
let dragPoint = { x: 0, y: 0 };
|
||||
let cleanupFailed = false;
|
||||
let outcome: FileDropTransactionResult | RpcError = failure(
|
||||
"protocol_error",
|
||||
"file_drop_failed",
|
||||
"file-drop transaction ended without an outcome",
|
||||
"none",
|
||||
"cleanup",
|
||||
);
|
||||
|
||||
try {
|
||||
const geometry = await waitBounded(
|
||||
resolveNodeGeometry(
|
||||
options.cdp,
|
||||
options.actionTarget.tab.tabId,
|
||||
{
|
||||
target,
|
||||
backendNodeId: options.actionTarget.backendNodeId,
|
||||
...(options.actionTarget.frameId ? { frameId: options.actionTarget.frameId } : {}),
|
||||
},
|
||||
{ scrollIntoView: true },
|
||||
),
|
||||
deadline,
|
||||
options.signal,
|
||||
"resolving file-drop target timed out",
|
||||
);
|
||||
if (isRpcError(geometry)) {
|
||||
outcome = enrichFailure(geometry, "none", "resolve_target");
|
||||
return outcome;
|
||||
}
|
||||
const point = geometry.targetActionPoint;
|
||||
dragPoint = point;
|
||||
|
||||
outcome = await options.withOverlayHidden(async () => {
|
||||
try {
|
||||
try {
|
||||
const belongs = await waitBounded(
|
||||
pointBelongsToTarget(options, point.x, point.y, objectGroup),
|
||||
deadline,
|
||||
options.signal,
|
||||
"verifying file-drop target timed out",
|
||||
);
|
||||
if (!belongs) {
|
||||
return failure(
|
||||
"permission_denied",
|
||||
"file_drop_target_unavailable",
|
||||
"file-drop target is not the topmost element at its action point",
|
||||
"none",
|
||||
"resolve_target",
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
return failure(
|
||||
error instanceof BoundedWaitError ? "timeout" : "cdp_failed",
|
||||
"file_drop_target_unavailable",
|
||||
messageOf(error),
|
||||
"none",
|
||||
"resolve_target",
|
||||
);
|
||||
}
|
||||
|
||||
for (const [type, phase] of [
|
||||
["dragEnter", "drag_enter"],
|
||||
["dragOver", "drag_over"],
|
||||
] as const) {
|
||||
try {
|
||||
if (type === "dragEnter") dragStarted = true;
|
||||
await waitBounded(
|
||||
sendToCdpTarget(options.cdp, target, "Input.dispatchDragEvent", {
|
||||
type,
|
||||
x: point.x,
|
||||
y: point.y,
|
||||
data: dragData,
|
||||
}),
|
||||
deadline,
|
||||
options.signal,
|
||||
`${type} timed out`,
|
||||
);
|
||||
} catch (error) {
|
||||
const unsupported = methodUnavailable(error);
|
||||
if (type === "dragEnter" && unsupported) dragStarted = false;
|
||||
return failure(
|
||||
unsupported
|
||||
? "unsupported"
|
||||
: error instanceof BoundedWaitError
|
||||
? "timeout"
|
||||
: "cdp_failed",
|
||||
unsupported ? "upload_mechanism_unsupported" : "file_drop_failed",
|
||||
messageOf(error),
|
||||
"none",
|
||||
phase,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
dropSent = true;
|
||||
await waitBounded(
|
||||
sendToCdpTarget(options.cdp, target, "Input.dispatchDragEvent", {
|
||||
type: "drop",
|
||||
x: point.x,
|
||||
y: point.y,
|
||||
data: dragData,
|
||||
}),
|
||||
deadline,
|
||||
options.signal,
|
||||
"file drop timed out",
|
||||
);
|
||||
return { x: point.x, y: point.y };
|
||||
} catch (error) {
|
||||
outcome = failure(
|
||||
error instanceof BoundedWaitError ? "timeout" : "cdp_failed",
|
||||
"file_drop_failed",
|
||||
messageOf(error),
|
||||
"unknown",
|
||||
"drop",
|
||||
);
|
||||
return outcome;
|
||||
}
|
||||
} finally {
|
||||
if (dragStarted && !dropSent) {
|
||||
try {
|
||||
await waitBounded(
|
||||
sendToCdpTarget(options.cdp, target, "Input.dispatchDragEvent", {
|
||||
type: "dragCancel",
|
||||
x: dragPoint.x,
|
||||
y: dragPoint.y,
|
||||
data: dragData,
|
||||
}),
|
||||
Date.now() + CLEANUP_TIMEOUT_MS,
|
||||
undefined,
|
||||
"cancelling file drag timed out",
|
||||
);
|
||||
} catch {
|
||||
cleanupFailed = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
return outcome;
|
||||
} catch (error) {
|
||||
outcome = failure(
|
||||
error instanceof BoundedWaitError ? "timeout" : "cdp_failed",
|
||||
"file_drop_target_unavailable",
|
||||
messageOf(error),
|
||||
"none",
|
||||
"resolve_target",
|
||||
);
|
||||
return outcome;
|
||||
} finally {
|
||||
try {
|
||||
await waitBounded(
|
||||
sendToCdpTarget(options.cdp, target, "Runtime.releaseObjectGroup", { objectGroup }),
|
||||
Date.now() + CLEANUP_TIMEOUT_MS,
|
||||
undefined,
|
||||
"releasing file-drop object group timed out",
|
||||
);
|
||||
} catch {
|
||||
cleanupFailed = true;
|
||||
}
|
||||
|
||||
const effect = isRpcError(outcome)
|
||||
? (outcome.data?.effect_state as TransferEffectState | undefined)
|
||||
: "committed";
|
||||
if (effect === "unknown" || cleanupFailed) {
|
||||
await options.cdp.detach?.(target.tabId);
|
||||
}
|
||||
if (cleanupFailed && isRpcError(outcome)) {
|
||||
outcome.data = { ...outcome.data, cleanup_state: "failed" };
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,445 @@
|
||||
// One upload transaction with an explicit browser-side commit boundary.
|
||||
// Chrome interception prevents a native chooser from escaping automation;
|
||||
// chooser events and a frame-scoped DOM probe are independent input-location
|
||||
// signals, so event delivery is not required for standard file inputs.
|
||||
|
||||
import type { ClickResult, RpcError, TransferEffectState } from "@/transport/types";
|
||||
import { transferError } from "./errors";
|
||||
import type { ResolvedActionTarget } from "./interaction";
|
||||
import { type CdpRunner, isRpcError, sendToCdpTarget } from "./shared";
|
||||
import { BoundedWaitError, remainingMs, waitBounded } from "./transfer-transaction";
|
||||
|
||||
const CLEANUP_TIMEOUT_MS = 1_000;
|
||||
const ACTIVATION_GRACE_MS = 1_000;
|
||||
const PROBE_INTERVAL_MS = 20;
|
||||
|
||||
type UploadPhase =
|
||||
| "arm_interception"
|
||||
| "arm_input_probe"
|
||||
| "trigger"
|
||||
| "resolve_input"
|
||||
| "set_files"
|
||||
| "cleanup";
|
||||
|
||||
interface RuntimeReply {
|
||||
result?: { value?: unknown; objectId?: string };
|
||||
exceptionDetails?: { text?: string; exception?: { description?: string } };
|
||||
}
|
||||
|
||||
interface ChooserEvent {
|
||||
frameId?: string;
|
||||
backendNodeId?: number;
|
||||
mode?: "selectSingle" | "selectMultiple";
|
||||
}
|
||||
|
||||
export interface FileInputTransactionOptions {
|
||||
cdp: CdpRunner;
|
||||
actionTarget: ResolvedActionTarget;
|
||||
files: string[];
|
||||
timeoutMs: number;
|
||||
signal?: AbortSignal;
|
||||
trigger(): Promise<ClickResult | RpcError>;
|
||||
}
|
||||
|
||||
export interface FileInputTransactionResult {
|
||||
click: ClickResult;
|
||||
multiple: boolean;
|
||||
}
|
||||
|
||||
function runtimeError(reply: RuntimeReply, fallback: string): Error | null {
|
||||
if (!reply.exceptionDetails) return null;
|
||||
return new Error(
|
||||
reply.exceptionDetails.exception?.description ?? reply.exceptionDetails.text ?? fallback,
|
||||
);
|
||||
}
|
||||
|
||||
function transferFailure(
|
||||
code: RpcError["code"],
|
||||
reason: "file_input_probe_failed" | "file_input_not_activated" | "set_file_input_failed",
|
||||
message: string,
|
||||
effectState: TransferEffectState,
|
||||
phase: UploadPhase,
|
||||
): RpcError {
|
||||
return transferError(code, reason, message, { effectState, phase });
|
||||
}
|
||||
|
||||
function enrichFailure(
|
||||
error: RpcError,
|
||||
effectState: TransferEffectState,
|
||||
phase: UploadPhase,
|
||||
): RpcError {
|
||||
return {
|
||||
...error,
|
||||
data: { ...error.data, effect_state: effectState, phase },
|
||||
};
|
||||
}
|
||||
|
||||
function sameTarget(
|
||||
source: { tabId?: number; sessionId?: string },
|
||||
target: { tabId: number; sessionId?: string },
|
||||
): boolean {
|
||||
return source.tabId === target.tabId && source.sessionId === target.sessionId;
|
||||
}
|
||||
|
||||
async function delay(ms: number, signal?: AbortSignal): Promise<void> {
|
||||
await waitBounded(
|
||||
new Promise((resolve) => setTimeout(resolve, ms)),
|
||||
Date.now() + ms + 1,
|
||||
signal,
|
||||
"upload activation probe timed out",
|
||||
);
|
||||
}
|
||||
|
||||
async function callOnTrigger<T = RuntimeReply>(
|
||||
options: FileInputTransactionOptions,
|
||||
objectId: string,
|
||||
functionDeclaration: string,
|
||||
args: unknown[],
|
||||
returnByValue: boolean,
|
||||
): Promise<T> {
|
||||
return sendToCdpTarget<T>(options.cdp, options.actionTarget.cdpTarget, "Runtime.callFunctionOn", {
|
||||
objectId,
|
||||
functionDeclaration,
|
||||
arguments: args.map((value) => ({ value })),
|
||||
returnByValue,
|
||||
awaitPromise: false,
|
||||
});
|
||||
}
|
||||
|
||||
export async function uploadThroughActivatedFileInput(
|
||||
options: FileInputTransactionOptions,
|
||||
): Promise<FileInputTransactionResult | RpcError> {
|
||||
const deadline = Date.now() + options.timeoutMs;
|
||||
const target = options.actionTarget.cdpTarget;
|
||||
const objectGroup = `bsk-upload-${crypto.randomUUID()}`;
|
||||
const stateKey = `__bskUpload_${crypto.randomUUID().replaceAll("-", "")}`;
|
||||
const chooserEvents: ChooserEvent[] = [];
|
||||
let interceptionArmed = false;
|
||||
let probeArmed = false;
|
||||
let triggerObjectId: string | undefined;
|
||||
let outcome: FileInputTransactionResult | RpcError = transferFailure(
|
||||
"protocol_error",
|
||||
"file_input_probe_failed",
|
||||
"upload transaction ended without an outcome",
|
||||
"none",
|
||||
"cleanup",
|
||||
);
|
||||
|
||||
const chooserSubscription = options.cdp.onEvent?.((source, method, raw) => {
|
||||
if (method !== "Page.fileChooserOpened" || !sameTarget(source, target)) return;
|
||||
const event = raw as ChooserEvent;
|
||||
chooserEvents.push(event);
|
||||
});
|
||||
|
||||
try {
|
||||
try {
|
||||
await waitBounded(
|
||||
sendToCdpTarget(options.cdp, target, "Page.setInterceptFileChooserDialog", {
|
||||
enabled: true,
|
||||
}),
|
||||
deadline,
|
||||
options.signal,
|
||||
"arming native file chooser interception timed out",
|
||||
);
|
||||
interceptionArmed = true;
|
||||
} catch (err) {
|
||||
outcome = transferFailure(
|
||||
err instanceof BoundedWaitError ? "timeout" : "cdp_failed",
|
||||
"file_input_probe_failed",
|
||||
err instanceof Error ? err.message : String(err),
|
||||
"none",
|
||||
"arm_interception",
|
||||
);
|
||||
return outcome;
|
||||
}
|
||||
|
||||
try {
|
||||
const resolved = await waitBounded(
|
||||
sendToCdpTarget<{ object?: { objectId?: string } }>(
|
||||
options.cdp,
|
||||
target,
|
||||
"DOM.resolveNode",
|
||||
{
|
||||
backendNodeId: options.actionTarget.backendNodeId,
|
||||
objectGroup,
|
||||
},
|
||||
),
|
||||
deadline,
|
||||
options.signal,
|
||||
"resolving upload trigger timed out",
|
||||
);
|
||||
triggerObjectId = resolved.object?.objectId;
|
||||
if (!triggerObjectId) throw new Error("DOM.resolveNode returned no trigger objectId");
|
||||
|
||||
const armed = await waitBounded(
|
||||
callOnTrigger<RuntimeReply>(
|
||||
options,
|
||||
triggerObjectId,
|
||||
`function(key) {
|
||||
const doc = this.ownerDocument;
|
||||
const owner = doc.defaultView;
|
||||
if (!owner) return false;
|
||||
const state = { inputs: [], listener: null };
|
||||
Object.defineProperty(owner, key, { value: state, configurable: true });
|
||||
state.listener = event => {
|
||||
const path = typeof event.composedPath === "function" ? event.composedPath() : [];
|
||||
const candidate = path[0] || event.target;
|
||||
if (candidate && candidate.nodeType === 1 &&
|
||||
candidate.localName === "input" && candidate.type === "file") {
|
||||
if (!state.inputs.includes(candidate)) state.inputs.push(candidate);
|
||||
event.preventDefault();
|
||||
}
|
||||
};
|
||||
doc.addEventListener("click", state.listener, true);
|
||||
return true;
|
||||
}`,
|
||||
[stateKey],
|
||||
true,
|
||||
),
|
||||
deadline,
|
||||
options.signal,
|
||||
"arming frame-scoped file input probe timed out",
|
||||
);
|
||||
const armError = runtimeError(armed, "failed to arm file input probe");
|
||||
if (armError || armed.result?.value !== true) {
|
||||
throw armError ?? new Error("file input probe did not arm");
|
||||
}
|
||||
probeArmed = true;
|
||||
} catch (err) {
|
||||
outcome = transferFailure(
|
||||
err instanceof BoundedWaitError ? "timeout" : "cdp_failed",
|
||||
"file_input_probe_failed",
|
||||
err instanceof Error ? err.message : String(err),
|
||||
"none",
|
||||
"arm_input_probe",
|
||||
);
|
||||
return outcome;
|
||||
}
|
||||
|
||||
let click: ClickResult | RpcError;
|
||||
try {
|
||||
click = await waitBounded(
|
||||
options.trigger(),
|
||||
deadline,
|
||||
options.signal,
|
||||
"upload trigger timed out",
|
||||
);
|
||||
} catch (err) {
|
||||
outcome = transferFailure(
|
||||
err instanceof BoundedWaitError ? "timeout" : "cdp_failed",
|
||||
"file_input_probe_failed",
|
||||
err instanceof Error ? err.message : String(err),
|
||||
"none",
|
||||
"trigger",
|
||||
);
|
||||
return outcome;
|
||||
}
|
||||
if (isRpcError(click)) {
|
||||
outcome = enrichFailure(click, "none", "trigger");
|
||||
return outcome;
|
||||
}
|
||||
|
||||
try {
|
||||
const activationDeadline = Math.min(deadline, Date.now() + ACTIVATION_GRACE_MS);
|
||||
let summary: { count: number; multiple: boolean } = { count: 0, multiple: false };
|
||||
while (Date.now() < activationDeadline) {
|
||||
if (chooserEvents.length > 0) break;
|
||||
const reply = await callOnTrigger<RuntimeReply>(
|
||||
options,
|
||||
triggerObjectId,
|
||||
`function(key) {
|
||||
const state = this.ownerDocument.defaultView?.[key];
|
||||
return state
|
||||
? { count: state.inputs.length, multiple: state.inputs[0]?.multiple === true }
|
||||
: { count: 0, multiple: false };
|
||||
}`,
|
||||
[stateKey],
|
||||
true,
|
||||
);
|
||||
const summaryError = runtimeError(reply, "failed to inspect activated file input");
|
||||
if (summaryError) throw summaryError;
|
||||
const value = reply.result?.value as { count?: unknown; multiple?: unknown } | undefined;
|
||||
summary = {
|
||||
count: typeof value?.count === "number" ? value.count : 0,
|
||||
multiple: value?.multiple === true,
|
||||
};
|
||||
if (summary.count > 0) break;
|
||||
await delay(Math.min(PROBE_INTERVAL_MS, remainingMs(activationDeadline)), options.signal);
|
||||
}
|
||||
|
||||
if (chooserEvents.length > 1) {
|
||||
outcome = transferFailure(
|
||||
"unsupported",
|
||||
"file_input_not_activated",
|
||||
"upload trigger activated more than one file chooser",
|
||||
"none",
|
||||
"resolve_input",
|
||||
);
|
||||
return outcome;
|
||||
}
|
||||
|
||||
const chooser = chooserEvents[0];
|
||||
let backendNodeId: number | undefined;
|
||||
let multiple = summary.multiple;
|
||||
if (chooser) {
|
||||
if (options.actionTarget.frameId && chooser.frameId !== options.actionTarget.frameId) {
|
||||
outcome = transferFailure(
|
||||
"unsupported",
|
||||
"file_input_not_activated",
|
||||
"upload trigger activated a file chooser in a different frame",
|
||||
"none",
|
||||
"resolve_input",
|
||||
);
|
||||
return outcome;
|
||||
}
|
||||
if (typeof chooser.backendNodeId !== "number") {
|
||||
outcome = transferFailure(
|
||||
"unsupported",
|
||||
"file_input_not_activated",
|
||||
"upload trigger invoked a non-input file picker",
|
||||
"none",
|
||||
"resolve_input",
|
||||
);
|
||||
return outcome;
|
||||
}
|
||||
backendNodeId = chooser.backendNodeId;
|
||||
multiple = chooser.mode === "selectMultiple";
|
||||
} else {
|
||||
if (summary.count !== 1) {
|
||||
outcome = transferFailure(
|
||||
"unsupported",
|
||||
"file_input_not_activated",
|
||||
summary.count === 0
|
||||
? "upload trigger did not activate an input[type=file]"
|
||||
: "upload trigger activated more than one input[type=file]",
|
||||
"none",
|
||||
"resolve_input",
|
||||
);
|
||||
return outcome;
|
||||
}
|
||||
const input = await callOnTrigger<RuntimeReply>(
|
||||
options,
|
||||
triggerObjectId,
|
||||
`function(key) { return this.ownerDocument.defaultView?.[key]?.inputs[0]; }`,
|
||||
[stateKey],
|
||||
false,
|
||||
);
|
||||
const inputError = runtimeError(input, "failed to resolve activated file input object");
|
||||
if (inputError) throw inputError;
|
||||
if (!input.result?.objectId) throw new Error("activated file input returned no objectId");
|
||||
const described = await sendToCdpTarget<{ node?: { backendNodeId?: number } }>(
|
||||
options.cdp,
|
||||
target,
|
||||
"DOM.describeNode",
|
||||
{ objectId: input.result.objectId },
|
||||
);
|
||||
backendNodeId = described.node?.backendNodeId;
|
||||
if (typeof backendNodeId !== "number") {
|
||||
throw new Error("DOM.describeNode returned no file input backendNodeId");
|
||||
}
|
||||
}
|
||||
|
||||
if (!multiple && options.files.length !== 1) {
|
||||
outcome = enrichFailure(
|
||||
{ code: "invalid_params", message: "file input accepts exactly one file" },
|
||||
"none",
|
||||
"resolve_input",
|
||||
);
|
||||
return outcome;
|
||||
}
|
||||
|
||||
try {
|
||||
await waitBounded(
|
||||
sendToCdpTarget(options.cdp, target, "DOM.setFileInputFiles", {
|
||||
files: options.files,
|
||||
backendNodeId,
|
||||
}),
|
||||
deadline,
|
||||
options.signal,
|
||||
"setting file input files timed out",
|
||||
);
|
||||
} catch (err) {
|
||||
outcome = transferFailure(
|
||||
err instanceof BoundedWaitError ? "timeout" : "cdp_failed",
|
||||
"set_file_input_failed",
|
||||
err instanceof Error ? err.message : String(err),
|
||||
"unknown",
|
||||
"set_files",
|
||||
);
|
||||
return outcome;
|
||||
}
|
||||
outcome = { click, multiple };
|
||||
return outcome;
|
||||
} catch (err) {
|
||||
outcome = transferFailure(
|
||||
err instanceof BoundedWaitError ? "timeout" : "cdp_failed",
|
||||
"file_input_probe_failed",
|
||||
err instanceof Error ? err.message : String(err),
|
||||
"none",
|
||||
"resolve_input",
|
||||
);
|
||||
return outcome;
|
||||
}
|
||||
} finally {
|
||||
chooserSubscription?.dispose();
|
||||
let cleanupFailed = false;
|
||||
if (probeArmed && triggerObjectId) {
|
||||
try {
|
||||
await waitBounded(
|
||||
callOnTrigger(
|
||||
options,
|
||||
triggerObjectId,
|
||||
`function(key) {
|
||||
const owner = this.ownerDocument.defaultView;
|
||||
const state = owner?.[key];
|
||||
if (state?.listener) this.ownerDocument.removeEventListener("click", state.listener, true);
|
||||
if (owner) delete owner[key];
|
||||
}`,
|
||||
[stateKey],
|
||||
true,
|
||||
),
|
||||
Date.now() + CLEANUP_TIMEOUT_MS,
|
||||
undefined,
|
||||
"cleaning file input probe timed out",
|
||||
);
|
||||
} catch {
|
||||
cleanupFailed = true;
|
||||
}
|
||||
}
|
||||
if (interceptionArmed) {
|
||||
try {
|
||||
await waitBounded(
|
||||
sendToCdpTarget(options.cdp, target, "Page.setInterceptFileChooserDialog", {
|
||||
enabled: false,
|
||||
cancel: true,
|
||||
}),
|
||||
Date.now() + CLEANUP_TIMEOUT_MS,
|
||||
undefined,
|
||||
"disabling file chooser interception timed out",
|
||||
);
|
||||
} catch {
|
||||
cleanupFailed = true;
|
||||
}
|
||||
}
|
||||
try {
|
||||
await waitBounded(
|
||||
sendToCdpTarget(options.cdp, target, "Runtime.releaseObjectGroup", { objectGroup }),
|
||||
Date.now() + CLEANUP_TIMEOUT_MS,
|
||||
undefined,
|
||||
"releasing upload object group timed out",
|
||||
);
|
||||
} catch {
|
||||
cleanupFailed = true;
|
||||
}
|
||||
|
||||
const effect = isRpcError(outcome)
|
||||
? (outcome.data?.effect_state as TransferEffectState | undefined)
|
||||
: "committed";
|
||||
if (effect === "unknown" || cleanupFailed) {
|
||||
await options.cdp.detach?.(target.tabId);
|
||||
}
|
||||
if (cleanupFailed && isRpcError(outcome)) {
|
||||
outcome.data = { ...outcome.data, cleanup_state: "failed" };
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -29,7 +29,10 @@ export interface NodeAddress {
|
||||
export interface ResolvedNodeGeometry {
|
||||
topVisibleRegions: Region;
|
||||
topBounds: ViewportRect;
|
||||
/** Point in the top-level tab viewport, used by root-target input events. */
|
||||
actionPoint: Point;
|
||||
/** Point in the addressed CDP target's viewport, used by OOPIF-local input events. */
|
||||
targetActionPoint: Point;
|
||||
}
|
||||
|
||||
function geometryError(message: string): RpcError {
|
||||
@@ -262,11 +265,30 @@ export async function resolveNodeGeometry(
|
||||
if (isRpcError(localRegion)) return localRegion;
|
||||
|
||||
let topVisibleRegions: Region;
|
||||
let targetActionPoint: Point | null = null;
|
||||
if (address.frameId && graph) {
|
||||
const projection = await resolveFrameProjection(cdp, graph, address.frameId);
|
||||
if (!projection)
|
||||
return geometryError(`could not resolve frame geometry for ${address.frameId}`);
|
||||
topVisibleRegions = projectRegionToViewport(localRegion, projection);
|
||||
if (address.target.sessionId) {
|
||||
const localViewport = projection.edges[0]?.sourceViewport ?? projection.topViewport;
|
||||
const localVisibleRegions = localRegion
|
||||
.map((polygon) =>
|
||||
clipPolygon(
|
||||
polygon,
|
||||
rectPolygon({
|
||||
x: 0,
|
||||
y: 0,
|
||||
w: localViewport.width,
|
||||
h: localViewport.height,
|
||||
}),
|
||||
),
|
||||
)
|
||||
.filter((polygon) => polygon.length >= 3);
|
||||
const localActionRegion = largestRegion(localVisibleRegions);
|
||||
targetActionPoint = localActionRegion ? polygonCentroid(localActionRegion) : null;
|
||||
}
|
||||
} else {
|
||||
const viewport = await targetViewport(cdp, address.target);
|
||||
if (!viewport) return geometryError("could not resolve top viewport geometry");
|
||||
@@ -283,7 +305,11 @@ export async function resolveNodeGeometry(
|
||||
if (!topBounds || !actionPoint) {
|
||||
return { code: "permission_denied", message: "element not visible" };
|
||||
}
|
||||
return { topVisibleRegions, topBounds, actionPoint };
|
||||
if (address.target.sessionId && !targetActionPoint) {
|
||||
return { code: "permission_denied", message: "element not visible in its target" };
|
||||
}
|
||||
targetActionPoint ??= actionPoint;
|
||||
return { topVisibleRegions, topBounds, actionPoint, targetActionPoint };
|
||||
} catch (error) {
|
||||
return geometryError(error instanceof Error ? error.message : String(error));
|
||||
}
|
||||
|
||||
@@ -40,6 +40,7 @@ import {
|
||||
enforceAgentWindow,
|
||||
isRpcError,
|
||||
lookupSession,
|
||||
type ResolvedTargetTab,
|
||||
resolveTargetTab,
|
||||
} from "./shared";
|
||||
import { resolveSnapshotRef } from "./snapshot-ref";
|
||||
@@ -56,6 +57,15 @@ export interface InteractionDeps {
|
||||
keepOverlayBypassAfterHover?: boolean;
|
||||
}
|
||||
|
||||
export interface ResolvedActionTarget {
|
||||
tab: ResolvedTargetTab;
|
||||
backendNodeId: number;
|
||||
cdpTarget: CdpTarget;
|
||||
frameId?: string;
|
||||
usedRef?: string;
|
||||
usedSelector?: string;
|
||||
}
|
||||
|
||||
const DEFAULT_TIMEOUT_MS = 30_000;
|
||||
const DEFAULT_HOVER_SETTLE_MS = 200;
|
||||
|
||||
@@ -128,7 +138,7 @@ async function wait(ms: number, signal?: AbortSignal): Promise<void> {
|
||||
* `RpcError` if the caller supplied neither (or both), or if neither
|
||||
* lookup matched.
|
||||
*/
|
||||
async function resolveBackendNode(
|
||||
export async function resolveBackendNode(
|
||||
cdp: CdpRunner,
|
||||
ctx: SessionContext,
|
||||
target: { tabId: number },
|
||||
@@ -173,6 +183,8 @@ async function resolveBackendNode(
|
||||
}
|
||||
// selector path
|
||||
try {
|
||||
// Selector lookup itself attaches CDP, even when no element is found.
|
||||
cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
const doc = await cdp.send<{ root?: { nodeId?: number } }>(target.tabId, "DOM.getDocument", {
|
||||
depth: 0,
|
||||
});
|
||||
@@ -215,6 +227,17 @@ async function resolveBackendNode(
|
||||
}
|
||||
}
|
||||
|
||||
export async function resolveActionTarget(
|
||||
cdp: CdpRunner,
|
||||
ctx: SessionContext,
|
||||
target: ResolvedTargetTab,
|
||||
params: { ref?: string; selector?: string },
|
||||
toolName: string,
|
||||
): Promise<ResolvedActionTarget | RpcError> {
|
||||
const node = await resolveBackendNode(cdp, ctx, target, params, toolName);
|
||||
return isRpcError(node) ? node : { tab: target, ...node };
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// tool.click
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -233,10 +256,19 @@ export async function handleClick(
|
||||
if (isRpcError(target)) return target;
|
||||
const denied = enforceAgentWindow(ctx, target, "click");
|
||||
if (denied) return denied;
|
||||
const dialogCursor = markDialogCursor(deps.cdp, target.tabId);
|
||||
const resolved = await resolveActionTarget(deps.cdp, ctx, target, params, "click");
|
||||
if (isRpcError(resolved)) return resolved;
|
||||
return clickResolvedTarget(ctx, resolved, params, deps);
|
||||
}
|
||||
|
||||
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "click");
|
||||
if (isRpcError(node)) return node;
|
||||
export async function clickResolvedTarget(
|
||||
ctx: SessionContext,
|
||||
resolved: ResolvedActionTarget,
|
||||
params: Pick<ClickParams, "button" | "click_count" | "modifiers">,
|
||||
deps: InteractionDeps,
|
||||
): Promise<ClickResult | RpcError> {
|
||||
const { tab: target } = resolved;
|
||||
const dialogCursor = markDialogCursor(deps.cdp, target.tabId);
|
||||
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "click aborted" };
|
||||
@@ -247,9 +279,9 @@ export async function handleClick(
|
||||
deps.cdp,
|
||||
target.tabId,
|
||||
{
|
||||
target: node.cdpTarget,
|
||||
backendNodeId: node.backendNodeId,
|
||||
...(node.frameId ? { frameId: node.frameId } : {}),
|
||||
target: resolved.cdpTarget,
|
||||
backendNodeId: resolved.backendNodeId,
|
||||
...(resolved.frameId ? { frameId: resolved.frameId } : {}),
|
||||
},
|
||||
{ scrollIntoView: true },
|
||||
);
|
||||
@@ -337,8 +369,8 @@ export async function handleClick(
|
||||
|
||||
return attachDialogs(deps.cdp, target.tabId, dialogCursor, {
|
||||
tab_id: target.tabId,
|
||||
used_ref: node.usedRef,
|
||||
used_selector: node.usedSelector,
|
||||
used_ref: resolved.usedRef,
|
||||
used_selector: resolved.usedSelector,
|
||||
x: centre.x,
|
||||
y: centre.y,
|
||||
});
|
||||
@@ -577,6 +609,53 @@ type SelectMutationResult =
|
||||
// tool.fill
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type FillFailureReason =
|
||||
| "target_not_fillable"
|
||||
| "fill_value_invalid"
|
||||
| "fill_target_changed"
|
||||
| "fill_focus_lost"
|
||||
| "fill_value_mismatch"
|
||||
| "fill_failed";
|
||||
|
||||
type FillPreparation =
|
||||
| { before: string; expected: string }
|
||||
| { reason: FillFailureReason; error: string };
|
||||
|
||||
type FillReadiness = "ready" | "background" | "fill_target_changed" | "fill_focus_lost";
|
||||
|
||||
function fillError(reason: FillFailureReason, message: string): RpcError {
|
||||
return rpcError(
|
||||
reason === "target_not_fillable" || reason === "fill_value_invalid"
|
||||
? "invalid_params"
|
||||
: "cdp_failed",
|
||||
reason,
|
||||
message,
|
||||
);
|
||||
}
|
||||
|
||||
interface FillScriptReply<T> {
|
||||
result?: { value?: T };
|
||||
exceptionDetails?: unknown;
|
||||
}
|
||||
|
||||
const FILL_EDITABLE_FUNCTION = `function() {
|
||||
const tag = this.tagName.toLowerCase();
|
||||
const supported = tag === 'input'
|
||||
? ['text', 'search', 'tel', 'url', 'email', 'password', 'number'].includes(this.type)
|
||||
: tag === 'textarea' || this.isContentEditable;
|
||||
return this.isConnected && supported && !this.readOnly && !this.matches(':disabled');
|
||||
}`;
|
||||
|
||||
// Chrome renders a trailing editable newline with an empty <div><br></div>.
|
||||
// innerText includes the padding break; it is not an extra typed character.
|
||||
const FILL_VALUE_FUNCTION = `function() {
|
||||
if (!this.isContentEditable) return this.value;
|
||||
const value = this.innerText;
|
||||
const tail = this.lastChild;
|
||||
const padding = tail && tail.nodeName === 'DIV' && tail.childNodes.length === 1 && tail.firstChild.nodeName === 'BR';
|
||||
return padding && value.endsWith('\\n\\n') ? value.slice(0, -1) : value;
|
||||
}`;
|
||||
|
||||
export async function handleFill(
|
||||
manager: SessionManager,
|
||||
params: FillParams,
|
||||
@@ -628,12 +707,8 @@ export async function handleFill(
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
await nodeCdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
|
||||
} catch (err) {
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
message: err instanceof Error ? err.message : String(err),
|
||||
};
|
||||
return fillError("fill_failed", err instanceof Error ? err.message : String(err));
|
||||
}
|
||||
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
@@ -646,57 +721,252 @@ export async function handleFill(
|
||||
const clearBefore = params.clear_before ?? true;
|
||||
|
||||
try {
|
||||
if (clearBefore) {
|
||||
// Clear input/textarea value or wipe contenteditable innerText,
|
||||
// then fire `input` so frameworks observe the empty state.
|
||||
await nodeCdp.send(target.tabId, "Runtime.callFunctionOn", {
|
||||
objectId,
|
||||
functionDeclaration: `function() {
|
||||
if (this.isContentEditable) { this.textContent = ''; }
|
||||
else {
|
||||
const proto = this instanceof HTMLTextAreaElement
|
||||
? HTMLTextAreaElement.prototype
|
||||
: HTMLInputElement.prototype;
|
||||
const descriptor = Object.getOwnPropertyDescriptor(proto, 'value');
|
||||
if (descriptor && descriptor.set) descriptor.set.call(this, '');
|
||||
else this.value = '';
|
||||
}
|
||||
this.dispatchEvent(new Event('input', { bubbles: true }));
|
||||
}`,
|
||||
returnByValue: true,
|
||||
});
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
try {
|
||||
await nodeCdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
|
||||
} catch (error) {
|
||||
// Chrome rejects DOM.focus for disabled controls, including inherited
|
||||
// fieldset state. Classify the live target before treating this as a
|
||||
// browser failure; the normal path needs no extra round trip.
|
||||
const editable = await nodeCdp.send<FillScriptReply<boolean>>(
|
||||
target.tabId,
|
||||
"Runtime.callFunctionOn",
|
||||
{ objectId, functionDeclaration: FILL_EDITABLE_FUNCTION, returnByValue: true },
|
||||
);
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
if (!editable.exceptionDetails && editable.result?.value === false) {
|
||||
return fillError(
|
||||
"target_not_fillable",
|
||||
"fill target is not editable or its input type is unsupported",
|
||||
);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
// Check editability before clearing. Keep the expected result tied to
|
||||
// this object, including the existing value on the append path.
|
||||
const prepared = await nodeCdp.send<FillScriptReply<FillPreparation>>(
|
||||
target.tabId,
|
||||
"Runtime.callFunctionOn",
|
||||
{
|
||||
objectId,
|
||||
functionDeclaration: `function(value, clearBefore) {
|
||||
const tag = this.tagName.toLowerCase();
|
||||
const native = tag === 'input' || tag === 'textarea';
|
||||
if (!(${FILL_EDITABLE_FUNCTION}).call(this)) return { reason: 'target_not_fillable', error: 'fill target is not editable or its input type is unsupported' };
|
||||
if (this.getRootNode().activeElement !== this) return { reason: 'fill_focus_lost', error: 'fill target does not have focus' };
|
||||
const before = clearBefore ? '' : (${FILL_VALUE_FUNCTION}).call(this);
|
||||
let expected = before + value;
|
||||
if (native) {
|
||||
// Use the browser's own value sanitization (e.g. textarea
|
||||
// line endings) without changing the live control.
|
||||
const normalizer = this.ownerDocument.createElement(tag);
|
||||
if (tag === 'input') {
|
||||
normalizer.type = this.type;
|
||||
normalizer.multiple = this.multiple;
|
||||
}
|
||||
// insertText treats line breaks in a single-line input as spaces.
|
||||
if (tag === 'input') expected = expected.replace(/\\r\\n|\\r|\\n/g, ' ');
|
||||
normalizer.value = expected;
|
||||
if (expected !== '' && normalizer.value === '' && this.type === 'number') {
|
||||
return { reason: 'fill_value_invalid', error: 'fill value is not valid for a number input' };
|
||||
}
|
||||
expected = normalizer.value;
|
||||
if (this.type !== 'number' && this.maxLength >= 0 && expected.length > this.maxLength) {
|
||||
return { reason: 'fill_value_invalid', error: 'fill value exceeds the target maxlength' };
|
||||
}
|
||||
} else {
|
||||
expected = expected.replace(/\\r\\n?/g, '\\n');
|
||||
}
|
||||
if (clearBefore) {
|
||||
if (native) {
|
||||
const proto = tag === 'textarea' ? HTMLTextAreaElement.prototype : HTMLInputElement.prototype;
|
||||
Object.getOwnPropertyDescriptor(proto, 'value').set.call(this, '');
|
||||
} else {
|
||||
this.textContent = '';
|
||||
}
|
||||
this.dispatchEvent(new Event('input', { bubbles: true }));
|
||||
}
|
||||
return { before, expected };
|
||||
}`,
|
||||
arguments: [{ value: params.value }, { value: clearBefore }],
|
||||
returnByValue: true,
|
||||
},
|
||||
);
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
if (prepared.exceptionDetails) {
|
||||
return fillError("fill_failed", "fill preparation script failed");
|
||||
}
|
||||
const preparation = prepared.result?.value;
|
||||
if (preparation && "error" in preparation) {
|
||||
return fillError(preparation.reason, preparation.error);
|
||||
}
|
||||
if (typeof preparation?.before !== "string" || typeof preparation?.expected !== "string") {
|
||||
return fillError("fill_failed", "fill preparation returned an unexpected result");
|
||||
}
|
||||
// A separate call runs after the clearing event's microtasks drain.
|
||||
// DOM focus works in background tabs. Use document.hasFocus() only to
|
||||
// detect deferred focus events, never to reject background input.
|
||||
const checkReady = async (): Promise<FillReadiness | RpcError> => {
|
||||
const reply = await nodeCdp.send<FillScriptReply<FillReadiness>>(
|
||||
target.tabId,
|
||||
"Runtime.callFunctionOn",
|
||||
{
|
||||
objectId,
|
||||
functionDeclaration: `function(before) {
|
||||
if (!(${FILL_EDITABLE_FUNCTION}).call(this) ||
|
||||
(${FILL_VALUE_FUNCTION}).call(this) !== before) return 'fill_target_changed';
|
||||
if (this.getRootNode().activeElement !== this) return 'fill_focus_lost';
|
||||
return this.ownerDocument.hasFocus() ? 'ready' : 'background';
|
||||
}`,
|
||||
arguments: [{ value: preparation.before }],
|
||||
returnByValue: true,
|
||||
},
|
||||
);
|
||||
const state = reply.result?.value;
|
||||
if (
|
||||
reply.exceptionDetails ||
|
||||
(state !== "ready" &&
|
||||
state !== "background" &&
|
||||
state !== "fill_target_changed" &&
|
||||
state !== "fill_focus_lost")
|
||||
) {
|
||||
return fillError(
|
||||
"fill_failed",
|
||||
"fill readiness script failed or returned an unexpected result",
|
||||
);
|
||||
}
|
||||
return state;
|
||||
};
|
||||
let ready = await checkReady();
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
// Restore focus once only if clearing left this same target editable
|
||||
// and unchanged. Recheck after focus handlers have run.
|
||||
if (clearBefore && ready === "fill_focus_lost") {
|
||||
await nodeCdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
ready = await checkReady();
|
||||
}
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
if (typeof ready !== "string") return ready;
|
||||
if (ready !== "ready" && ready !== "background") {
|
||||
return fillError(ready, "fill target changed or lost focus before typing");
|
||||
}
|
||||
if (params.value !== "" && (!clearBefore || ready === "background")) {
|
||||
// Native editing commands also support number/email inputs, whose
|
||||
// selection APIs cannot set a caret, and multiline contenteditables.
|
||||
// Also do this for background replacement: CDP input focuses the
|
||||
// renderer, delivering deferred focus events before it inserts text.
|
||||
// Use a command-only event so this does not invoke an End shortcut.
|
||||
try {
|
||||
await deps.cdp.send(target.tabId, "Input.dispatchKeyEvent", {
|
||||
type: "rawKeyDown",
|
||||
commands: ["moveToEndOfDocument"],
|
||||
});
|
||||
} finally {
|
||||
await deps.cdp.send(target.tabId, "Input.dispatchKeyEvent", { type: "keyUp" });
|
||||
}
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
// Focus and key handlers can change the target or move focus too.
|
||||
ready = await checkReady();
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
if (typeof ready !== "string") return ready;
|
||||
if (ready !== "ready" && ready !== "background") {
|
||||
return fillError(ready, "fill target changed or lost focus while positioning the caret");
|
||||
}
|
||||
}
|
||||
// CDP `Input.insertText` handles IME / multi-byte input out of the
|
||||
// box, much more reliably than per-key `dispatchKeyEvent`.
|
||||
await deps.cdp.send(target.tabId, "Input.insertText", { text: params.value });
|
||||
if (params.value !== "") {
|
||||
await deps.cdp.send(target.tabId, "Input.insertText", { text: params.value });
|
||||
}
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
// Fire `input` + `change` so React / Vue controlled inputs commit.
|
||||
await nodeCdp.send(target.tabId, "Runtime.callFunctionOn", {
|
||||
// Keep the existing notifications, then read in a separate call so
|
||||
// nested microtasks queued by the page cannot race the verification.
|
||||
const notified = await nodeCdp.send<FillScriptReply<unknown>>(
|
||||
target.tabId,
|
||||
"Runtime.callFunctionOn",
|
||||
{
|
||||
objectId,
|
||||
functionDeclaration: `function() {
|
||||
this.dispatchEvent(new Event('input', { bubbles: true }));
|
||||
this.dispatchEvent(new Event('change', { bubbles: true }));
|
||||
}`,
|
||||
returnByValue: true,
|
||||
},
|
||||
);
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
if (notified.exceptionDetails) {
|
||||
return fillError("fill_failed", "fill notification script failed");
|
||||
}
|
||||
const verified = await nodeCdp.send<
|
||||
FillScriptReply<{ connected: boolean; matches: boolean; valueLength: number }>
|
||||
>(target.tabId, "Runtime.callFunctionOn", {
|
||||
objectId,
|
||||
functionDeclaration: `function() {
|
||||
this.dispatchEvent(new Event('input', { bubbles: true }));
|
||||
this.dispatchEvent(new Event('change', { bubbles: true }));
|
||||
}`,
|
||||
functionDeclaration: `function(expected) {
|
||||
const value = (${FILL_VALUE_FUNCTION}).call(this);
|
||||
return { connected: this.isConnected, matches: value === expected, valueLength: value.length };
|
||||
}`,
|
||||
arguments: [{ value: preparation.expected }],
|
||||
returnByValue: true,
|
||||
});
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
const verification = verified.result?.value;
|
||||
if (
|
||||
verified.exceptionDetails ||
|
||||
typeof verification?.connected !== "boolean" ||
|
||||
typeof verification?.matches !== "boolean" ||
|
||||
typeof verification?.valueLength !== "number"
|
||||
) {
|
||||
return fillError(
|
||||
"fill_failed",
|
||||
"fill verification script failed or returned an unexpected result",
|
||||
);
|
||||
}
|
||||
if (!verification.connected) {
|
||||
return fillError("fill_target_changed", "fill target was removed or replaced during input");
|
||||
}
|
||||
if (!verification.matches) {
|
||||
return fillError(
|
||||
"fill_value_mismatch",
|
||||
"fill could not verify the expected value; observe the page before deciding whether to retry",
|
||||
);
|
||||
}
|
||||
return attachDialogs(deps.cdp, target.tabId, dialogCursor, {
|
||||
tab_id: target.tabId,
|
||||
used_ref: node.usedRef,
|
||||
used_selector: node.usedSelector,
|
||||
value_length: verification.valueLength,
|
||||
});
|
||||
} catch (err) {
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
message: err instanceof Error ? err.message : String(err),
|
||||
};
|
||||
return fillError("fill_failed", err instanceof Error ? err.message : String(err));
|
||||
} finally {
|
||||
await nodeCdp.send(target.tabId, "Runtime.releaseObject", { objectId }).catch(() => undefined);
|
||||
}
|
||||
|
||||
return attachDialogs(deps.cdp, target.tabId, dialogCursor, {
|
||||
tab_id: target.tabId,
|
||||
used_ref: node.usedRef,
|
||||
used_selector: node.usedSelector,
|
||||
value_length: params.value.length,
|
||||
});
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
@@ -28,7 +28,13 @@ import type {
|
||||
RpcError,
|
||||
WaitUntil,
|
||||
} from "@/transport/types";
|
||||
import {
|
||||
type BrowserNavigationApi,
|
||||
chromeBrowserNavigationApi,
|
||||
navigateWithBrowserApi,
|
||||
} from "./browser-navigation";
|
||||
import { attachDialogs, markDialogCursor } from "./dialogs";
|
||||
import { cdpError, isCdpExtensionAccessDenied } from "./errors";
|
||||
import {
|
||||
type CdpRunner,
|
||||
type ChromeTabsApi,
|
||||
@@ -42,6 +48,7 @@ import {
|
||||
export interface NavigationDeps {
|
||||
cdp: CdpRunner;
|
||||
tabsApi: ChromeTabsApi;
|
||||
browserNavigation?: BrowserNavigationApi;
|
||||
/** Optional AbortSignal — M7 abort hook (M10.2 will wire the full chain). */
|
||||
signal?: AbortSignal;
|
||||
/** Override default timeout when the caller omits `timeout_ms`. */
|
||||
@@ -479,6 +486,81 @@ async function readTabUrl(api: ChromeTabsApi, tabId: number): Promise<string | u
|
||||
}
|
||||
}
|
||||
|
||||
/** Recover only a failed preflight; never replay an already-dispatched navigation. */
|
||||
async function recoverBrowserNavigation(
|
||||
deps: NavigationDeps,
|
||||
tabId: number,
|
||||
action: (api: BrowserNavigationApi) => Promise<unknown>,
|
||||
waitUntil: WaitUntil,
|
||||
timeoutMs: number,
|
||||
): Promise<ReloadResult | RpcError> {
|
||||
const abort = linkedAbortSignal(deps.signal);
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
const api = deps.browserNavigation ?? chromeBrowserNavigationApi;
|
||||
try {
|
||||
let outcome = await navigateWithBrowserApi(
|
||||
api,
|
||||
tabId,
|
||||
() => action(api),
|
||||
waitUntil === "networkidle" ? "commit" : waitUntil,
|
||||
timeoutMs,
|
||||
abort.signal,
|
||||
);
|
||||
if (outcome.reached === "failed") return outcome.error;
|
||||
if (outcome.reached === "cancelled" || abort.signal.aborted) {
|
||||
return { code: "cancelled", message: "navigation recovery aborted" };
|
||||
}
|
||||
if (outcome.reached === "match") {
|
||||
// A completed reload is not a recovery if the restricted frame remains.
|
||||
await deps.cdp.send(tabId, "Page.enable", {});
|
||||
if (abort.signal.aborted)
|
||||
return { code: "cancelled", message: "navigation recovery aborted" };
|
||||
if (waitUntil === "networkidle") {
|
||||
// webNavigation cannot prove network idle. Subscribe on the new
|
||||
// document before enabling CDP lifecycle events after its commit.
|
||||
const frame = await readMainFrameInfo(deps.cdp, tabId);
|
||||
if (!frame.frameId) return cdpError("no main frame after navigation");
|
||||
const remaining = deadline - Date.now();
|
||||
if (remaining <= 0) {
|
||||
outcome = { reached: "timeout", lastLifecycle: "commit" };
|
||||
} else {
|
||||
const wait = startLifecycleWait(
|
||||
deps.cdp,
|
||||
tabId,
|
||||
frame.frameId,
|
||||
"networkIdle",
|
||||
remaining,
|
||||
abort.signal,
|
||||
{ loaderId: frame.loaderId ?? undefined },
|
||||
);
|
||||
await deps.cdp.send(tabId, "Page.setLifecycleEventsEnabled", { enabled: true });
|
||||
outcome = await wait.promise;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (outcome.reached === "cancelled" || abort.signal.aborted) {
|
||||
return { code: "cancelled", message: "navigation recovery aborted" };
|
||||
}
|
||||
return {
|
||||
tab_id: tabId,
|
||||
final_url: await readTabUrl(deps.tabsApi, tabId),
|
||||
reached: outcome.reached === "match" ? waitUntil : "timeout",
|
||||
...(outcome.reached === "timeout"
|
||||
? {
|
||||
error_text: `timed out waiting for ${waitUntil} during navigation recovery after ${timeoutMs}ms`,
|
||||
}
|
||||
: {}),
|
||||
};
|
||||
} catch (error) {
|
||||
return abort.signal.aborted
|
||||
? { code: "cancelled", message: "navigation recovery aborted" }
|
||||
: cdpError(error);
|
||||
} finally {
|
||||
abort.abort();
|
||||
abort.cleanup();
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// tool.navigate
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -505,7 +587,20 @@ export async function handleNavigate(
|
||||
|
||||
try {
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
await ensureCdpReady(deps.cdp, target.tabId);
|
||||
try {
|
||||
await ensureCdpReady(deps.cdp, target.tabId);
|
||||
} catch (error) {
|
||||
if (!isCdpExtensionAccessDenied(error)) throw error;
|
||||
const recovered = await recoverBrowserNavigation(
|
||||
deps,
|
||||
target.tabId,
|
||||
(api) => api.update(target.tabId, { url: params.url }),
|
||||
waitUntil,
|
||||
timeoutMs,
|
||||
);
|
||||
if (isRpcError(recovered)) return recovered;
|
||||
return attachDialogs(deps.cdp, target.tabId, dialogCursor, { ...recovered, url: params.url });
|
||||
}
|
||||
const expected = cdpLifecycleName(waitUntil);
|
||||
const beforeReadyState = await probeMainFrameReadyState(deps.cdp, target.tabId);
|
||||
const beforeFrame = await readMainFrameInfo(deps.cdp, target.tabId);
|
||||
@@ -574,10 +669,7 @@ export async function handleNavigate(
|
||||
}`,
|
||||
});
|
||||
} catch (err) {
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
message: err instanceof Error ? err.message : String(err),
|
||||
};
|
||||
return cdpError(err);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -727,7 +819,24 @@ export async function handleReload(
|
||||
|
||||
try {
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
await ensureCdpReady(deps.cdp, target.tabId);
|
||||
try {
|
||||
await ensureCdpReady(deps.cdp, target.tabId);
|
||||
} catch (error) {
|
||||
if (!isCdpExtensionAccessDenied(error)) throw error;
|
||||
const previousUrl = await readTabUrl(deps.tabsApi, target.tabId);
|
||||
const recovered = await recoverBrowserNavigation(
|
||||
deps,
|
||||
target.tabId,
|
||||
(api) => api.reload(target.tabId, { bypassCache: ignoreCache }),
|
||||
waitUntil,
|
||||
timeoutMs,
|
||||
);
|
||||
if (isRpcError(recovered)) return recovered;
|
||||
return attachDialogs(deps.cdp, target.tabId, dialogCursor, {
|
||||
...recovered,
|
||||
previous_url: previousUrl,
|
||||
});
|
||||
}
|
||||
const previousUrl = await readTabUrl(deps.tabsApi, target.tabId);
|
||||
const beforeFrame = await readMainFrameInfo(deps.cdp, target.tabId);
|
||||
const expected = cdpLifecycleName(waitUntil);
|
||||
@@ -776,10 +885,7 @@ export async function handleReload(
|
||||
}`,
|
||||
});
|
||||
} catch (err) {
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
message: err instanceof Error ? err.message : String(err),
|
||||
};
|
||||
return cdpError(err);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -15,22 +15,37 @@ import {
|
||||
type RecordQueryResponse,
|
||||
type RecordStartAck,
|
||||
type RecordStartMessage,
|
||||
type RecordStepAck,
|
||||
type RecordStopMessage,
|
||||
} from "@/lib/record-bridge";
|
||||
import { appendRecordedPayload, observeRecordedNavigation } from "@/lib/recording/step-buffer";
|
||||
import {
|
||||
type RecordFrameCoordinator,
|
||||
type RecordingCaptureScope,
|
||||
recordFrameCoordinator,
|
||||
} from "@/lib/recording/frame-coordinator";
|
||||
import { RecordingObservationRuntime } from "@/lib/recording/recording-runtime";
|
||||
import {
|
||||
appendRecordedPayload,
|
||||
observeRecordedNavigation,
|
||||
type RecordingStepBuffer,
|
||||
} from "@/lib/recording/step-buffer";
|
||||
import { RecordingTabCoordinator, type TabActivation } from "@/lib/recording/tab-coordinator";
|
||||
import { buildTraceV2 } from "@/lib/recording/trace-reducer-v2";
|
||||
import type { RecordingDraftStep } from "@/lib/recording/types";
|
||||
import type { SessionManager } from "@/session-manager/manager";
|
||||
import { EXTENSION_VERSION } from "@/transport/handshake";
|
||||
import type {
|
||||
RecordAwaitParams,
|
||||
RecordAwaitResult,
|
||||
RecordedTrace,
|
||||
RecordStartParams,
|
||||
RecordStartResult,
|
||||
RecordStopParams,
|
||||
RecordStopResult,
|
||||
RpcError,
|
||||
TraceV2,
|
||||
StopReason,
|
||||
} from "@/transport/types";
|
||||
import { TRACE_VERSION_V3 } from "@/transport/types";
|
||||
import { handleNavigate } from "./navigation";
|
||||
import {
|
||||
type CdpRunner,
|
||||
@@ -43,21 +58,46 @@ import {
|
||||
|
||||
interface ActiveRecording {
|
||||
requestId: string;
|
||||
tabId: number;
|
||||
tabs: RecordingTabCoordinator;
|
||||
agentWindowId: number;
|
||||
startUrl?: string;
|
||||
purpose?: string;
|
||||
steps: RecordingDraftStep[];
|
||||
startedAt: string;
|
||||
startedAtMs: number;
|
||||
finishPromise: Promise<TraceV2>;
|
||||
resolveFinish: (trace: TraceV2) => void;
|
||||
traceVersion: 2 | 3;
|
||||
supportsTabSwitchSteps: boolean;
|
||||
finishPromise: Promise<RecordedTrace>;
|
||||
resolveFinish: (trace: RecordedTrace) => void;
|
||||
rejectFinish: (err: Error) => void;
|
||||
settled: boolean;
|
||||
finishing: boolean;
|
||||
currentUrl?: string;
|
||||
pendingNavigation: boolean;
|
||||
pendingNavigationDeadline?: number;
|
||||
finishAttempt: Promise<RecordedTrace | null> | null;
|
||||
observation: RecordingObservationRuntime | null;
|
||||
stoppedBy: StopReason;
|
||||
/** Navigation callbacks tracked from event receipt through action enqueue. */
|
||||
navigationCallbacks: Set<Promise<void>>;
|
||||
/** Synchronous intake gate closed only after finish drains to stability. */
|
||||
acceptingNavigation: boolean;
|
||||
/**
|
||||
* Serializes step appends with navigation observation so a click is always
|
||||
* in `steps` before a same-turn `webNavigation` tries to annotate it.
|
||||
*/
|
||||
actionQueue: Promise<void>;
|
||||
/** Last accepted sequence for each content-script document producer. */
|
||||
lastStepSequenceByProducer: Map<string, number>;
|
||||
}
|
||||
|
||||
function enqueueRecordingAction(
|
||||
recording: ActiveRecording,
|
||||
task: () => Promise<void>,
|
||||
): Promise<void> {
|
||||
const queued = recording.actionQueue.then(task, task);
|
||||
recording.actionQueue = queued.catch(() => {});
|
||||
return queued;
|
||||
}
|
||||
|
||||
function isRecordingFinishing(recording: ActiveRecording): boolean {
|
||||
return recording.settled || recording.finishAttempt !== null;
|
||||
}
|
||||
|
||||
const recordings = new Map<string, ActiveRecording>();
|
||||
@@ -78,6 +118,9 @@ function sleep(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
/** Recording producer version mirrored into trace.recorder.bsk. */
|
||||
export const BSK_TRACE_VERSION = EXTENSION_VERSION;
|
||||
|
||||
/** Injectable http(s) landing page when `tool.record_start` omits `url`. */
|
||||
export const RECORD_DEFAULT_START_URL = "https://example.com/";
|
||||
|
||||
@@ -137,9 +180,11 @@ async function sendRecordStartWithAck(
|
||||
tabId: number,
|
||||
msg: RecordStartMessage,
|
||||
sendToTab: RecordDeps["sendToTab"],
|
||||
cancelled?: () => boolean,
|
||||
): Promise<void> {
|
||||
let lastError: unknown;
|
||||
for (let attempt = 0; attempt < RECORD_START_RETRIES; attempt += 1) {
|
||||
if (cancelled?.()) throw new Error("record start cancelled");
|
||||
try {
|
||||
const response = await sendToTab(tabId, msg);
|
||||
if (isRecordStartAck(response)) return;
|
||||
@@ -147,6 +192,7 @@ async function sendRecordStartWithAck(
|
||||
} catch (err) {
|
||||
lastError = err;
|
||||
}
|
||||
if (cancelled?.()) throw new Error("record start cancelled");
|
||||
if (attempt + 1 < RECORD_START_RETRIES) {
|
||||
await sleep(RECORD_START_RETRY_DELAY_MS);
|
||||
}
|
||||
@@ -154,7 +200,30 @@ async function sendRecordStartWithAck(
|
||||
throw lastError ?? new Error("failed to start recording in content script");
|
||||
}
|
||||
|
||||
function buildTrace(recording: ActiveRecording): TraceV2 {
|
||||
function negotiatedTraceVersion(params: RecordStartParams): 2 | 3 | RpcError {
|
||||
if (params.trace_version === undefined) return 2;
|
||||
if (params.trace_version === TRACE_VERSION_V3) return 3;
|
||||
return {
|
||||
code: "invalid_params",
|
||||
message: `unsupported trace_version ${params.trace_version}; supported values are omitted (v2) or ${TRACE_VERSION_V3} (v3)`,
|
||||
};
|
||||
}
|
||||
|
||||
function buildTrace(recording: ActiveRecording): RecordedTrace {
|
||||
if (recording.traceVersion === 3 && recording.observation) {
|
||||
return recording.observation.buildTrace({
|
||||
drafts: recording.steps,
|
||||
startedAt: recording.startedAt,
|
||||
purpose: recording.purpose,
|
||||
startUrl: recording.startUrl,
|
||||
stoppedBy: recording.stoppedBy,
|
||||
bskVersion: BSK_TRACE_VERSION,
|
||||
includeTabSwitches: recording.supportsTabSwitchSteps,
|
||||
});
|
||||
}
|
||||
if (recording.traceVersion === 3) {
|
||||
throw new Error("trace v3 observation runtime is unavailable");
|
||||
}
|
||||
return buildTraceV2({
|
||||
steps: recording.steps,
|
||||
startedAt: recording.startedAt,
|
||||
@@ -163,6 +232,51 @@ function buildTrace(recording: ActiveRecording): TraceV2 {
|
||||
});
|
||||
}
|
||||
|
||||
function stepBufferFor(
|
||||
recording: ActiveRecording,
|
||||
tabId: number,
|
||||
fallbackUrl?: string,
|
||||
): RecordingStepBuffer {
|
||||
return {
|
||||
steps: recording.steps,
|
||||
navigation: recording.tabs.navigation(tabId, fallbackUrl),
|
||||
};
|
||||
}
|
||||
|
||||
async function activateRecordingTab(
|
||||
recording: ActiveRecording,
|
||||
targetTabId: number,
|
||||
): Promise<void> {
|
||||
const previousTabId = recording.tabs.currentTabId;
|
||||
if (targetTabId === previousTabId) return;
|
||||
|
||||
const transition = recording.observation
|
||||
? await recording.observation.captureTabTransition(previousTabId, targetTabId)
|
||||
: {};
|
||||
const { targetUrl, ...stateLinks } = transition;
|
||||
const draft: Extract<RecordingDraftStep, { op: "switch_tab" }> = {
|
||||
op: "switch_tab",
|
||||
...stateLinks,
|
||||
};
|
||||
recording.steps.push(draft);
|
||||
recording.observation?.bindTabTransition(draft, recording.steps.length);
|
||||
recording.tabs.commit(targetTabId, targetUrl);
|
||||
}
|
||||
|
||||
async function processRecordedStep(
|
||||
recording: ActiveRecording,
|
||||
draftIndex: number,
|
||||
tabId: number,
|
||||
producerId?: string,
|
||||
): Promise<void> {
|
||||
if (!recording.observation) return;
|
||||
try {
|
||||
await recording.observation.processDraft(tabId, recording.steps, draftIndex, producerId);
|
||||
} catch (err) {
|
||||
console.warn(`[bsk record] observation failed for step ${draftIndex + 1}`, err);
|
||||
}
|
||||
}
|
||||
|
||||
export interface RecordDeps {
|
||||
tabsApi: ChromeTabsApi;
|
||||
sendToTab(
|
||||
@@ -170,16 +284,25 @@ export interface RecordDeps {
|
||||
msg: RecordStartMessage | RecordStopMessage | RecordCancelMessage,
|
||||
): Promise<unknown>;
|
||||
bypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
|
||||
frameCoordinator?: Pick<
|
||||
RecordFrameCoordinator,
|
||||
"begin" | "armTab" | "sourceFor" | "stop" | "cancel"
|
||||
>;
|
||||
cdp?: CdpRunner;
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
export type RecordRuntimeDeps = Omit<RecordDeps, "frameCoordinator" | "signal"> & {
|
||||
frameCoordinator: NonNullable<RecordDeps["frameCoordinator"]>;
|
||||
};
|
||||
|
||||
let defaultDeps: RecordDeps | null = null;
|
||||
function getDefaultDeps(): RecordDeps {
|
||||
if (!defaultDeps) {
|
||||
defaultDeps = {
|
||||
tabsApi: chromeTabsApi,
|
||||
sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
|
||||
frameCoordinator: recordFrameCoordinator,
|
||||
};
|
||||
}
|
||||
return defaultDeps;
|
||||
@@ -237,16 +360,58 @@ export function releaseBrowserObservationListenersIfIdle(): void {
|
||||
detachBrowserObservation = null;
|
||||
}
|
||||
|
||||
export function attachRecordStepListener(): () => void {
|
||||
export function attachRecordStepListener(deps: RecordDeps = getDefaultDeps()): () => void {
|
||||
const listener = (
|
||||
message: unknown,
|
||||
_sender: chrome.runtime.MessageSender,
|
||||
_sendResponse: () => void,
|
||||
sender: chrome.runtime.MessageSender,
|
||||
sendResponse: (response: RecordStepAck) => void,
|
||||
) => {
|
||||
if (!isRecordStepMessage(message)) return false;
|
||||
for (const recording of recordings.values()) {
|
||||
if (recording.requestId !== message.requestId) continue;
|
||||
appendRecordedPayload(recording, message.step);
|
||||
const source: RecordingCaptureScope | null | undefined = deps.frameCoordinator
|
||||
? deps.frameCoordinator.sourceFor(message.requestId, message.producerId, sender)
|
||||
: undefined;
|
||||
if (deps.frameCoordinator && !source) return false;
|
||||
const sourceTabId = source?.tabId ?? sender.tab?.id ?? recording.tabs.currentTabId;
|
||||
const sourceWasActive = sender.tab?.active ?? sourceTabId === recording.tabs.activeTabId;
|
||||
const producerKey = source
|
||||
? `${source.tabId}:${source.documentId}:${source.producerId}`
|
||||
: `${sourceTabId}:${message.producerId}`;
|
||||
const expectedSequence = (recording.lastStepSequenceByProducer.get(producerKey) ?? 0) + 1;
|
||||
if (message.sequence < expectedSequence) {
|
||||
sendResponse({ ok: true, sequence: message.sequence });
|
||||
return false;
|
||||
}
|
||||
if (message.sequence > expectedSequence) {
|
||||
sendResponse({
|
||||
ok: false,
|
||||
expectedSequence,
|
||||
error: `out-of-order recorded step ${message.sequence}`,
|
||||
});
|
||||
return false;
|
||||
}
|
||||
|
||||
recording.lastStepSequenceByProducer.set(producerKey, message.sequence);
|
||||
enqueueRecordingAction(recording, async () => {
|
||||
if (sourceTabId !== recording.tabs.currentTabId) {
|
||||
if (!sourceWasActive) return;
|
||||
if (sourceTabId !== recording.tabs.activeTabId)
|
||||
recording.tabs.noteActivation(sourceTabId);
|
||||
await activateRecordingTab(recording, sourceTabId);
|
||||
}
|
||||
await recording.observation?.flushRedirects(sourceTabId);
|
||||
const targetHint = message.step.geometry ? { geometry: message.step.geometry } : undefined;
|
||||
const draftIndex = appendRecordedPayload(
|
||||
stepBufferFor(recording, sourceTabId, message.step.page_url),
|
||||
message.step,
|
||||
targetHint,
|
||||
);
|
||||
if (draftIndex !== null) {
|
||||
await processRecordedStep(recording, draftIndex, sourceTabId, source?.producerId);
|
||||
}
|
||||
});
|
||||
sendResponse({ ok: true, sequence: message.sequence });
|
||||
return false;
|
||||
}
|
||||
return false;
|
||||
@@ -273,7 +438,7 @@ export function attachRecordFinishListener(deps: RecordDeps = getDefaultDeps()):
|
||||
|
||||
function findRecordingByTabId(tabId: number): ActiveRecording | null {
|
||||
for (const recording of recordings.values()) {
|
||||
if (recording.tabId === tabId && !recording.settled) return recording;
|
||||
if (recording.tabs.currentTabId === tabId && !recording.settled) return recording;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
@@ -310,7 +475,7 @@ async function clearRearmTimersForRecording(
|
||||
recording: ActiveRecording,
|
||||
deps: RecordDeps,
|
||||
): Promise<void> {
|
||||
clearRearmTimer(recording.tabId);
|
||||
clearRearmTimer(recording.tabs.currentTabId);
|
||||
try {
|
||||
const tabs = await deps.tabsApi.query({ windowId: recording.agentWindowId });
|
||||
for (const tab of tabs) {
|
||||
@@ -325,13 +490,16 @@ async function stopRecordingOnAllAgentTabs(
|
||||
recording: ActiveRecording,
|
||||
deps: RecordDeps,
|
||||
): Promise<void> {
|
||||
if (deps.frameCoordinator && !(await deps.frameCoordinator.stop(recording.requestId))) {
|
||||
throw new Error("failed to flush one or more recording documents");
|
||||
}
|
||||
const stopMsg: RecordStopMessage = { type: RECORD_STOP, requestId: recording.requestId };
|
||||
let tabIds = [recording.tabId];
|
||||
let tabIds = [recording.tabs.currentTabId];
|
||||
try {
|
||||
const tabs = await deps.tabsApi.query({ windowId: recording.agentWindowId });
|
||||
tabIds = [
|
||||
...new Set([
|
||||
recording.tabId,
|
||||
recording.tabs.currentTabId,
|
||||
...tabs.flatMap((tab) => (typeof tab.id === "number" ? [tab.id] : [])),
|
||||
]),
|
||||
];
|
||||
@@ -342,11 +510,11 @@ async function stopRecordingOnAllAgentTabs(
|
||||
for (const tabId of tabIds) {
|
||||
try {
|
||||
const response = await deps.sendToTab(tabId, stopMsg);
|
||||
if (tabId === recording.tabId && !isRecordStartAck(response)) {
|
||||
if (tabId === recording.tabs.currentTabId && !isRecordStartAck(response)) {
|
||||
throw new Error("content script did not confirm recorded steps");
|
||||
}
|
||||
} catch {
|
||||
if (tabId === recording.tabId) {
|
||||
if (tabId === recording.tabs.currentTabId) {
|
||||
throw new Error("failed to flush recorded steps");
|
||||
}
|
||||
}
|
||||
@@ -364,22 +532,39 @@ async function rearmRecording(
|
||||
recording: ActiveRecording,
|
||||
targetTabId: number,
|
||||
deps: RecordDeps,
|
||||
activation?: TabActivation,
|
||||
): Promise<boolean> {
|
||||
// Do NOT toggle automation-bypass here: each retry used to increment the
|
||||
// content-script counter, and a single stop decrement left the ControlOverlay
|
||||
// stuck with pointer-events:none (page usable, Interrupt dead). RecordOverlay
|
||||
// already hides the control chrome while activeRecord is set.
|
||||
const isFinishing = () => isRecordingFinishing(recording);
|
||||
for (let attempt = 0; attempt < RECORD_REARM_MAX_ATTEMPTS; attempt += 1) {
|
||||
if (isFinishing()) return false;
|
||||
const startMsg: RecordStartMessage = {
|
||||
type: RECORD_START,
|
||||
requestId: recording.requestId,
|
||||
startedAtMs: recording.startedAtMs,
|
||||
};
|
||||
try {
|
||||
await sendRecordStartWithAck(targetTabId, startMsg, deps.sendToTab);
|
||||
recording.tabId = targetTabId;
|
||||
if (deps.frameCoordinator) {
|
||||
const frameStarted = await deps.frameCoordinator.armTab(recording.requestId, targetTabId);
|
||||
if (!frameStarted) throw new Error("recording document did not start");
|
||||
}
|
||||
await sendRecordStartWithAck(targetTabId, startMsg, deps.sendToTab, isFinishing);
|
||||
if (isFinishing()) return false;
|
||||
if (activation) {
|
||||
if (!recording.tabs.isLatest(activation)) return true;
|
||||
await enqueueRecordingAction(recording, async () => {
|
||||
if (isFinishing() || !recording.tabs.isLatest(activation)) {
|
||||
return;
|
||||
}
|
||||
await activateRecordingTab(recording, targetTabId);
|
||||
});
|
||||
}
|
||||
return true;
|
||||
} catch {
|
||||
if (isFinishing()) return false;
|
||||
if (attempt + 1 < RECORD_REARM_MAX_ATTEMPTS) {
|
||||
await sleep(RECORD_REARM_RETRY_DELAY_MS);
|
||||
}
|
||||
@@ -388,7 +573,7 @@ async function rearmRecording(
|
||||
return false;
|
||||
}
|
||||
|
||||
function scheduleRearmForTab(tabId: number, deps: RecordDeps): void {
|
||||
function scheduleRearmForTab(tabId: number, deps: RecordDeps, activation?: TabActivation): void {
|
||||
const existing = rearmTimers.get(tabId);
|
||||
if (existing) clearTimeout(existing);
|
||||
rearmTimers.set(
|
||||
@@ -397,7 +582,7 @@ function scheduleRearmForTab(tabId: number, deps: RecordDeps): void {
|
||||
rearmTimers.delete(tabId);
|
||||
void (async () => {
|
||||
const current = await findRecordingForTab(tabId, deps);
|
||||
if (current) await rearmRecording(current, tabId, deps);
|
||||
if (current) await rearmRecording(current, tabId, deps, activation);
|
||||
})();
|
||||
}, RECORD_REARM_DEBOUNCE_MS),
|
||||
);
|
||||
@@ -409,14 +594,20 @@ export function attachRecordTabListener(deps: RecordDeps = getDefaultDeps()): ()
|
||||
const windowId = tab.windowId;
|
||||
if (tabId === undefined || windowId === undefined) return;
|
||||
for (const recording of recordings.values()) {
|
||||
if (recording.settled || recording.agentWindowId !== windowId) continue;
|
||||
if (isRecordingFinishing(recording) || recording.agentWindowId !== windowId) continue;
|
||||
scheduleRearmForTab(tabId, deps);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
const onActivated = (activeInfo: chrome.tabs.TabActiveInfo) => {
|
||||
scheduleRearmForTab(activeInfo.tabId, deps);
|
||||
for (const recording of recordings.values()) {
|
||||
if (isRecordingFinishing(recording) || recording.agentWindowId !== activeInfo.windowId)
|
||||
continue;
|
||||
const activation = recording.tabs.noteActivation(activeInfo.tabId);
|
||||
scheduleRearmForTab(activeInfo.tabId, deps, activation);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
chrome.tabs.onCreated.addListener(onCreated);
|
||||
@@ -428,13 +619,47 @@ export function attachRecordTabListener(deps: RecordDeps = getDefaultDeps()): ()
|
||||
}
|
||||
|
||||
export function attachRecordNavigationListener(deps: RecordDeps = getDefaultDeps()): () => void {
|
||||
const observeMainFrame = (tabId: number, url?: string, causedByAction?: boolean) => {
|
||||
void (async () => {
|
||||
const recording = await findRecordingForTab(tabId, deps);
|
||||
if (recording && url) {
|
||||
observeRecordedNavigation(recording, url, causedByAction);
|
||||
}
|
||||
})();
|
||||
const observeMainFrame = (
|
||||
tabId: number,
|
||||
url?: string,
|
||||
causedByAction?: boolean,
|
||||
transitionType?: string,
|
||||
transitionQualifiers?: string[],
|
||||
) => {
|
||||
if (!url) return;
|
||||
const candidates = [...recordings.values()].filter(
|
||||
(recording) =>
|
||||
!recording.settled && recording.acceptingNavigation && recording.tabs.activeTabId === tabId,
|
||||
);
|
||||
for (const recording of candidates) {
|
||||
const queued = enqueueRecordingAction(recording, async () => {
|
||||
if (tabId !== recording.tabs.currentTabId) {
|
||||
await activateRecordingTab(recording, tabId);
|
||||
}
|
||||
const result = observeRecordedNavigation(
|
||||
stepBufferFor(recording, tabId, url),
|
||||
url,
|
||||
causedByAction,
|
||||
transitionType,
|
||||
transitionQualifiers,
|
||||
);
|
||||
if (result.kind === "coalesce_redirect") {
|
||||
recording.observation?.scheduleRedirect(tabId, recording.steps, result.url);
|
||||
return;
|
||||
}
|
||||
|
||||
if (result.kind === "noop") return;
|
||||
recording.observation?.clearRedirect(tabId);
|
||||
if (result.kind === "appended") {
|
||||
await processRecordedStep(recording, result.index, tabId);
|
||||
} else {
|
||||
recording.observation?.scheduleSettle(tabId, recording.steps, result.index);
|
||||
}
|
||||
});
|
||||
const tracked = queued.catch(() => {});
|
||||
recording.navigationCallbacks.add(tracked);
|
||||
void tracked.finally(() => recording.navigationCallbacks.delete(tracked));
|
||||
}
|
||||
};
|
||||
const onMainFrameComplete = (tabId: number, url?: string) => {
|
||||
void (async () => {
|
||||
@@ -454,18 +679,13 @@ export function attachRecordNavigationListener(deps: RecordDeps = getDefaultDeps
|
||||
details: chrome.webNavigation.WebNavigationTransitionCallbackDetails,
|
||||
) => {
|
||||
if (details.frameId !== 0) return;
|
||||
const causedByAction =
|
||||
details.transitionType === "link" || details.transitionType === "form_submit"
|
||||
? true
|
||||
: details.transitionType === "typed" ||
|
||||
details.transitionType === "auto_bookmark" ||
|
||||
details.transitionType === "generated" ||
|
||||
details.transitionType === "keyword" ||
|
||||
details.transitionType === "keyword_generated" ||
|
||||
details.transitionType === "reload"
|
||||
? false
|
||||
: undefined;
|
||||
observeMainFrame(details.tabId, details.url, causedByAction);
|
||||
observeMainFrame(
|
||||
details.tabId,
|
||||
details.url,
|
||||
undefined,
|
||||
details.transitionType,
|
||||
details.transitionQualifiers,
|
||||
);
|
||||
};
|
||||
chrome.webNavigation.onCompleted.addListener(completedListener);
|
||||
chrome.webNavigation.onCommitted?.addListener(committedListener);
|
||||
@@ -483,6 +703,26 @@ export function attachRecordNavigationListener(deps: RecordDeps = getDefaultDeps
|
||||
return () => chrome.tabs.onUpdated.removeListener(listener);
|
||||
}
|
||||
|
||||
const MAX_FINISH_DRAIN_ROUNDS = 10;
|
||||
|
||||
async function drainRecordingToStability(recording: ActiveRecording): Promise<boolean> {
|
||||
for (let round = 0; round < MAX_FINISH_DRAIN_ROUNDS; round += 1) {
|
||||
await Promise.all([...recording.navigationCallbacks]);
|
||||
const actionTail = recording.actionQueue;
|
||||
await actionTail;
|
||||
await recording.observation?.flush();
|
||||
|
||||
if (recording.navigationCallbacks.size === 0 && recording.actionQueue === actionTail) {
|
||||
// No event or promise continuation can interleave with this synchronous
|
||||
// check-and-close, so work accepted before the cutoff is fully drained.
|
||||
recording.acceptingNavigation = false;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
console.warn("[bsk record] navigation/action queues did not stabilize at stop");
|
||||
return false;
|
||||
}
|
||||
|
||||
export function attachRecordQueryListener(deps: RecordDeps = getDefaultDeps()): () => void {
|
||||
const listener = (
|
||||
message: unknown,
|
||||
@@ -501,7 +741,8 @@ export function attachRecordQueryListener(deps: RecordDeps = getDefaultDeps()):
|
||||
sendResponse({ active: false });
|
||||
return;
|
||||
}
|
||||
await rearmRecording(recording, tabId, deps);
|
||||
const activation = sender.tab?.active ? recording.tabs.noteActivation(tabId) : undefined;
|
||||
await rearmRecording(recording, tabId, deps, activation);
|
||||
sendResponse({
|
||||
active: true,
|
||||
requestId: recording.requestId,
|
||||
@@ -523,23 +764,50 @@ async function finishRecordingByRequest(
|
||||
if (recording.requestId !== requestId || recording.settled) continue;
|
||||
const match = await findRecordingForTab(tabId, deps);
|
||||
if (match !== recording) continue;
|
||||
await finishRecording(sessionId, deps);
|
||||
await finishRecording(sessionId, deps, "user_finish");
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
async function finishRecording(sessionId: string, deps: RecordDeps): Promise<TraceV2 | null> {
|
||||
async function finishRecording(
|
||||
sessionId: string,
|
||||
deps: RecordDeps,
|
||||
stoppedBy: StopReason,
|
||||
): Promise<RecordedTrace | null> {
|
||||
const recording = recordings.get(sessionId);
|
||||
if (!recording || recording.settled || recording.finishing) return null;
|
||||
recording.finishing = true;
|
||||
if (!recording || recording.settled) return null;
|
||||
if (recording.finishAttempt) return recording.finishAttempt;
|
||||
recording.stoppedBy = stoppedBy;
|
||||
|
||||
const attempt = finishRecordingAttempt(sessionId, recording, deps);
|
||||
recording.finishAttempt = attempt;
|
||||
try {
|
||||
return await attempt;
|
||||
} finally {
|
||||
if (!recording.settled) {
|
||||
recording.finishAttempt = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function finishRecordingAttempt(
|
||||
sessionId: string,
|
||||
recording: ActiveRecording,
|
||||
deps: RecordDeps,
|
||||
): Promise<RecordedTrace | null> {
|
||||
await clearRearmTimersForRecording(recording, deps);
|
||||
try {
|
||||
// Disposing content capture may commit a final dirty fill. Flush content
|
||||
// first so all resulting RECORD_STEP messages enter actionQueue before it
|
||||
// and the observation queues are drained.
|
||||
await stopRecordingOnAllAgentTabs(recording, deps);
|
||||
} catch {
|
||||
recording.finishing = false;
|
||||
return null;
|
||||
}
|
||||
if (!(await drainRecordingToStability(recording))) {
|
||||
return null;
|
||||
}
|
||||
await recording.observation?.settleTrailing(recording.tabs.currentTabId, recording.steps);
|
||||
|
||||
recording.settled = true;
|
||||
recordings.delete(sessionId);
|
||||
@@ -554,6 +822,15 @@ export async function handleRecordStart(
|
||||
params: RecordStartParams,
|
||||
deps: RecordDeps = getDefaultDeps(),
|
||||
): Promise<RecordStartResult | RpcError> {
|
||||
const traceVersionOrErr = negotiatedTraceVersion(params);
|
||||
if (typeof traceVersionOrErr !== "number") return traceVersionOrErr;
|
||||
if (traceVersionOrErr === 3 && !deps.cdp) {
|
||||
return {
|
||||
code: "protocol_error",
|
||||
message: "trace v3 recording requires an active CDP connection",
|
||||
};
|
||||
}
|
||||
|
||||
const ctxOrErr = lookupSession(manager, params, "record_start");
|
||||
if (isRpcError(ctxOrErr)) return ctxOrErr;
|
||||
const ctx = ctxOrErr;
|
||||
@@ -570,37 +847,64 @@ export async function handleRecordStart(
|
||||
// on the destination page can RECORD_QUERY → rearm → show RecordOverlay
|
||||
// instead of flashing ControlOverlay ("Agent 正在控制").
|
||||
const requestId = makeRequestId(target.tabId);
|
||||
let resolveFinish!: (trace: TraceV2) => void;
|
||||
let resolveFinish!: (trace: RecordedTrace) => void;
|
||||
let rejectFinish!: (err: Error) => void;
|
||||
const finishPromise = new Promise<TraceV2>((resolve, reject) => {
|
||||
const finishPromise = new Promise<RecordedTrace>((resolve, reject) => {
|
||||
resolveFinish = resolve;
|
||||
rejectFinish = reject;
|
||||
});
|
||||
const navigateUrl = params.url ?? RECORD_DEFAULT_START_URL;
|
||||
const startedAtMs = Date.now();
|
||||
const maxPageTokens = params.max_page_tokens;
|
||||
const redactValues = params.redact_values ?? false;
|
||||
recordings.set(params.session_id, {
|
||||
requestId,
|
||||
tabId: target.tabId,
|
||||
tabs: new RecordingTabCoordinator(target.tabId, navigateUrl),
|
||||
agentWindowId: ctx.agentWindowId,
|
||||
startUrl: navigateUrl,
|
||||
...(params.purpose ? { purpose: params.purpose } : {}),
|
||||
steps: [],
|
||||
startedAt: new Date(startedAtMs).toISOString(),
|
||||
startedAtMs,
|
||||
traceVersion: traceVersionOrErr,
|
||||
supportsTabSwitchSteps: params.supports_tab_switch_steps === true,
|
||||
finishPromise,
|
||||
resolveFinish,
|
||||
rejectFinish,
|
||||
settled: false,
|
||||
finishing: false,
|
||||
currentUrl: navigateUrl,
|
||||
pendingNavigation: false,
|
||||
pendingNavigationDeadline: undefined,
|
||||
finishAttempt: null,
|
||||
observation:
|
||||
traceVersionOrErr === 3 && deps.cdp
|
||||
? new RecordingObservationRuntime({
|
||||
cdp: deps.cdp,
|
||||
tabsApi: deps.tabsApi,
|
||||
maxTokens: maxPageTokens,
|
||||
redactValues,
|
||||
})
|
||||
: null,
|
||||
stoppedBy: "user_finish",
|
||||
navigationCallbacks: new Set(),
|
||||
acceptingNavigation: true,
|
||||
actionQueue: Promise.resolve(),
|
||||
lastStepSequenceByProducer: new Map(),
|
||||
});
|
||||
deps.frameCoordinator?.begin(requestId, startedAtMs, target.tabId, async (scope) => {
|
||||
const recording = recordings.get(params.session_id);
|
||||
if (!recording || recording.requestId !== requestId || recording.settled) return;
|
||||
try {
|
||||
await recording.observation?.refreshDocument(scope.tabId, scope.producerId);
|
||||
} catch {
|
||||
// The normal action-time capture remains available if a child Document
|
||||
// is replaced while its readiness refresh is in flight.
|
||||
}
|
||||
});
|
||||
// Observe navigations for the whole recording lifetime; attach before
|
||||
// optional navigate so the destination load can rearm capture.
|
||||
ensureBrowserObservationListeners(deps);
|
||||
|
||||
const abortPending = async (notifyContent: boolean) => {
|
||||
recordings.get(params.session_id)?.observation?.cancel();
|
||||
deps.frameCoordinator?.cancel(requestId);
|
||||
recordings.delete(params.session_id);
|
||||
releaseBrowserObservationListenersIfIdle();
|
||||
if (notifyContent) {
|
||||
@@ -682,7 +986,7 @@ export async function handleRecordStart(
|
||||
return cancelledError();
|
||||
}
|
||||
active.startUrl = startUrl;
|
||||
active.currentUrl = startUrl;
|
||||
active.tabs.navigation(target.tabId, startUrl).currentUrl = startUrl;
|
||||
|
||||
if (isContentScriptRestrictedUrl(startUrl)) {
|
||||
await abortPending(false);
|
||||
@@ -690,7 +994,7 @@ export async function handleRecordStart(
|
||||
code: "invalid_params",
|
||||
message: params.url
|
||||
? `cannot record on restricted URL (${startUrl}); use an http(s) page`
|
||||
: `cannot record on restricted URL (${startUrl}); default start page must be injectable http(s)`,
|
||||
: `cannot record on restricted URL (${startUrl}); default start page https://example.com/ did not load — pass --url with a page you can open`,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -717,6 +1021,10 @@ export async function handleRecordStart(
|
||||
const startMsg: RecordStartMessage = { type: RECORD_START, requestId, startedAtMs };
|
||||
|
||||
try {
|
||||
if (deps.frameCoordinator) {
|
||||
const frameStarted = await deps.frameCoordinator.armTab(requestId, target.tabId);
|
||||
if (!frameStarted) throw new Error("top recording document did not start");
|
||||
}
|
||||
await sendRecordStartWithAck(target.tabId, startMsg, deps.sendToTab);
|
||||
} catch {
|
||||
await abortPending(true);
|
||||
@@ -727,6 +1035,17 @@ export async function handleRecordStart(
|
||||
};
|
||||
}
|
||||
|
||||
if (active.observation) {
|
||||
const activeRecording = recordings.get(params.session_id);
|
||||
if (activeRecording) {
|
||||
try {
|
||||
await activeRecording.observation?.captureInitial(target.tabId);
|
||||
} catch {
|
||||
// Proceed without initial observation; steps may be dropped by reducer.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
const cancelled = await abortIfCancelled(true);
|
||||
if (cancelled) return cancelled;
|
||||
@@ -751,7 +1070,7 @@ export async function handleRecordStop(
|
||||
};
|
||||
}
|
||||
|
||||
const trace = await finishRecording(params.session_id, deps);
|
||||
const trace = await finishRecording(params.session_id, deps, "cli_stop");
|
||||
if (!trace) {
|
||||
return {
|
||||
code: "protocol_error",
|
||||
@@ -781,9 +1100,9 @@ export async function handleRecordAwait(
|
||||
return { code: "cancelled", message: "record_await aborted" };
|
||||
}
|
||||
|
||||
const outcome = await new Promise<{ trace: TraceV2 } | { error: RpcError }>((resolve) => {
|
||||
const outcome = await new Promise<{ trace: RecordedTrace } | { error: RpcError }>((resolve) => {
|
||||
let settled = false;
|
||||
const finish = (result: { trace: TraceV2 } | { error: RpcError }) => {
|
||||
const finish = (result: { trace: RecordedTrace } | { error: RpcError }) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
if (timer) clearTimeout(timer);
|
||||
@@ -825,6 +1144,8 @@ export function clearRecordingForSession(sessionId: string): void {
|
||||
}
|
||||
void clearRearmTimersForRecording(recording, getDefaultDeps());
|
||||
if (!recording.settled) {
|
||||
getDefaultDeps().frameCoordinator?.cancel(recording.requestId);
|
||||
recording.observation?.cancel();
|
||||
recording.settled = true;
|
||||
recording.rejectFinish(new Error("recording cleared"));
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ import { type SessionManager, SessionStartCleanupError } from "@/session-manager
|
||||
import type { RpcError } from "@/transport/types";
|
||||
import { rpcError } from "./errors";
|
||||
import { clearRecordingForSession } from "./record";
|
||||
import type { CdpRunner, ChromeTabsApi } from "./shared";
|
||||
import { isRpcError } from "./shared";
|
||||
import { returnBorrowedTab, type TabManagementDeps } from "./tabs";
|
||||
|
||||
@@ -73,11 +74,17 @@ export interface SessionStopResult {
|
||||
/** Tab ids whose return path failed; those entries remain borrowed so
|
||||
* shutdown can be retried without closing the Agent Window. */
|
||||
return_failures?: Array<{ tab_id: number; code: string; message: string }>;
|
||||
/**
|
||||
* True when the Agent Window was *released* to the user (instead of being
|
||||
* closed) because it still contained user-created tabs after the agent's own
|
||||
* tabs were closed. Single-session scope.
|
||||
*/
|
||||
window_released?: boolean;
|
||||
}
|
||||
|
||||
export interface SessionStopDeps {
|
||||
signal?: AbortSignal;
|
||||
cdp?: {
|
||||
cdp?: Pick<CdpRunner, "releaseSessionTab"> & {
|
||||
detachSession(sessionId: string): Promise<void>;
|
||||
};
|
||||
/**
|
||||
@@ -87,6 +94,13 @@ export interface SessionStopDeps {
|
||||
* without a real browser.
|
||||
*/
|
||||
tabManagement?: TabManagementDeps;
|
||||
/**
|
||||
* Read-only Chrome tabs API used to *query* remaining tabs before deciding
|
||||
* whether to release the window. Kept separate from `tabManagement.tabs`
|
||||
* (a `TabMutationApi` that has no `query`) so the mutation interface stays
|
||||
* pure. Tests can inject a fake `ChromeTabsApi` too.
|
||||
*/
|
||||
tabsQuery?: ChromeTabsApi;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -152,8 +166,10 @@ export async function handleSessionStart(
|
||||
* cleanly (review parity with M6).
|
||||
* 3. Detach CDP sessions the extension still holds for this
|
||||
* session (no-op if M6/M7 didn't attach to any tab).
|
||||
* 4. Close the Agent Window. SessionManager.stop() removes the
|
||||
* Chrome window and forgets the context.
|
||||
* 4. Close the agent-created tabs and the home tab (design §3.2).
|
||||
* 5. If any tab remains — those are user-created tabs — release the
|
||||
* window to the user (dropOnly) instead of closing it, so user
|
||||
* tabs survive. Otherwise close the window as before.
|
||||
*
|
||||
* Failures in step 1 keep the Agent Window open: a failed borrowed tab
|
||||
* may still be there, so closing the window would risk losing user
|
||||
@@ -188,15 +204,19 @@ export async function handleSessionStop(
|
||||
const returnFailures: SessionStopResult["return_failures"] = [];
|
||||
const borrowedIds = Array.from(ctx.borrowedTabs.keys());
|
||||
for (const tabId of borrowedIds) {
|
||||
// A close event may have removed this borrow while another tab returned.
|
||||
if (!ctx.borrowedTabs.has(tabId)) continue;
|
||||
try {
|
||||
const tabManagement = {
|
||||
...(deps.tabManagement ?? {}),
|
||||
cdp: deps.cdp ?? deps.tabManagement?.cdp,
|
||||
signal: deps.signal,
|
||||
isAgentWindowId:
|
||||
deps.tabManagement?.isAgentWindowId ??
|
||||
((windowId: number) => manager.findByWindowId(windowId) !== null),
|
||||
};
|
||||
const outcome = await returnBorrowedTab(ctx, tabId, tabManagement);
|
||||
if (!ctx.borrowedTabs.has(tabId)) continue;
|
||||
if (typeof outcome === "object" && "code" in outcome) {
|
||||
console.warn(`[bsk session_stop] auto-return failed for tab ${tabId}`, outcome);
|
||||
returnFailures?.push({
|
||||
@@ -209,6 +229,7 @@ export async function handleSessionStop(
|
||||
ctx.borrowedTabs.delete(tabId);
|
||||
}
|
||||
} catch (err) {
|
||||
if (!ctx.borrowedTabs.has(tabId)) continue;
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
console.warn(`[bsk session_stop] auto-return threw for tab ${tabId}: ${message}`);
|
||||
returnFailures?.push({
|
||||
@@ -219,8 +240,15 @@ export async function handleSessionStop(
|
||||
}
|
||||
}
|
||||
|
||||
// A previously failed tab may have closed during a later return. Only
|
||||
// confirmed close events remove borrows; ordinary move failures stay retryable.
|
||||
const pendingReturnFailures = returnFailures.filter((failure) =>
|
||||
ctx.borrowedTabs.has(failure.tab_id),
|
||||
);
|
||||
if (deps.signal?.aborted) {
|
||||
const nonCancelFailures = returnFailures?.filter((failure) => failure.code !== "cancelled");
|
||||
const nonCancelFailures = pendingReturnFailures.filter(
|
||||
(failure) => failure.code !== "cancelled",
|
||||
);
|
||||
if (nonCancelFailures && nonCancelFailures.length > 0) {
|
||||
return {
|
||||
...(returnedTabIds.length > 0 ? { returned_tab_ids: returnedTabIds } : {}),
|
||||
@@ -232,8 +260,8 @@ export async function handleSessionStop(
|
||||
|
||||
const result: SessionStopResult = {};
|
||||
if (returnedTabIds.length > 0) result.returned_tab_ids = returnedTabIds;
|
||||
if (returnFailures && returnFailures.length > 0) {
|
||||
result.return_failures = returnFailures;
|
||||
if (pendingReturnFailures.length > 0) {
|
||||
result.return_failures = pendingReturnFailures;
|
||||
// A failed return means at least one borrowed user tab may still be
|
||||
// inside the Agent Window. Keep the session/window alive so the user
|
||||
// can retry `bsk session stop` or explicitly `bsk tab return` after the
|
||||
@@ -253,8 +281,66 @@ export async function handleSessionStop(
|
||||
return { code: "cancelled", message: "session_stop aborted before window close" };
|
||||
}
|
||||
|
||||
// Step 4: close the Agent Window and drop the context.
|
||||
await manager.stop(params.session_id);
|
||||
// Step 4: close every tab explicitly created by the agent, including the
|
||||
// home tab. `tabsApi` is a
|
||||
// TabMutationApi (remove only); `queryApi` is a separate read-only
|
||||
// ChromeTabsApi. If neither is injected, we conservatively fall back to
|
||||
// closing the window (see Step 5).
|
||||
const tabsApi = deps.tabManagement?.tabs;
|
||||
const queryApi = deps.tabsQuery;
|
||||
|
||||
if (tabsApi) {
|
||||
// Close each agent-created tab that still exists.
|
||||
const agentCreatedTabIds = Array.from(ctx.agentCreatedTabs);
|
||||
for (const tabId of agentCreatedTabIds) {
|
||||
try {
|
||||
await tabsApi.remove(tabId);
|
||||
ctx.agentCreatedTabs.delete(tabId);
|
||||
} catch (err) {
|
||||
// Tab may already be gone (closed by the user). Non-fatal.
|
||||
console.warn(`[bsk session_stop] failed to close agent tab ${tabId}`, err);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Step 5: decide whether to release (keep) the window or close it.
|
||||
let shouldRelease = false;
|
||||
if (queryApi) {
|
||||
try {
|
||||
const liveWindowTabs = await queryApi.query({ windowId: ctx.agentWindowId });
|
||||
// Only genuine *user* tabs count toward keeping the window open.
|
||||
// An agent tab that failed to close in Step 4 may still be present
|
||||
// here; if we counted it as a reason to release (dropOnly), the
|
||||
// window would be kept and that agent tab would leak (issue #57
|
||||
// regression). Exclude any id still tracked in agentCreatedTabs.
|
||||
const userTabs = liveWindowTabs.filter((t) => {
|
||||
if (t.id === undefined) return false;
|
||||
return !ctx.agentCreatedTabs.has(t.id);
|
||||
});
|
||||
const leakedAgentTabs = liveWindowTabs.filter(
|
||||
(t) => t.id !== undefined && ctx.agentCreatedTabs.has(t.id),
|
||||
);
|
||||
if (leakedAgentTabs.length > 0) {
|
||||
console.warn(
|
||||
`[bsk session_stop] ${leakedAgentTabs.length} agent tab(s) failed to close; forcing window close instead of release`,
|
||||
leakedAgentTabs.map((t) => t.id),
|
||||
);
|
||||
}
|
||||
shouldRelease = userTabs.length > 0;
|
||||
} catch {
|
||||
// Query failed (e.g. window already gone) — conservatively close it.
|
||||
shouldRelease = false;
|
||||
}
|
||||
}
|
||||
|
||||
if (shouldRelease) {
|
||||
// Keep the window + its user tabs; only drop the session binding.
|
||||
await manager.stop(params.session_id, { dropOnly: true });
|
||||
result.window_released = true;
|
||||
} else {
|
||||
// Window is empty (or we couldn't verify state) — close it.
|
||||
await manager.stop(params.session_id);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
@@ -53,9 +53,11 @@ export type { DialogCursor };
|
||||
export interface CdpRunner {
|
||||
send<T = unknown>(tabId: number, method: string, params?: object): Promise<T>;
|
||||
sendToTarget?<T = unknown>(target: CdpTarget, method: string, params?: object): Promise<T>;
|
||||
detach?(tabId: number): Promise<void>;
|
||||
getFrameGraph?(tabId: number): Promise<CdpFrameGraph>;
|
||||
ensureAttachedToUrl?(tabId: number, expectedUrl: string | undefined): Promise<void>;
|
||||
trackSessionTab?(sessionId: string, tabId: number): void;
|
||||
releaseSessionTab?(sessionId: string, tabId: number): Promise<void>;
|
||||
onEvent?(handler: (source: CdpDebuggee, method: string, params: unknown) => void): {
|
||||
dispose(): void;
|
||||
};
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user