Compare commits

..
Author SHA1 Message Date
“haonan” fd85b80c78 fix(record): preserve trace v2/v3 compatibility
Keep protocol 1.0 peers connected, return v2 traces to legacy clients, and allow the new CLI to export both v2 and v3 recordings.
2026-08-17 16:32:48 +08:00
“haonan”andCursor 464a8da86b style(record): format trace recording changes
Apply the repository formatter so the merged Trace v3 changes pass frontend lint checks.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-08-17 12:24:31 +08:00
“haonan” 9b6a525439 Merge remote-tracking branch 'origin/main' into feat/trace-v3-record 2026-08-17 12:14:26 +08:00
“haonan” 61a0e0d218 fix(record): keep committed empty fill steps in exported traces 2026-08-17 11:02:10 +08:00
“haonan” 67b7227abf fix(popup): point record prompt at the ./trace directory bundle 2026-08-13 19:55:34 +08:00
“haonan” 02d1d69379 docs(ux): clarify version mismatches and update record skill docs
Surface actionable update guidance in the popup and connection errors, document
the trace v3 directory bundle in SKILL.md.
2026-08-13 19:40:54 +08:00
“haonan” 025751ef03 feat(cli): write record output as a trace directory bundle
Persist `trace.json` plus per-state page VOM files under `--output`, tolerate
raced session teardown on stop, and clarify version-mismatch recovery hints.
2026-08-13 19:38:05 +08:00
“haonan” 13a61b2688 feat(extension): record user flows as trace v3 page-state observations
Capture settled page observations into a states/steps trace, match targets by
geometry with capture fallbacks, and reuse VOM rendering with richer refs and
optional value redaction for record bundles.
2026-08-13 19:34:37 +08:00
“haonan” 1508839bf9 feat(protocol): bump to 1.1 and redefine record traces as state-linked v3
Replace page-indexed record traces with states/steps + FillCommit/hover/scroll,
and raise daemon/extension protocol floors together so old 1.0 peers are rejected.
2026-08-13 19:31:35 +08:00
280 changed files with 7638 additions and 41225 deletions
-105
View File
@@ -1,105 +0,0 @@
name: Release dsh plugin
on:
push:
tags:
- dsh-plugin-v*
workflow_dispatch:
permissions:
contents: read
concurrency:
group: release-dsh-plugin-${{ github.ref }}
cancel-in-progress: false
env:
PACKAGE_NAME: "@wxg-prc-cpg/browser-skill-dsh-plugin"
PACKAGE_DIR: packages/dsh-plugin-browserskill
jobs:
resolve:
runs-on: ubuntu-latest
outputs:
version: ${{ steps.version.outputs.version }}
tag: ${{ steps.version.outputs.tag }}
steps:
- uses: actions/checkout@v6
- name: Resolve plugin version
id: version
shell: bash
run: |
set -euo pipefail
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
VERSION="$(node -p "require('./${{ env.PACKAGE_DIR }}/package.json').version")"
else
VERSION="${GITHUB_REF_NAME#dsh-plugin-v}"
fi
VERSION="${VERSION#v}"
echo "version=${VERSION}" >> "$GITHUB_OUTPUT"
echo "tag=dsh-plugin-v${VERSION}" >> "$GITHUB_OUTPUT"
guard:
needs: resolve
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
- name: Verify tag matches package.json name and version
shell: bash
run: |
set -euo pipefail
PKG_NAME="$(node -p "require('./${{ env.PACKAGE_DIR }}/package.json').name")"
PKG_VERSION="$(node -p "require('./${{ env.PACKAGE_DIR }}/package.json').version")"
RELEASE_VERSION="${{ needs.resolve.outputs.version }}"
if [ "$PKG_NAME" != "$PACKAGE_NAME" ]; then
echo "package.json name (${PKG_NAME}) does not match expected ${PACKAGE_NAME}"
exit 1
fi
if [ "$PKG_VERSION" != "$RELEASE_VERSION" ]; then
echo "${PACKAGE_DIR}/package.json version (${PKG_VERSION}) does not match release version (${RELEASE_VERSION})"
exit 1
fi
echo "Version guard passed: ${PKG_NAME}@${RELEASE_VERSION}"
publish:
needs: [resolve, guard]
runs-on: ubuntu-latest
# Secret NPM_TOKEN is stored on the "npm-publish" GitHub Environment.
environment: npm-publish
permissions:
contents: read
steps:
- uses: actions/checkout@v6
- uses: pnpm/action-setup@v6
- uses: actions/setup-node@v6
with:
node-version: 22
cache: pnpm
registry-url: https://registry.npmjs.org
scope: "@wxg-prc-cpg"
- name: Install dependencies
run: pnpm install --frozen-lockfile
- name: Typecheck plugin
run: pnpm --filter "${{ env.PACKAGE_NAME }}" typecheck
- name: Test plugin
run: pnpm --filter "${{ env.PACKAGE_NAME }}" test
- name: Publish to npm
working-directory: ${{ env.PACKAGE_DIR }}
env:
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
run: |
set -euo pipefail
if [ -z "${NODE_AUTH_TOKEN:-}" ]; then
echo "::error::Environment secret NPM_TOKEN is not configured"
exit 1
fi
pnpm publish --access public --no-git-checks
echo "Published ${PACKAGE_NAME}@${{ needs.resolve.outputs.version }}"
-177
View File
@@ -325,180 +325,3 @@ jobs:
echo "Chrome Web Store submission created successfully:" echo "Chrome Web Store submission created successfully:"
jq '{itemId, state, warningInfo}' "$PUBLISH_RESPONSE" jq '{itemId, state, warningInfo}' "$PUBLISH_RESPONSE"
publish-edge-addons:
needs: [resolve, build-extension]
runs-on: ubuntu-latest
environment: edge-addons
permissions:
contents: read
steps:
- name: Download build artifact
uses: actions/download-artifact@v8
with:
name: extension-zip
path: dist
- name: Validate Microsoft Edge Add-ons configuration
shell: bash
env:
API_KEY: ${{ secrets.EDGE_ADDONS_API_KEY }}
CLIENT_ID: ${{ vars.EDGE_ADDONS_CLIENT_ID }}
PRODUCT_ID: ${{ vars.EDGE_ADDONS_PRODUCT_ID }}
run: |
set -euo pipefail
if [ -z "$API_KEY" ]; then
echo "::error::Environment secret EDGE_ADDONS_API_KEY is not configured"
exit 1
fi
if [ -z "$CLIENT_ID" ]; then
echo "::error::Environment variable EDGE_ADDONS_CLIENT_ID is not configured"
exit 1
fi
if [ -z "$PRODUCT_ID" ]; then
echo "::error::Environment variable EDGE_ADDONS_PRODUCT_ID is not configured"
exit 1
fi
if ! [[ "$PRODUCT_ID" =~ ^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$ ]]; then
echo "::error::EDGE_ADDONS_PRODUCT_ID must be the Partner Center product GUID, not the public Edge Add-ons extension ID"
exit 1
fi
ARCHIVE="$(find dist -maxdepth 1 -name '*.zip' -type f -print -quit)"
if [ -z "$ARCHIVE" ]; then
echo "::error::No extension zip found in the downloaded artifact"
exit 1
fi
echo "ARCHIVE=$ARCHIVE" >> "$GITHUB_ENV"
- name: Upload and publish Microsoft Edge Add-ons extension
shell: bash
env:
API_KEY: ${{ secrets.EDGE_ADDONS_API_KEY }}
CLIENT_ID: ${{ vars.EDGE_ADDONS_CLIENT_ID }}
PRODUCT_ID: ${{ vars.EDGE_ADDONS_PRODUCT_ID }}
RELEASE_VERSION: ${{ needs.resolve.outputs.version }}
run: |
set -euo pipefail
API_ROOT="https://api.addons.microsoftedge.microsoft.com/v1"
UPLOAD_HEADERS="$RUNNER_TEMP/edge-upload-headers.txt"
UPLOAD_RESPONSE="$RUNNER_TEMP/edge-upload-response.json"
STATUS_RESPONSE="$RUNNER_TEMP/edge-status-response.json"
PUBLISH_HEADERS="$RUNNER_TEMP/edge-publish-headers.txt"
PUBLISH_RESPONSE="$RUNNER_TEMP/edge-publish-response.json"
operation_id_from_headers() {
local headers="$1"
local location
location="$(awk '
tolower($1) == "location:" {
sub(/\r$/, "")
sub(/^[^:]*:[[:space:]]*/, "")
print
exit
}
' "$headers")"
location="${location%%\?*}"
printf '%s' "${location##*/}"
}
poll_operation() {
local operation_name="$1"
local status_url="$2"
local status="InProgress"
local http_status
for attempt in $(seq 1 30); do
sleep 5
http_status="$(curl --silent --show-error \
--output "$STATUS_RESPONSE" \
--write-out '%{http_code}' \
--header "Authorization: ApiKey $API_KEY" \
--header "X-ClientID: $CLIENT_ID" \
"$status_url")"
if [ "$http_status" -lt 200 ] || [ "$http_status" -ge 300 ]; then
echo "::error::Microsoft Edge Add-ons $operation_name status check failed with HTTP $http_status"
jq . "$STATUS_RESPONSE" || cat "$STATUS_RESPONSE"
exit 1
fi
status="$(jq -r '.status // empty' "$STATUS_RESPONSE")"
echo "$operation_name status check $attempt/30: $status"
case "$status" in
Succeeded)
return 0
;;
InProgress)
;;
Failed)
echo "::error::Microsoft Edge Add-ons $operation_name failed"
jq . "$STATUS_RESPONSE"
exit 1
;;
*)
echo "::error::Microsoft Edge Add-ons returned an unknown $operation_name status: $status"
jq . "$STATUS_RESPONSE"
exit 1
;;
esac
done
echo "::error::Microsoft Edge Add-ons $operation_name did not complete after 30 checks"
jq . "$STATUS_RESPONSE"
exit 1
}
HTTP_STATUS="$(curl --silent --show-error \
--output "$UPLOAD_RESPONSE" \
--dump-header "$UPLOAD_HEADERS" \
--write-out '%{http_code}' \
--request POST \
--header "Authorization: ApiKey $API_KEY" \
--header "X-ClientID: $CLIENT_ID" \
--header "Content-Type: application/zip" \
--data-binary "@$ARCHIVE" \
"$API_ROOT/products/$PRODUCT_ID/submissions/draft/package")"
if [ "$HTTP_STATUS" != "202" ]; then
echo "::error::Microsoft Edge Add-ons upload failed with HTTP $HTTP_STATUS"
jq . "$UPLOAD_RESPONSE" || cat "$UPLOAD_RESPONSE"
exit 1
fi
UPLOAD_OPERATION_ID="$(operation_id_from_headers "$UPLOAD_HEADERS")"
if [ -z "$UPLOAD_OPERATION_ID" ]; then
echo "::error::Microsoft Edge Add-ons upload response has no Location operation ID"
cat "$UPLOAD_HEADERS"
exit 1
fi
poll_operation \
"package upload" \
"$API_ROOT/products/$PRODUCT_ID/submissions/draft/package/operations/$UPLOAD_OPERATION_ID"
HTTP_STATUS="$(curl --silent --show-error \
--output "$PUBLISH_RESPONSE" \
--dump-header "$PUBLISH_HEADERS" \
--write-out '%{http_code}' \
--request POST \
--header "Authorization: ApiKey $API_KEY" \
--header "X-ClientID: $CLIENT_ID" \
--header "Content-Type: application/json" \
--data "{\"notes\":\"Automated BrowserSkill extension v${RELEASE_VERSION} release from GitHub Actions.\"}" \
"$API_ROOT/products/$PRODUCT_ID/submissions")"
if [ "$HTTP_STATUS" != "202" ]; then
echo "::error::Microsoft Edge Add-ons publish failed with HTTP $HTTP_STATUS"
jq . "$PUBLISH_RESPONSE" || cat "$PUBLISH_RESPONSE"
exit 1
fi
PUBLISH_OPERATION_ID="$(operation_id_from_headers "$PUBLISH_HEADERS")"
if [ -z "$PUBLISH_OPERATION_ID" ]; then
echo "::error::Microsoft Edge Add-ons publish response has no Location operation ID"
cat "$PUBLISH_HEADERS"
exit 1
fi
poll_operation \
"submission" \
"$API_ROOT/products/$PRODUCT_ID/submissions/operations/$PUBLISH_OPERATION_ID"
echo "Microsoft Edge Add-ons submission created successfully:"
jq '{status, message, errorCode, errors}' "$STATUS_RESPONSE"
-4
View File
@@ -11,10 +11,6 @@ dist/
# Logs # Logs
*.log *.log
# Local browser evaluation artifacts
evals/browser/results/
evals/browser/agents.local.json
# macOS # macOS
.DS_Store .DS_Store
-3
View File
@@ -3,6 +3,3 @@
**/.output/** **/.output/**
**/target/** **/target/**
**/node_modules/** **/node_modules/**
# Generated client CSS (tailwind compile output); built by
# packages/dsh-plugin-browserskill/scripts/build-client-css.mjs.
**/dsh-plugin-browserskill/src/client/bsk-ui.nomodule.css
+5 -9
View File
@@ -34,17 +34,13 @@ where only `extension connected` fails is expected; go to Step 3.
## 3. Open the extension install page ## 3. Open the extension install page
If `extension connected` is `FAIL` (`0 browsers connected`), the user likely If `extension connected` is `FAIL` (`0 browsers connected`), the user likely
has not installed or enabled the browser extension yet. Open the install page has not installed or enabled the browser extension yet. Open the
matching the user's browser — [Chrome Web Store install page](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi)
[Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi) in the user's browser, then tell the user:
for Chrome and other Chromium browsers,
[Edge Add-ons](https://microsoftedge.microsoft.com/addons/detail/browserskill/emacgiaaaiojkkpkddmmdfhmokgmnikg)
for Microsoft Edge — then tell the user:
> Install **BrowserSkill** from the > Install **BrowserSkill** from the
> [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi) > [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi),
> (or [Edge Add-ons](https://microsoftedge.microsoft.com/addons/detail/browserskill/emacgiaaaiojkkpkddmmdfhmokgmnikg) > open the popup, and wait until it turns green. Reply when done.
> on Edge), open the popup, and wait until it turns green. Reply when done.
If opening the page fails, give the user the same link. Then run `bsk doctor` If opening the page fails, give the user the same link. Then run `bsk doctor`
once more. All `ok`/`na` → tell the user it's ready. once more. All `ok`/`na` → tell the user it's ready.
Generated
+2 -2
View File
@@ -134,7 +134,7 @@ dependencies = [
[[package]] [[package]]
name = "bsk" name = "bsk"
version = "0.1.11" version = "0.1.10"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"bsk-protocol", "bsk-protocol",
@@ -168,7 +168,7 @@ dependencies = [
[[package]] [[package]]
name = "bsk-protocol" name = "bsk-protocol"
version = "0.1.11" version = "0.1.10"
dependencies = [ dependencies = [
"schemars", "schemars",
"semver", "semver",
+1 -1
View File
@@ -7,7 +7,7 @@ members = [
default-members = ["crates/bsk-cli"] default-members = ["crates/bsk-cli"]
[workspace.package] [workspace.package]
version = "0.1.11" version = "0.1.10"
edition = "2024" edition = "2024"
rust-version = "1.85" rust-version = "1.85"
license = "MIT" license = "MIT"
+5 -38
View File
@@ -13,7 +13,7 @@
</p> </p>
**BrowserSkill** connects Cursor, Claude Code, Codex, OpenClaw, CodeBuddy, **BrowserSkill** connects Cursor, Claude Code, Codex, OpenClaw, CodeBuddy,
WorkBuddy, Pi, Hermes Agent, DeepSeek Harness, and other AI agents to your already logged-in WorkBuddy, Pi, Hermes Agent, and other shell-capable AI agents to your already logged-in
browser. browser.
Need the agent to touch a tab you already have open? It must borrow that tab Need the agent to touch a tab you already have open? It must borrow that tab
@@ -67,8 +67,7 @@ Set up browser-skill on this machine by following https://raw.githubusercontent.
<br> <br>
Install the CLI, then install the extension from the [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi) Install the CLI, then install the extension from the [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi).
or [Edge Add-ons](https://microsoftedge.microsoft.com/addons/detail/browserskill/emacgiaaaiojkkpkddmmdfhmokgmnikg).
#### 1. Install the `bsk` CLI #### 1. Install the `bsk` CLI
@@ -92,14 +91,7 @@ bsk --version
#### 2. Install the browser extension #### 2. Install the browser extension
Install BrowserSkill from your browser's store: Install BrowserSkill from the [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi).
| Browser | Store listing |
| --- | --- |
| Chrome | [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi) |
| Microsoft Edge | [Edge Add-ons](https://microsoftedge.microsoft.com/addons/detail/browserskill/emacgiaaaiojkkpkddmmdfhmokgmnikg) |
On other Chromium-based browsers, install the Chrome Web Store build.
#### 3. Install the skill #### 3. Install the skill
@@ -131,8 +123,7 @@ internal variants and install paths.
Other shell-capable agent harnesses are supported too. Copy Other shell-capable agent harnesses are supported too. Copy
[`skill/SKILL.md`](skill/SKILL.md) into your harness's skills directory as [`skill/SKILL.md`](skill/SKILL.md) into your harness's skills directory as
`browser-skill/SKILL.md` to install the skill manually. DeepSeek Harness uses a `browser-skill/SKILL.md` to install the skill manually.
dedicated plugin instead — see [DeepSeek Harness plugin](#deepseek-harness-plugin).
</details> </details>
@@ -142,26 +133,6 @@ Start a new Agent session and write a prompt that needs the browser, for example
/browser-skill open example.com and summarize what is on the page. /browser-skill open example.com and summarize what is on the page.
``` ```
## DeepSeek Harness plugin
Using [DeepSeek Harness](https://github.com/deepseek-ai/deepseek-harness) (`dsh`)?
BrowserSkill ships a first-class dsh plugin on npm as
[`@wxg-prc-cpg/browser-skill-dsh-plugin`](https://www.npmjs.com/package/@wxg-prc-cpg/browser-skill-dsh-plugin).
It injects native `browser_*` tools (no shelling out to `bsk`) and a live Web UI
overlay of each Agent Window.
Add it to a dsh profile, then start that profile:
```sh
dsh plugin --profile web add @wxg-prc-cpg/browser-skill-dsh-plugin
dsh --profile web
```
The plugin carries its own copy of the skill, so `bsk install-skill` is not needed
for dsh — but the `bsk` CLI and the browser extension are still prerequisites. See
the [plugin README](packages/dsh-plugin-browserskill/README.md) for the tool list,
configuration, and the observation overlay.
## How It Works ## How It Works
BrowserSkill is a local bridge between your agent harness and your browser. BrowserSkill is a local bridge between your agent harness and your browser.
@@ -195,9 +166,7 @@ flowchart TB
The agent never talks to the browser directly. It asks the `bsk` CLI to perform a The agent never talks to the browser directly. It asks the `bsk` CLI to perform a
browser task; the local daemon routes that request to the extension; the browser task; the local daemon routes that request to the extension; the
extension runs it in an Agent Window. DeepSeek Harness takes the same path extension runs it in an Agent Window.
through the [plugin](#deepseek-harness-plugin): the agent calls injected
`browser_*` tools, and the plugin invokes `bsk` on its behalf.
## For Developers ## For Developers
@@ -207,8 +176,6 @@ The repository is a Cargo + pnpm workspace:
- `crates/bsk-protocol` — shared wire types and JSON schemas - `crates/bsk-protocol` — shared wire types and JSON schemas
- `apps/extension` — browser extension - `apps/extension` — browser extension
- `packages/ui` and `packages/i18n` — shared extension UI support - `packages/ui` and `packages/i18n` — shared extension UI support
- `packages/dsh-plugin-browserskill` — DeepSeek Harness plugin (`@wxg-prc-cpg/browser-skill-dsh-plugin`)
- [`evals/browser`](evals/browser/README.md) — deterministic local pages and agent-neutral browser capability evaluation
## License ## License
+7 -33
View File
@@ -12,7 +12,7 @@
<a href="README.md">English</a> · 中文 <a href="README.md">English</a> · 中文
</p> </p>
**BrowserSkill** 把 Cursor、Claude Code、Codex、OpenClaw、CodeBuddy、WorkBuddy、Pi、Hermes Agent、DeepSeek Harness 等 AI Agent 连接到你已登录的浏览器。 **BrowserSkill** 把 Cursor、Claude Code、Codex、OpenClaw、CodeBuddy、WorkBuddy、Pi、Hermes Agent 等支持 Shell 的 AI Agent 连接到你已登录的浏览器。
需要 Agent 操作你已打开的标签页?必须显式借用该标签,任务结束后归还,其余浏览器窗口不受影响。 需要 Agent 操作你已打开的标签页?必须显式借用该标签,任务结束后归还,其余浏览器窗口不受影响。
@@ -54,8 +54,7 @@ BrowserSkill 由两个本地运行组件组成:`bsk` CLI/daemon 和浏览器
<br> <br>
先安装 CLI,再从 [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi) 先安装 CLI,再从 [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi) 安装浏览器扩展。
或 [Edge 加载项商店](https://microsoftedge.microsoft.com/addons/detail/browserskill/emacgiaaaiojkkpkddmmdfhmokgmnikg) 安装浏览器扩展。
#### 1. 安装 `bsk` CLI #### 1. 安装 `bsk` CLI
@@ -65,11 +64,8 @@ BrowserSkill 由两个本地运行组件组成:`bsk` CLI/daemon 和浏览器
curl -fsSL https://raw.githubusercontent.com/Tencent/BrowserSkill/main/install.sh | sh curl -fsSL https://raw.githubusercontent.com/Tencent/BrowserSkill/main/install.sh | sh
``` ```
**Windows**(PowerShell,安装到 `~/.local/bin`): **Windows**:从 [最新 CLI release](https://github.com/Tencent/BrowserSkill/releases/latest)
下载 `bsk-v<version>-x86_64-pc-windows-msvc.zip`,解压后将 `bsk.exe` 加入 `PATH`。
```powershell
irm https://raw.githubusercontent.com/Tencent/BrowserSkill/main/install.ps1 | iex
```
验证二进制: 验证二进制:
@@ -79,14 +75,7 @@ bsk --version
#### 2. 安装浏览器扩展 #### 2. 安装浏览器扩展
在对应浏览器的商店安装 BrowserSkill: 从 [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi) 安装 BrowserSkill。
| 浏览器 | 商店页面 |
| --- | --- |
| Chrome | [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi) |
| Microsoft Edge | [Edge 加载项商店](https://microsoftedge.microsoft.com/addons/detail/browserskill/emacgiaaaiojkkpkddmmdfhmokgmnikg) |
其他基于 Chromium 的浏览器,安装 Chrome Web Store 版本即可。
#### 3. 安装 skill #### 3. 安装 skill
@@ -113,7 +102,7 @@ bsk install-skill
用 <kbd>Space</kbd> 选择需要安装的 Agent harness,然后按 <kbd>Enter</kbd> 安装 skill。运行 `bsk install-skill --list` 可查看 internal 变体及安装路径。 用 <kbd>Space</kbd> 选择需要安装的 Agent harness,然后按 <kbd>Enter</kbd> 安装 skill。运行 `bsk install-skill --list` 可查看 internal 变体及安装路径。
其他支持 Shell 的 Agent harness 也可使用 BrowserSkill,但需手动将 [`skill/SKILL.md`](skill/SKILL.md) 复制到对应 skills 目录下的 `browser-skill/SKILL.md`。DeepSeek Harness 走独立插件,见 [DeepSeek Harness 插件](#deepseek-harness-插件)。 其他支持 Shell 的 Agent harness 也可使用 BrowserSkill,但需手动将 [`skill/SKILL.md`](skill/SKILL.md) 复制到对应 skills 目录下的 `browser-skill/SKILL.md`。
</details> </details>
@@ -123,19 +112,6 @@ bsk install-skill
/browser-skill open example.com and summarize what is on the page. /browser-skill open example.com and summarize what is on the page.
``` ```
## DeepSeek Harness 插件
在用 [DeepSeek Harness](https://github.com/deepseek-ai/deepseek-harness)(`dsh`)?BrowserSkill 提供了官方 dsh 插件,已发布到 npm:[`@wxg-prc-cpg/browser-skill-dsh-plugin`](https://www.npmjs.com/package/@wxg-prc-cpg/browser-skill-dsh-plugin)。它会注入原生 `browser_*` 工具(无需再通过 Shell 调用 `bsk`),并在 Web UI 中实时观察每个 Agent Window。
把它装进某个 dsh profile,然后启动该 profile:
```sh
dsh plugin --profile web add @wxg-prc-cpg/browser-skill-dsh-plugin
dsh --profile web
```
插件自带 skill,所以在 dsh 下无需执行 `bsk install-skill`;但 `bsk` CLI 和浏览器扩展仍是前置条件。工具清单、配置项与观察浮层见[插件 README](packages/dsh-plugin-browserskill/README.md)。
## 工作原理 ## 工作原理
BrowserSkill 是 Agent 运行时与浏览器之间的本地桥接层。 BrowserSkill 是 Agent 运行时与浏览器之间的本地桥接层。
@@ -167,7 +143,7 @@ flowchart TB
style UserWindows fill:#f8fafc,stroke:#cbd5e1,color:#334155 style UserWindows fill:#f8fafc,stroke:#cbd5e1,color:#334155
``` ```
Agent 不直接与浏览器通信。它通过 `bsk` CLI 下发浏览器任务;本地 daemon 把请求路由到扩展;扩展在 Agent Window 中执行。DeepSeek Harness 走同一条链路,只是经由 [插件](#deepseek-harness-插件):Agent 调用注入的 `browser_*` 工具,由插件代为执行 `bsk`。 Agent 不直接与浏览器通信。它通过 `bsk` CLI 下发浏览器任务;本地 daemon 把请求路由到扩展;扩展在 Agent Window 中执行。
## 面向开发者 ## 面向开发者
@@ -177,8 +153,6 @@ Agent 不直接与浏览器通信。它通过 `bsk` CLI 下发浏览器任务;
- `crates/bsk-protocol` — 共享协议类型与 JSON Schema - `crates/bsk-protocol` — 共享协议类型与 JSON Schema
- `apps/extension` — 浏览器扩展 - `apps/extension` — 浏览器扩展
- `packages/ui` 和 `packages/i18n` — 扩展 UI 共享支持 - `packages/ui` 和 `packages/i18n` — 扩展 UI 共享支持
- `packages/dsh-plugin-browserskill` — DeepSeek Harness 插件(`@wxg-prc-cpg/browser-skill-dsh-plugin`)
- [`evals/browser`](evals/browser/README.zh-CN.md) — 确定性本地页面与 Agent 无关的浏览器能力测试台
## 许可证 ## 许可证
+1 -1
View File
@@ -1,6 +1,6 @@
{ {
"name": "@browser-skill/extension", "name": "@browser-skill/extension",
"version": "0.1.7", "version": "0.1.6",
"private": true, "private": true,
"type": "module", "type": "module",
"scripts": { "scripts": {
@@ -1,5 +1,5 @@
import { describe, expect, it, vi } from "vitest"; import { describe, expect, it, vi } from "vitest";
import { type CdpDebuggee, type CdpDebuggerApi, ChromiumCdp } from "../chromium-cdp"; import { type CdpDebuggerApi, ChromiumCdp } from "../chromium-cdp";
function fakeChromeEvent<TArgs extends unknown[]>() { function fakeChromeEvent<TArgs extends unknown[]>() {
const listeners = new Set<(...args: TArgs) => void>(); const listeners = new Set<(...args: TArgs) => void>();
@@ -14,7 +14,7 @@ function fakeChromeEvent<TArgs extends unknown[]>() {
} }
function fakeApi() { function fakeApi() {
const onEvent = fakeChromeEvent<[CdpDebuggee, string, unknown]>(); const onEvent = fakeChromeEvent<[chrome.debugger.Debuggee, string, unknown]>();
const onDetach = fakeChromeEvent<[chrome.debugger.Debuggee, string]>(); const onDetach = fakeChromeEvent<[chrome.debugger.Debuggee, string]>();
const api: CdpDebuggerApi = { const api: CdpDebuggerApi = {
attach: vi.fn(async () => {}), attach: vi.fn(async () => {}),
@@ -29,135 +29,6 @@ function fakeApi() {
} }
describe("ChromiumCdp", () => { describe("ChromiumCdp", () => {
it("discovers multiple iframe targets and recursively routes nested OOPIF commands", async () => {
const { api, onEvent } = fakeApi();
(api.sendCommand as ReturnType<typeof vi.fn>).mockImplementation(
async (
target: chrome.debugger.Debuggee & { sessionId?: string },
method: string,
params?: { frameId?: string },
) => {
if (method === "DOM.getFrameOwner") {
return { backendNodeId: params?.frameId === "nested" ? 300 : 200 };
}
if (method !== "Page.getFrameTree") return {};
if (target.sessionId === "right-session") {
return {
frameTree: {
frame: { id: "right", url: "https://right.test" },
childFrames: [
{ frame: { id: "nested", parentId: "right", url: "https://nested.test" } },
],
},
};
}
if (target.sessionId === "nested-session") {
return { frameTree: { frame: { id: "nested", url: "https://nested.test" } } };
}
return {
frameTree: {
frame: { id: "main", url: "https://app.test" },
childFrames: [
{ frame: { id: "left", parentId: "main", url: "https://left.test" } },
{
frame: { id: "right", parentId: "main", url: "https://right.test" },
childFrames: [
{ frame: { id: "nested", parentId: "right", url: "https://nested.test" } },
],
},
],
},
};
},
);
const cdp = new ChromiumCdp(api);
await cdp.ensureAttached(4);
onEvent.fire({ tabId: 4 }, "Target.attachedToTarget", { sessionId: "right-session" });
onEvent.fire({ tabId: 4 } as chrome.debugger.Debuggee, "Target.attachedToTarget", {
sessionId: "nested-session",
});
const graph = await cdp.getFrameGraph(4);
expect(graph.frames).toHaveLength(4);
expect(graph.frames.find((frame) => frame.frameId === "left")?.target).toEqual({ tabId: 4 });
expect(graph.frames.find((frame) => frame.frameId === "right")?.target.sessionId).toBe(
"right-session",
);
expect(graph.frames.find((frame) => frame.frameId === "nested")?.target.sessionId).toBe(
"nested-session",
);
expect(graph.frames.find((frame) => frame.frameId === "right")?.ownerBackendNodeId).toBe(200);
expect(graph.frames.find((frame) => frame.frameId === "nested")?.ownerBackendNodeId).toBe(300);
expect(api.sendCommand).toHaveBeenCalledWith(
{ tabId: 4, sessionId: "right-session" },
"DOM.getFrameOwner",
{ frameId: "nested" },
);
expect(api.sendCommand).toHaveBeenCalledWith(
{ tabId: 4, sessionId: "nested-session" },
"Target.setAutoAttach",
expect.objectContaining({ flatten: true }),
);
});
it("waits for recursively attached iframe targets without a depth limit", async () => {
const { api, onEvent } = fakeApi();
const depth = 6;
const scheduled = new Set<string>();
const nestedTree = (index: number): Record<string, unknown> => ({
frame: {
id: index === 0 ? "main" : `frame-${index}`,
...(index > 0 ? { parentId: index === 1 ? "main" : `frame-${index - 1}` } : {}),
},
...(index < depth ? { childFrames: [nestedTree(index + 1)] } : {}),
});
(api.sendCommand as ReturnType<typeof vi.fn>).mockImplementation(
async (target: CdpDebuggee, method: string, params?: { frameId?: string }) => {
if (method === "Target.setAutoAttach") {
const parentIndex = target.sessionId
? Number(target.sessionId.replace("session-", ""))
: 0;
const nextIndex = parentIndex + 1;
const key = `${target.sessionId ?? "root"}:${nextIndex}`;
if (nextIndex <= depth && !scheduled.has(key)) {
scheduled.add(key);
setTimeout(() => {
onEvent.fire(
{ tabId: 4, ...(target.sessionId ? { sessionId: target.sessionId } : {}) },
"Target.attachedToTarget",
{
sessionId: `session-${nextIndex}`,
targetInfo: { type: "iframe" },
},
);
}, 5);
}
return {};
}
if (method === "Page.getFrameTree") {
if (!target.sessionId) return { frameTree: nestedTree(0) };
const index = Number(target.sessionId.replace("session-", ""));
return { frameTree: nestedTree(index) };
}
if (method === "DOM.getFrameOwner") {
return { backendNodeId: Number(params?.frameId?.replace("frame-", "")) + 100 };
}
return {};
},
);
const cdp = new ChromiumCdp(api);
const graph = await cdp.getFrameGraph(4);
expect(graph.frames).toHaveLength(depth + 1);
for (let index = 1; index <= depth; index += 1) {
expect(
graph.frames.find((frame) => frame.frameId === `frame-${index}`)?.target.sessionId,
).toBe(`session-${index}`);
}
});
it("coalesces concurrent attach calls for the same tab", async () => { it("coalesces concurrent attach calls for the same tab", async () => {
const { api } = fakeApi(); const { api } = fakeApi();
let releaseAttach!: () => void; let releaseAttach!: () => void;
@@ -1,136 +0,0 @@
import { describe, expect, it } from "vitest";
import { buildFrameGraph, type CdpFrameTreeNode, type CdpFrameTreeSource } from "../frame-graph";
describe("buildFrameGraph", () => {
it("keeps sibling frames distinct and routes nested OOPIFs to their child sessions", () => {
const graph = buildFrameGraph([
{
target: { tabId: 4 },
tree: {
frame: { id: "main", url: "https://app.test" },
childFrames: [
{ frame: { id: "left", parentId: "main", url: "https://left.test" } },
{
frame: { id: "right", parentId: "main", url: "https://right.test" },
childFrames: [
{ frame: { id: "nested", parentId: "right", url: "https://nested.test" } },
],
},
],
},
},
{
target: { tabId: 4, sessionId: "right-session" },
tree: {
frame: { id: "right", url: "https://right.test" },
childFrames: [{ frame: { id: "nested", parentId: "right", url: "https://nested.test" } }],
},
},
{
target: { tabId: 4, sessionId: "nested-session" },
tree: { frame: { id: "nested", url: "https://nested.test" } },
},
]);
expect(graph?.frames).toHaveLength(4);
expect(graph?.frames.find((frame) => frame.frameId === "left")?.target).toEqual({ tabId: 4 });
expect(graph?.frames.find((frame) => frame.frameId === "right")).toMatchObject({
parentFrameId: "main",
target: { tabId: 4, sessionId: "right-session" },
});
expect(graph?.frames.find((frame) => frame.frameId === "nested")).toMatchObject({
parentFrameId: "right",
target: { tabId: 4, sessionId: "nested-session" },
});
});
it("propagates each target boundary to its same-process descendants", () => {
const sources: CdpFrameTreeSource[] = [
{
target: { tabId: 4 },
tree: {
frame: { id: "main" },
childFrames: [
{
frame: { id: "oopif-a", parentId: "main" },
childFrames: [
{
frame: { id: "same-process-b", parentId: "oopif-a" },
childFrames: [
{
frame: { id: "oopif-c", parentId: "same-process-b" },
childFrames: [{ frame: { id: "same-process-d", parentId: "oopif-c" } }],
},
],
},
],
},
],
},
},
{
target: { tabId: 4, sessionId: "session-a" },
tree: {
frame: { id: "oopif-a" },
childFrames: [
{
frame: { id: "same-process-b", parentId: "oopif-a" },
childFrames: [{ frame: { id: "oopif-c", parentId: "same-process-b" } }],
},
],
},
},
{
target: { tabId: 4, sessionId: "session-c" },
tree: {
frame: { id: "oopif-c" },
childFrames: [{ frame: { id: "same-process-d", parentId: "oopif-c" } }],
},
},
];
const graph = buildFrameGraph(sources);
expect(graph?.frames.map((frame) => [frame.frameId, frame.target.sessionId])).toEqual([
["main", undefined],
["oopif-a", "session-a"],
["same-process-b", "session-a"],
["oopif-c", "session-c"],
["same-process-d", "session-c"],
]);
expect(graph?.frames.find((frame) => frame.frameId === "oopif-a")?.parentFrameId).toBe("main");
expect(graph?.frames.find((frame) => frame.frameId === "oopif-c")?.parentFrameId).toBe(
"same-process-b",
);
const reordered = buildFrameGraph([sources[2], sources[1], sources[0]]);
expect(reordered).toEqual(graph);
});
it("walks deeply nested frame trees without consuming the JavaScript call stack", () => {
const depth = 5_000;
const root: CdpFrameTreeNode = { frame: { id: "frame-0" } };
let parent = root;
for (let index = 1; index <= depth; index += 1) {
const child: CdpFrameTreeNode = {
frame: { id: `frame-${index}`, parentId: `frame-${index - 1}` },
};
parent.childFrames = [child];
parent = child;
}
const graph = buildFrameGraph([{ target: { tabId: 4 }, tree: root }]);
expect(graph?.frames).toHaveLength(depth + 1);
expect(graph?.frames.at(-1)?.frameId).toBe(`frame-${depth}`);
});
it("does not loop when an in-memory frame tree contains an object cycle", () => {
const root: CdpFrameTreeNode = { frame: { id: "main" } };
root.childFrames = [root];
expect(buildFrameGraph([{ target: { tabId: 4 }, tree: root }])?.frames).toEqual([
{ frameId: "main", target: { tabId: 4 } },
]);
});
});
+12 -211
View File
@@ -31,15 +31,6 @@ import type {
NetworkEntryKind, NetworkEntryKind,
NetworkResult, NetworkResult,
} from "@/transport/types"; } from "@/transport/types";
import {
buildFrameGraph,
type CdpFrameGraph,
type CdpFrameTreeNode,
type CdpFrameTreeSource,
type CdpTarget,
} from "./frame-graph";
export type CdpDebuggee = chrome.debugger.Debuggee & { sessionId?: string };
/** /**
* Minimal slice of `chrome.debugger` the rest of the extension * Minimal slice of `chrome.debugger` the rest of the extension
@@ -47,15 +38,21 @@ export type CdpDebuggee = chrome.debugger.Debuggee & { sessionId?: string };
* fake without monkey-patching the real `chrome` global. * fake without monkey-patching the real `chrome` global.
*/ */
export interface CdpDebuggerApi { export interface CdpDebuggerApi {
attach(target: CdpDebuggee, requiredVersion: string): Promise<void>; attach(target: chrome.debugger.Debuggee, requiredVersion: string): Promise<void>;
detach(target: CdpDebuggee): Promise<void>; detach(target: chrome.debugger.Debuggee): Promise<void>;
sendCommand(target: CdpDebuggee, method: string, commandParams?: object): Promise<unknown>; sendCommand(
target: chrome.debugger.Debuggee,
method: string,
commandParams?: object,
): Promise<unknown>;
/** /**
* Fires for every CDP event (`Page.lifecycleEvent`, `DOM.documentUpdated`, * Fires for every CDP event (`Page.lifecycleEvent`, `DOM.documentUpdated`,
* …). The first callback argument is the source debuggee; the second * …). The first callback argument is the source debuggee; the second
* is the CDP method name; the third is the payload. * is the CDP method name; the third is the payload.
*/ */
onEvent: chrome.events.Event<(source: CdpDebuggee, method: string, params: unknown) => void>; onEvent: chrome.events.Event<
(source: chrome.debugger.Debuggee, method: string, params: unknown) => void
>;
/** /**
* Fires when Chrome unilaterally detaches us — most commonly because * Fires when Chrome unilaterally detaches us — most commonly because
* the tab navigated to a chrome:// URL or the user clicked * the tab navigated to a chrome:// URL or the user clicked
@@ -113,8 +110,6 @@ const MAX_CONSOLE_STACK_FRAMES = 20;
const MAX_NETWORK_BUFFER = 200; const MAX_NETWORK_BUFFER = 200;
const MAX_NETWORK_FIELD_LENGTH = 4096; const MAX_NETWORK_FIELD_LENGTH = 4096;
const MAX_NETWORK_REQUEST_META = 1024; const MAX_NETWORK_REQUEST_META = 1024;
const FRAME_DISCOVERY_TIMEOUT_MS = 1000;
const FRAME_DISCOVERY_QUIET_MS = 20;
interface ParsedDialogOpening { interface ParsedDialogOpening {
type: JavaScriptDialogType; type: JavaScriptDialogType;
@@ -136,27 +131,6 @@ interface NetworkRequestMeta {
truncated: boolean; truncated: boolean;
} }
interface FrameDiscoveryState {
sessions: Set<string>;
pending: Set<Promise<void>>;
generation: number;
}
async function settleBeforeDeadline(promises: Promise<void>[], deadline: number): Promise<boolean> {
if (promises.length === 0) return true;
const remaining = deadline - Date.now();
if (remaining <= 0) return false;
let timer: ReturnType<typeof setTimeout> | undefined;
const timedOut = await Promise.race([
Promise.allSettled(promises).then(() => false),
new Promise<boolean>((resolve) => {
timer = setTimeout(() => resolve(true), remaining);
}),
]);
if (timer) clearTimeout(timer);
return !timedOut;
}
/** /**
* Wrapper around `chrome.debugger` that owns the "attach once per * Wrapper around `chrome.debugger` that owns the "attach once per
* tabId" cache and exposes typed `send<T>()`. * tabId" cache and exposes typed `send<T>()`.
@@ -175,12 +149,10 @@ export class ChromiumCdp {
private readonly networkSequences = new Map<number, number>(); private readonly networkSequences = new Map<number, number>();
private readonly networkDomainsEnabledTabs = new Set<number>(); private readonly networkDomainsEnabledTabs = new Set<number>();
private readonly networkRequestMeta = new Map<number, Map<string, NetworkRequestMeta>>(); private readonly networkRequestMeta = new Map<number, Map<string, NetworkRequestMeta>>();
private readonly frameDiscovery = new Map<number, FrameDiscoveryState>();
private detachSubscription: { dispose(): void } | null = null; private detachSubscription: { dispose(): void } | null = null;
private dialogSubscription: { dispose(): void } | null = null; private dialogSubscription: { dispose(): void } | null = null;
private consoleSubscription: { dispose(): void } | null = null; private consoleSubscription: { dispose(): void } | null = null;
private networkSubscription: { dispose(): void } | null = null; private networkSubscription: { dispose(): void } | null = null;
private frameTargetSubscription: { dispose(): void } | null = null;
constructor(api: CdpDebuggerApi = chromeDebuggerApi) { constructor(api: CdpDebuggerApi = chromeDebuggerApi) {
this.api = api; this.api = api;
@@ -188,7 +160,6 @@ export class ChromiumCdp {
this.bindDialogHandler(); this.bindDialogHandler();
this.bindConsoleHandler(); this.bindConsoleHandler();
this.bindNetworkHandler(); this.bindNetworkHandler();
this.bindFrameTargetHandler();
} }
/** Attach to `tabId` if we haven't already in this driver. */ /** Attach to `tabId` if we haven't already in this driver. */
@@ -206,9 +177,6 @@ export class ChromiumCdp {
await this.enableConsoleDomains(tabId); await this.enableConsoleDomains(tabId);
await this.enableNetworkDomainBestEffort(tabId); await this.enableNetworkDomainBestEffort(tabId);
this.attachedTabs.add(tabId); this.attachedTabs.add(tabId);
await this.enableFrameDiscovery({ tabId }).catch((err) => {
console.debug("[bsk cdp] frame discovery unavailable", { tabId, err });
});
} catch (err) { } catch (err) {
// A CDP domain enable failed after the raw attach succeeded // A CDP domain enable failed after the raw attach succeeded
// (e.g. `Page.enable` rejects because the tab just navigated to // (e.g. `Page.enable` rejects because the tab just navigated to
@@ -254,73 +222,6 @@ export class ChromiumCdp {
} }
} }
async sendToTarget<T = unknown>(target: CdpTarget, method: string, params?: object): Promise<T> {
if (!this.attachedTabs.has(target.tabId)) {
await this.ensureAttached(target.tabId);
}
try {
return (await this.api.sendCommand(target, method, params ?? {})) as T;
} catch (err) {
throw normalizeError(err);
}
}
async getFrameGraph(tabId: number): Promise<CdpFrameGraph> {
await this.ensureAttached(tabId);
await this.enableFrameDiscovery({ tabId }).catch(() => {});
await this.drainFrameAttachTasks(tabId);
const sources: CdpFrameTreeSource[] = [];
const root = await this.sendToTarget<{ frameTree?: CdpFrameTreeNode }>(
{ tabId },
"Page.getFrameTree",
{},
);
if (root.frameTree) sources.push({ target: { tabId }, tree: root.frameTree });
const sessions = [...(this.frameDiscovery.get(tabId)?.sessions ?? [])];
const childTrees = await Promise.all(
sessions.map(async (sessionId): Promise<CdpFrameTreeSource | null> => {
const target = { tabId, sessionId };
try {
const reply = await this.sendToTarget<{ frameTree?: CdpFrameTreeNode }>(
target,
"Page.getFrameTree",
{},
);
return reply.frameTree ? { target, tree: reply.frameTree } : null;
} catch {
return null;
}
}),
);
for (const source of childTrees) {
if (source) sources.push(source);
}
const graph = buildFrameGraph(sources);
if (!graph) throw new Error("Page.getFrameTree returned no root frame");
const frameById = new Map(graph.frames.map((frame) => [frame.frameId, frame]));
await Promise.all(
graph.frames.map(async (frame) => {
if (!frame.parentFrameId) return;
const parent = frameById.get(frame.parentFrameId);
if (!parent) return;
try {
const owner = await this.sendToTarget<{ backendNodeId?: number }>(
parent.target,
"DOM.getFrameOwner",
{ frameId: frame.frameId },
);
if (owner.backendNodeId !== undefined) frame.ownerBackendNodeId = owner.backendNodeId;
} catch {
// The frame may have navigated between tree capture and owner lookup.
}
}),
);
return graph;
}
/** Return a cursor marking the current dialog sequence for `tabId`. */ /** Return a cursor marking the current dialog sequence for `tabId`. */
dialogCursor(tabId: number): DialogCursor { dialogCursor(tabId: number): DialogCursor {
return this.dialogSequences.get(tabId) ?? 0; return this.dialogSequences.get(tabId) ?? 0;
@@ -431,7 +332,6 @@ export class ChromiumCdp {
this.clearDialogState(tabId); this.clearDialogState(tabId);
this.clearConsoleState(tabId); this.clearConsoleState(tabId);
this.clearNetworkState(tabId); this.clearNetworkState(tabId);
this.clearFrameState(tabId);
try { try {
await this.api.detach({ tabId }); await this.api.detach({ tabId });
} catch (err) { } catch (err) {
@@ -454,7 +354,7 @@ export class ChromiumCdp {
} }
/** Subscribe to all CDP events. Returned disposable removes the listener. */ /** Subscribe to all CDP events. Returned disposable removes the listener. */
onEvent(handler: (source: CdpDebuggee, method: string, params: unknown) => void): { onEvent(handler: (source: chrome.debugger.Debuggee, method: string, params: unknown) => void): {
dispose(): void; dispose(): void;
} { } {
this.api.onEvent.addListener(handler); this.api.onEvent.addListener(handler);
@@ -478,7 +378,6 @@ export class ChromiumCdp {
this.networkSequences.clear(); this.networkSequences.clear();
this.networkDomainsEnabledTabs.clear(); this.networkDomainsEnabledTabs.clear();
this.networkRequestMeta.clear(); this.networkRequestMeta.clear();
this.frameDiscovery.clear();
await Promise.all( await Promise.all(
tabs.map(async (tabId) => { tabs.map(async (tabId) => {
try { try {
@@ -529,104 +428,9 @@ export class ChromiumCdp {
} }
} }
private async enableFrameDiscovery(target: CdpTarget): Promise<void> {
await this.api.sendCommand(target, "Target.setAutoAttach", {
autoAttach: true,
waitForDebuggerOnStart: false,
flatten: true,
filter: [{ type: "iframe", exclude: false }],
});
}
private bindFrameTargetHandler(): void {
if (this.frameTargetSubscription) return;
const listener = (source: CdpDebuggee, method: string, params: unknown) => {
const tabId = source.tabId;
if (typeof tabId !== "number") return;
const raw = (params ?? {}) as Record<string, unknown>;
const sessionId = typeof raw.sessionId === "string" ? raw.sessionId : undefined;
if (!sessionId) return;
if (method === "Target.detachedFromTarget") {
const state = this.frameDiscovery.get(tabId);
if (state?.sessions.delete(sessionId)) state.generation += 1;
return;
}
if (method !== "Target.attachedToTarget") return;
const targetInfo = raw.targetInfo as { type?: string } | undefined;
if (targetInfo?.type && targetInfo.type !== "iframe") return;
const state = this.frameDiscoveryState(tabId);
if (state.sessions.has(sessionId)) return;
state.sessions.add(sessionId);
state.generation += 1;
const task = this.initializeFrameTarget({ tabId, sessionId });
state.pending.add(task);
void task.finally(() => {
state.pending.delete(task);
});
};
this.api.onEvent.addListener(listener);
this.frameTargetSubscription = {
dispose: () => this.api.onEvent.removeListener(listener),
};
}
private async initializeFrameTarget(target: CdpTarget): Promise<void> {
try {
await this.enableFrameDiscovery(target);
} catch (err) {
const state = this.frameDiscovery.get(target.tabId);
if (state?.sessions.delete(target.sessionId as string)) state.generation += 1;
console.debug("[bsk cdp] child frame target initialization failed", { target, err });
}
}
private async drainFrameAttachTasks(tabId: number): Promise<void> {
const deadline = Date.now() + FRAME_DISCOVERY_TIMEOUT_MS;
while (Date.now() < deadline) {
const state = this.frameDiscovery.get(tabId);
const generation = state?.generation ?? 0;
if (state?.pending.size && !(await settleBeforeDeadline([...state.pending], deadline))) break;
// Target.setAutoAttach may enqueue the next attachedToTarget event after
// its command promise settles. Wait for a short quiet window, then finish
// only if neither the state object nor its generation changed.
const quietTime = Math.min(FRAME_DISCOVERY_QUIET_MS, deadline - Date.now());
if (quietTime <= 0) break;
await new Promise<void>((resolve) => setTimeout(resolve, quietTime));
const current = this.frameDiscovery.get(tabId);
if (
current === state &&
(current?.generation ?? 0) === generation &&
!current?.pending.size
) {
return;
}
}
console.debug("[bsk cdp] frame discovery did not reach quiescence before timeout", { tabId });
}
private frameDiscoveryState(tabId: number): FrameDiscoveryState {
const existing = this.frameDiscovery.get(tabId);
if (existing) return existing;
const created: FrameDiscoveryState = {
sessions: new Set(),
pending: new Set(),
generation: 0,
};
this.frameDiscovery.set(tabId, created);
return created;
}
private clearFrameState(tabId: number): void {
this.frameDiscovery.delete(tabId);
}
private bindDialogHandler(): void { private bindDialogHandler(): void {
if (this.dialogSubscription) return; if (this.dialogSubscription) return;
const listener = (source: CdpDebuggee, method: string, params: unknown) => { const listener = (source: chrome.debugger.Debuggee, method: string, params: unknown) => {
if (method !== "Page.javascriptDialogOpening") return; if (method !== "Page.javascriptDialogOpening") return;
const tabId = source.tabId; const tabId = source.tabId;
if (typeof tabId !== "number") return; if (typeof tabId !== "number") return;
@@ -810,7 +614,6 @@ export class ChromiumCdp {
this.clearDialogState(source.tabId); this.clearDialogState(source.tabId);
this.clearConsoleState(source.tabId); this.clearConsoleState(source.tabId);
this.clearNetworkState(source.tabId); this.clearNetworkState(source.tabId);
this.clearFrameState(source.tabId);
} }
}; };
this.api.onDetach.addListener(listener); this.api.onDetach.addListener(listener);
@@ -829,8 +632,6 @@ export class ChromiumCdp {
this.consoleSubscription = null; this.consoleSubscription = null;
this.networkSubscription?.dispose(); this.networkSubscription?.dispose();
this.networkSubscription = null; this.networkSubscription = null;
this.frameTargetSubscription?.dispose();
this.frameTargetSubscription = null;
} }
/** Detach tabs only when no other live session has claimed them. */ /** Detach tabs only when no other live session has claimed them. */
@@ -1,107 +0,0 @@
export interface CdpTarget {
tabId: number;
sessionId?: string;
}
export interface CdpFrame {
frameId: string;
parentFrameId?: string;
ownerBackendNodeId?: number;
url?: string;
target: CdpTarget;
}
export interface CdpFrameGraph {
rootFrameId: string;
frames: CdpFrame[];
}
export interface CdpFrameTreeNode {
frame: {
id: string;
parentId?: string;
url?: string;
};
childFrames?: CdpFrameTreeNode[];
}
export interface CdpFrameTreeSource {
target: CdpTarget;
tree: CdpFrameTreeNode;
}
export function cdpTargetKey(target: CdpTarget): string {
return `${target.tabId}:${target.sessionId ?? "root"}`;
}
function mergeFrame(
frames: Map<string, CdpFrame>,
order: string[],
node: CdpFrameTreeNode,
target: CdpTarget,
): void {
const existing = frames.get(node.frame.id);
if (!existing) order.push(node.frame.id);
const parentFrameId = node.frame.parentId || existing?.parentFrameId;
const url = node.frame.url || existing?.url;
frames.set(node.frame.id, {
frameId: node.frame.id,
...(parentFrameId ? { parentFrameId } : {}),
...(url ? { url } : {}),
target,
});
}
function walkFrameTree(
source: CdpFrameTreeSource,
targetByRootFrameId: ReadonlyMap<string, CdpTarget>,
frames: Map<string, CdpFrame>,
order: string[],
): void {
const stack: Array<{ node: CdpFrameTreeNode; inheritedTarget: CdpTarget }> = [
{ node: source.tree, inheritedTarget: source.target },
];
const expanded = new WeakSet<CdpFrameTreeNode>();
while (stack.length > 0) {
const current = stack.pop();
if (!current || expanded.has(current.node)) continue;
expanded.add(current.node);
const target = targetByRootFrameId.get(current.node.frame.id) ?? current.inheritedTarget;
mergeFrame(frames, order, current.node, target);
const children = current.node.childFrames ?? [];
for (let index = children.length - 1; index >= 0; index -= 1) {
stack.push({ node: children[index], inheritedTarget: target });
}
}
}
export function buildFrameGraph(sources: CdpFrameTreeSource[]): CdpFrameGraph | null {
const rootSource = sources.find((source) => source.target.sessionId === undefined) ?? sources[0];
if (!rootSource) return null;
const frames = new Map<string, CdpFrame>();
const order: string[] = [];
const targetByRootFrameId = new Map(
sources.map((source) => [source.tree.frame.id, source.target] as const),
);
// The top-level source is authoritative if malformed input reports the same
// frame as both a root target and a child target.
targetByRootFrameId.set(rootSource.tree.frame.id, rootSource.target);
walkFrameTree(rootSource, targetByRootFrameId, frames, order);
for (const source of sources) {
if (source === rootSource) continue;
walkFrameTree(source, targetByRootFrameId, frames, order);
}
return {
rootFrameId: rootSource.tree.frame.id,
frames: order.flatMap((frameId) => {
const frame = frames.get(frameId);
return frame ? [frame] : [];
}),
};
}
@@ -8,7 +8,6 @@ import {
useRef, useRef,
useState, useState,
} from "react"; } from "react";
import type { HelpHighlightRect } from "@/lib/help-bridge";
import logoUrl from "../../assets/logo.png"; import logoUrl from "../../assets/logo.png";
export interface HelpRequestData { export interface HelpRequestData {
@@ -20,10 +19,6 @@ export interface HelpRequestData {
displayMode?: "full" | "compact"; displayMode?: "full" | "compact";
/** CSS selectors to scroll to + flash-highlight. */ /** CSS selectors to scroll to + flash-highlight. */
selectors: string[]; selectors: string[];
/** Pre-resolved top-viewport rectangles, used for cross-frame targets. */
rects?: HelpHighlightRect[];
/** Re-resolve cross-frame rectangles after scroll, resize, or layout changes. */
refreshRects?: () => Promise<HelpHighlightRect[] | undefined>;
onContinue: (note: string) => void; onContinue: (note: string) => void;
onCancel: () => void; onCancel: () => void;
} }
@@ -54,7 +49,6 @@ const PANEL_WIDTH = 420;
const FALLBACK_PANEL_H = 180; const FALLBACK_PANEL_H = 180;
const FALLBACK_PANEL_H_COLLAPSED = 44; const FALLBACK_PANEL_H_COLLAPSED = 44;
const EMPTY_SELECTORS: string[] = []; const EMPTY_SELECTORS: string[] = [];
const EMPTY_RECTS: HelpHighlightRect[] = [];
function clamp(value: number, min: number, max: number): number { function clamp(value: number, min: number, max: number): number {
return Math.max(min, Math.min(max, value)); return Math.max(min, Math.min(max, value));
@@ -185,8 +179,6 @@ export function HelpRequestOverlay({ request }: Props) {
const { t } = useTranslation("extension"); const { t } = useTranslation("extension");
const isCompact = request?.displayMode === "compact"; const isCompact = request?.displayMode === "compact";
const effectiveSelectors = isCompact ? EMPTY_SELECTORS : (request?.selectors ?? EMPTY_SELECTORS); const effectiveSelectors = isCompact ? EMPTY_SELECTORS : (request?.selectors ?? EMPTY_SELECTORS);
const [liveRects, setLiveRects] = useState<HelpHighlightRect[] | null>(null);
const effectiveRects = isCompact ? EMPTY_RECTS : (liveRects ?? request?.rects ?? EMPTY_RECTS);
const [note, setNote] = useState(""); const [note, setNote] = useState("");
const [collapsed, setCollapsed] = useState(false); const [collapsed, setCollapsed] = useState(false);
const [boxes, setBoxes] = useState<Box[]>([]); const [boxes, setBoxes] = useState<Box[]>([]);
@@ -209,7 +201,6 @@ export function HelpRequestOverlay({ request }: Props) {
// Reset the note and collapse state whenever a new request appears. // Reset the note and collapse state whenever a new request appears.
useEffect(() => { useEffect(() => {
setNote(""); setNote("");
setLiveRects(null);
setCollapsed(false); setCollapsed(false);
placementRef.current = null; placementRef.current = null;
setDragPos(null); setDragPos(null);
@@ -218,32 +209,6 @@ export function HelpRequestOverlay({ request }: Props) {
dragStartRef.current = null; dragStartRef.current = null;
}, [request?.id]); }, [request?.id]);
useEffect(() => {
if (!request || isCompact || !request.refreshRects) return;
let disposed = false;
let resolving = false;
const refresh = async () => {
if (resolving) return;
resolving = true;
try {
const rects = await request.refreshRects?.();
if (!disposed && rects) setLiveRects(rects);
} finally {
resolving = false;
}
};
const onViewportChange = () => void refresh();
window.addEventListener("scroll", onViewportChange, true);
window.addEventListener("resize", onViewportChange);
const interval = window.setInterval(onViewportChange, 500);
return () => {
disposed = true;
window.removeEventListener("scroll", onViewportChange, true);
window.removeEventListener("resize", onViewportChange);
window.clearInterval(interval);
};
}, [request, isCompact]);
// Scroll the first matched target into view once per distinct request id. // Scroll the first matched target into view once per distinct request id.
useLayoutEffect(() => { useLayoutEffect(() => {
if (!request) return; if (!request) return;
@@ -266,7 +231,7 @@ export function HelpRequestOverlay({ request }: Props) {
return; return;
} }
const update = () => { const update = () => {
const nextBoxes = [...measure(effectiveSelectors), ...effectiveRects]; const nextBoxes = measure(effectiveSelectors);
setBoxes((prev) => (boxesEqual(prev, nextBoxes) ? prev : nextBoxes)); setBoxes((prev) => (boxesEqual(prev, nextBoxes) ? prev : nextBoxes));
}; };
update(); update();
@@ -278,7 +243,7 @@ export function HelpRequestOverlay({ request }: Props) {
window.removeEventListener("resize", update); window.removeEventListener("resize", update);
window.clearInterval(interval); window.clearInterval(interval);
}; };
}, [request, effectiveSelectors, effectiveRects]); }, [request, effectiveSelectors]);
const onHeaderPointerDown = useCallback( const onHeaderPointerDown = useCallback(
(e: ReactPointerEvent<HTMLDivElement>) => { (e: ReactPointerEvent<HTMLDivElement>) => {
@@ -340,7 +305,7 @@ export function HelpRequestOverlay({ request }: Props) {
return; return;
} }
if (userMovedRef.current) return; if (userMovedRef.current) return;
const measured = boxes.length > 0 ? boxes : [...measure(effectiveSelectors), ...effectiveRects]; const measured = boxes.length > 0 ? boxes : measure(effectiveSelectors);
const anchor = unionBox(measured); const anchor = unionBox(measured);
const banner = bannerRef.current; const banner = bannerRef.current;
if (!anchor || !banner) { if (!anchor || !banner) {
@@ -364,7 +329,7 @@ export function HelpRequestOverlay({ request }: Props) {
} }
const nextPos = placePanel(anchor, panelW, panelH, placementRef.current.placement); const nextPos = placePanel(anchor, panelW, panelH, placementRef.current.placement);
setPanelPos((prev) => (panelPosEqual(prev, nextPos) ? prev : nextPos)); setPanelPos((prev) => (panelPosEqual(prev, nextPos) ? prev : nextPos));
}, [request, boxes, collapsed, isCompact, effectiveSelectors, effectiveRects]); }, [request, boxes, collapsed, isCompact, effectiveSelectors]);
if (!request) return null; if (!request) return null;
@@ -1,6 +1,6 @@
import { i18n } from "@browser-skill/i18n"; import { i18n } from "@browser-skill/i18n";
import { I18nextProvider } from "@browser-skill/i18n/react"; import { I18nextProvider } from "@browser-skill/i18n/react";
import { cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react"; import { cleanup, fireEvent, render, screen } from "@testing-library/react";
import { createElement } from "react"; import { createElement } from "react";
import { afterEach, describe, expect, it, vi } from "vitest"; import { afterEach, describe, expect, it, vi } from "vitest";
import { type HelpRequestData, HelpRequestOverlay } from "../HelpRequestOverlay"; import { type HelpRequestData, HelpRequestOverlay } from "../HelpRequestOverlay";
@@ -38,36 +38,6 @@ describe("HelpRequestOverlay", () => {
expect(screen.getByText("Please complete the captcha")).toBeTruthy(); expect(screen.getByText("Please complete the captcha")).toBeTruthy();
}); });
it("renders explicit viewport rectangles for cross-frame targets", () => {
const { container } = renderOverlay(
baseRequest({ rects: [{ top: 30, left: 40, width: 120, height: 50 }] }),
);
const highlight = container.querySelector<HTMLElement>("[data-slot='help-highlight']");
expect(highlight?.style.top).toBe("30px");
expect(highlight?.style.left).toBe("40px");
expect(highlight?.style.width).toBe("120px");
expect(highlight?.style.height).toBe("50px");
});
it("re-resolves cross-frame rectangles after the viewport changes", async () => {
const refreshRects = vi.fn(async () => [{ top: 80, left: 90, width: 140, height: 60 }]);
const { container } = renderOverlay(
baseRequest({
rects: [{ top: 30, left: 40, width: 120, height: 50 }],
refreshRects,
}),
);
window.dispatchEvent(new Event("scroll"));
await waitFor(() => {
const highlight = container.querySelector<HTMLElement>("[data-slot='help-highlight']");
expect(highlight?.style.top).toBe("80px");
expect(highlight?.style.left).toBe("90px");
});
expect(refreshRects).toHaveBeenCalledTimes(1);
});
it("keeps the inactive render stable", () => { it("keeps the inactive render stable", () => {
const { container, rerender } = renderOverlay(null); const { container, rerender } = renderOverlay(null);
@@ -1,64 +0,0 @@
import { describe, expect, it, vi } from "vitest";
import { createHelpRequestData } from "../help-request";
describe("createHelpRequestData", () => {
it("gives live delivery and recovery the same rectangle refresh mechanism", async () => {
const finish = vi.fn();
const query = vi.fn(async () => ({
active: true,
request: {
requestId: "help-1",
prompt: "complete the challenge",
selectors: [],
rects: [{ top: 80, left: 90, width: 140, height: 60 }],
timeoutMs: 1_000,
},
}));
const request = createHelpRequestData(
{
requestId: "help-1",
prompt: "complete the challenge",
selectors: [],
rects: [{ top: 10, left: 20, width: 30, height: 40 }],
timeoutMs: 1_000,
},
{ finish, query },
);
await expect(request.refreshRects?.()).resolves.toEqual([
{ top: 80, left: 90, width: 140, height: 60 },
]);
request.onContinue(" done ");
request.onCancel();
expect(query).toHaveBeenCalledOnce();
expect(finish).toHaveBeenNthCalledWith(1, "help-1", "continued", " done ");
expect(finish).toHaveBeenNthCalledWith(2, "help-1", "cancelled");
});
it("ignores rectangle replies for a replaced request", async () => {
const request = createHelpRequestData(
{
requestId: "help-1",
prompt: "complete the challenge",
selectors: [],
timeoutMs: 1_000,
},
{
finish: vi.fn(),
query: vi.fn(async () => ({
active: true,
request: {
requestId: "help-2",
prompt: "new challenge",
selectors: [],
rects: [{ top: 1, left: 2, width: 3, height: 4 }],
timeoutMs: 1_000,
},
})),
},
);
await expect(request.refreshRects?.()).resolves.toBeUndefined();
});
});
@@ -1,12 +1,10 @@
import { beforeEach, describe, expect, it, vi } from "vitest"; import { beforeEach, describe, expect, it, vi } from "vitest";
import type { RecordStepPayload } from "@/lib/record-bridge"; import { RECORD_START, RECORD_STOP, type RecordStepPayload } from "@/lib/record-bridge";
import { startRecordCapture } from "../record-capture"; import { handleRecordContentMessage, startRecordCapture } from "../record-capture";
vi.stubGlobal("chrome", { vi.stubGlobal("chrome", {
runtime: { runtime: {
sendMessage: vi.fn((message: { sequence?: number }) => sendMessage: vi.fn(() => Promise.resolve()),
Promise.resolve({ ok: true, sequence: message.sequence }),
),
}, },
}); });
@@ -48,6 +46,117 @@ function click(el: Element): void {
el.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 })); el.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 }));
} }
describe("handleRecordContentMessage stop/cancel", () => {
it("ignores STOP when no recording is active", () => {
const dispose = vi.fn();
const onStop = vi.fn();
const setActiveRequestId = vi.fn();
const setCapture = vi.fn();
const sendResponse = vi.fn();
const needsAsync = handleRecordContentMessage(
{ type: RECORD_STOP, requestId: "rec-stale" },
{
activeRequestId: null,
capture: { dispose },
setActiveRequestId,
setCapture,
onStart: vi.fn(),
onStop,
},
sendResponse,
);
expect(needsAsync).toBe(false);
expect(dispose).not.toHaveBeenCalled();
expect(onStop).not.toHaveBeenCalled();
expect(setActiveRequestId).not.toHaveBeenCalled();
expect(setCapture).not.toHaveBeenCalled();
expect(sendResponse).not.toHaveBeenCalled();
});
it("ignores STOP for a mismatched requestId", () => {
const dispose = vi.fn();
const onStop = vi.fn();
const needsAsync = handleRecordContentMessage(
{ type: RECORD_STOP, requestId: "rec-other" },
{
activeRequestId: "rec-1",
capture: { dispose },
setActiveRequestId: vi.fn(),
setCapture: vi.fn(),
onStart: vi.fn(),
onStop,
},
vi.fn(),
);
expect(needsAsync).toBe(false);
expect(dispose).not.toHaveBeenCalled();
expect(onStop).not.toHaveBeenCalled();
});
it("keeps a failed STOP retryable and redelivers its recorded step", async () => {
document.body.innerHTML = `<label for="retry">Draft</label><input id="retry" />`;
const sendMessage = vi.mocked(chrome.runtime.sendMessage);
sendMessage.mockReset();
sendMessage.mockRejectedValueOnce(new Error("service worker unavailable"));
sendMessage.mockResolvedValueOnce(undefined);
let activeRequestId: string | null = null;
let capture: ReturnType<typeof startRecordCapture> | null = null;
const onStop = vi.fn();
const dispatch = (
message:
| { type: typeof RECORD_START; requestId: string; startedAtMs?: number }
| { type: typeof RECORD_STOP; requestId: string },
sendResponse?: (response: unknown) => void,
) =>
handleRecordContentMessage(
message,
{
activeRequestId,
capture,
setActiveRequestId: (id) => {
activeRequestId = id;
},
setCapture: (next) => {
capture = next;
},
onStart: vi.fn(),
onStop,
},
sendResponse,
);
dispatch({ type: RECORD_START, requestId: "rec-retry" });
const input = document.querySelector("input")!;
input.dispatchEvent(new FocusEvent("focusin", { bubbles: true }));
input.value = "dirty final value";
input.dispatchEvent(new Event("input", { bubbles: true }));
const firstResponse = vi.fn();
expect(dispatch({ type: RECORD_STOP, requestId: "rec-retry" }, firstResponse)).toBe(true);
await vi.waitFor(() =>
expect(firstResponse).toHaveBeenCalledWith({
ok: false,
error: "failed to deliver one or more recorded steps",
}),
);
expect(activeRequestId).toBe("rec-retry");
expect(onStop).not.toHaveBeenCalled();
const retryResponse = vi.fn();
expect(dispatch({ type: RECORD_STOP, requestId: "rec-retry" }, retryResponse)).toBe(true);
await vi.waitFor(() => expect(retryResponse).toHaveBeenCalledWith({ ok: true }));
expect(sendMessage).toHaveBeenCalledTimes(2);
expect(activeRequestId).toBeNull();
expect(onStop).toHaveBeenCalledTimes(1);
});
});
describe("record-capture semantic", () => { describe("record-capture semantic", () => {
let steps: RecordStepPayload[]; let steps: RecordStepPayload[];
@@ -124,19 +233,6 @@ describe("record-capture semantic", () => {
]); ]);
}); });
it("does not emit page navigation from a child Document capture", () => {
const originalUrl = location.href;
const capture = startRecordCapture("rec-child", (step) => steps.push(step), {
captureNavigation: false,
});
history.pushState({}, "", "#inside-frame");
expect(steps).toEqual([]);
capture.dispose();
history.replaceState({}, "", originalUrl);
});
it("does not record clicks on anonymous layout divs", () => { it("does not record clicks on anonymous layout divs", () => {
document.body.innerHTML = ` document.body.innerHTML = `
<div id="chrome">page chrome</div> <div id="chrome">page chrome</div>
@@ -1,102 +0,0 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import type { RecordFramePortMessage } from "@/lib/recording/frame-bridge";
import { RECORD_FRAME_START } from "@/lib/recording/frame-bridge";
import { RECORD_DOCUMENT_ATTRIBUTE } from "@/shared/recording-document-identity";
import { RecordFrameAgent } from "../recording/frame-agent";
class PortListeners<T extends (...args: never[]) => unknown> {
readonly values = new Set<T>();
addListener = (listener: T) => this.values.add(listener);
removeListener = (listener: T) => this.values.delete(listener);
}
function portHarness() {
const onMessage = new PortListeners<(message: unknown) => void>();
const onDisconnect = new PortListeners<() => void>();
const outbound: RecordFramePortMessage[] = [];
const port = {
name: "bsk-record-frame",
onMessage,
onDisconnect,
postMessage(message: RecordFramePortMessage) {
outbound.push(message);
if (message.type === "ready") {
queueMicrotask(() => {
for (const listener of onMessage.values) {
listener({
type: "ready_ack",
requestId: message.requestId,
producerId: message.producerId,
});
}
});
}
},
disconnect: vi.fn(),
} as unknown as chrome.runtime.Port;
return {
port,
outbound,
receive(message: RecordFramePortMessage) {
for (const listener of onMessage.values) listener(message);
},
};
}
describe("RecordFrameAgent", () => {
afterEach(() => {
document.documentElement.removeAttribute(RECORD_DOCUMENT_ATTRIBUTE);
document.body.replaceChildren();
});
it("keeps a failed stop retryable and flushes the final dirty fill", async () => {
const harness = portHarness();
const sendMessage = vi
.fn<(message: { sequence: number }) => Promise<unknown>>()
.mockRejectedValueOnce(new Error("offline"))
.mockRejectedValueOnce(new Error("still offline"))
.mockImplementation(async (message) => ({ ok: true, sequence: message.sequence }));
vi.stubGlobal("chrome", {
runtime: {
connect: vi.fn(() => harness.port),
sendMessage,
},
});
document.body.innerHTML = `<label for="draft">Draft</label><input id="draft" />`;
const agent = new RecordFrameAgent();
await expect(
agent.start({ type: RECORD_FRAME_START, requestId: "rec-1", startedAtMs: 10 }),
).resolves.toEqual({ ok: true });
expect(document.documentElement.hasAttribute(RECORD_DOCUMENT_ATTRIBUTE)).toBe(true);
const input = document.querySelector("input")!;
input.dispatchEvent(new FocusEvent("focusin", { bubbles: true }));
input.value = "final value";
input.dispatchEvent(new Event("input", { bubbles: true }));
harness.receive({ type: "stop", requestId: "rec-1", commandId: "stop-1" });
await vi.waitFor(() =>
expect(harness.outbound).toContainEqual({
type: "stopped",
requestId: "rec-1",
commandId: "stop-1",
ok: false,
error: "failed to deliver one or more recorded steps",
}),
);
expect(document.documentElement.hasAttribute(RECORD_DOCUMENT_ATTRIBUTE)).toBe(true);
harness.receive({ type: "stop", requestId: "rec-1", commandId: "stop-2" });
await vi.waitFor(() =>
expect(harness.outbound).toContainEqual({
type: "stopped",
requestId: "rec-1",
commandId: "stop-2",
ok: true,
}),
);
expect(sendMessage).toHaveBeenCalledTimes(3);
expect(document.documentElement.hasAttribute(RECORD_DOCUMENT_ATTRIBUTE)).toBe(false);
});
});
@@ -1,36 +0,0 @@
import { describe, expect, it, vi } from "vitest";
import type { RecordStepMessage } from "@/lib/record-bridge";
import { RecordStepDelivery } from "../record-step-delivery";
describe("RecordStepDelivery", () => {
it("retries from the first unacknowledged sequence before sending later steps", async () => {
const sent: RecordStepMessage[] = [];
const send = vi.fn(async (message: RecordStepMessage) => {
sent.push(message);
if (sent.length === 1) throw new Error("service worker unavailable");
return { ok: true, sequence: message.sequence };
});
const delivery = new RecordStepDelivery("rec-ordered", send, "document-1");
delivery.enqueue({ op: "click", target: { tag: "button", name: "First" } });
delivery.enqueue({ op: "click", target: { tag: "button", name: "Second" } });
await expect(delivery.flush()).resolves.toBe(true);
expect(sent.map((message) => message.sequence)).toEqual([1, 1, 2]);
expect(sent.map((message) => message.step.target?.name)).toEqual(["First", "First", "Second"]);
});
it("keeps unacknowledged steps pending for a later flush", async () => {
const send = vi
.fn<(message: RecordStepMessage) => Promise<unknown>>()
.mockRejectedValueOnce(new Error("offline"))
.mockRejectedValueOnce(new Error("still offline"))
.mockImplementation(async (message) => ({ ok: true, sequence: message.sequence }));
const delivery = new RecordStepDelivery("rec-retry", send, "document-1");
delivery.enqueue({ op: "click", target: { tag: "button", name: "Save" } });
await expect(delivery.flush()).resolves.toBe(false);
await expect(delivery.flush()).resolves.toBe(true);
expect(send).toHaveBeenCalledTimes(3);
});
});
@@ -1,31 +0,0 @@
import type { HelpQueryResponse, HelpRequestMessage } from "@/lib/help-bridge";
import type { HelpRequestData } from "./HelpRequestOverlay";
export interface HelpRequestDataDeps {
finish(requestId: string, outcome: "continued" | "cancelled", note?: string): void;
query(): Promise<HelpQueryResponse | undefined>;
}
/** Build the content-side representation shared by live delivery and recovery. */
export function createHelpRequestData(
message: Omit<HelpRequestMessage, "type">,
deps: HelpRequestDataDeps,
): HelpRequestData {
return {
id: message.requestId,
prompt: message.prompt,
...(message.title ? { title: message.title } : {}),
...(message.displayMode ? { displayMode: message.displayMode } : {}),
selectors: message.selectors,
rects: message.rects,
refreshRects: async () => {
const response = await deps.query();
return response?.active && response.request?.requestId === message.requestId
? response.request.rects
: undefined;
},
onContinue: (note: string) =>
deps.finish(message.requestId, "continued", note.trim() ? note : undefined),
onCancel: () => deps.finish(message.requestId, "cancelled"),
};
}
+174 -31
View File
@@ -12,8 +12,22 @@ import {
hasDirectHoverInteractiveSignal, hasDirectHoverInteractiveSignal,
hasStrongHoverExpansionSignal, hasStrongHoverExpansionSignal,
} from "@/lib/hover-trigger-policy"; } from "@/lib/hover-trigger-policy";
import type { RecordStepPayload } from "@/lib/record-bridge"; import {
import { shouldRecordPress } from "@/lib/recording/draft-policy"; isRecordCancelMessage,
isRecordStartMessage,
isRecordStopMessage,
RECORD_CANCEL,
RECORD_START,
RECORD_STEP,
RECORD_STOP,
type RecordCancelMessage,
type RecordStartAck,
type RecordStartMessage,
type RecordStepPayload,
type RecordStopAck,
type RecordStopMessage,
} from "@/lib/record-bridge";
import { shouldRecordPress } from "@/lib/trace-reducer";
import { import {
closestHoverSurfaceCandidate, closestHoverSurfaceCandidate,
collectHoverSurfaceStates, collectHoverSurfaceStates,
@@ -23,6 +37,37 @@ import {
isLikelyHoverSurfaceOwner, isLikelyHoverSurfaceOwner,
} from "./record-hover-surface"; } from "./record-hover-surface";
const pendingStepSends = new Map<string, Set<Promise<boolean>>>();
const failedStepDeliveries = new Set<string>();
const failedStepPayloads = new Map<string, RecordStepPayload[]>();
const pendingStopFlushes = new Map<string, Promise<RecordStopAck>>();
const knownRecordRequests = new Set<string>();
function deliverRecordStep(requestId: string, step: RecordStepPayload): Promise<boolean> {
const payload = { type: RECORD_STEP, requestId, step };
return Promise.resolve(chrome.runtime.sendMessage(payload)).then(
() => true,
() => {
failedStepDeliveries.add(requestId);
const failed = failedStepPayloads.get(requestId) ?? [];
failed.push(step);
failedStepPayloads.set(requestId, failed);
return false;
},
);
}
function sendRecordStep(requestId: string, step: RecordStepPayload): void {
const pending = deliverRecordStep(requestId, step);
const sends = pendingStepSends.get(requestId) ?? new Set<Promise<boolean>>();
sends.add(pending);
pendingStepSends.set(requestId, sends);
void pending.then(() => {
sends.delete(pending);
if (sends.size === 0) pendingStepSends.delete(requestId);
});
}
export interface RecordCaptureController { export interface RecordCaptureController {
dispose(): void; dispose(): void;
} }
@@ -124,8 +169,12 @@ function nearbyFillableFromSearchChrome(target: Element): FillableElement | null
function captureGeometry(el: Element): RecordStepPayload["geometry"] { function captureGeometry(el: Element): RecordStepPayload["geometry"] {
const rect = el.getBoundingClientRect(); const rect = el.getBoundingClientRect();
const style = window.getComputedStyle(el);
return { return {
rect: { x: rect.x, y: rect.y, w: rect.width, h: rect.height }, rect: { x: rect.x, y: rect.y, w: rect.width, h: rect.height },
scrollX: window.scrollX,
scrollY: window.scrollY,
position: style.position,
tag: el.tagName.toLowerCase(), tag: el.tagName.toLowerCase(),
}; };
} }
@@ -359,7 +408,6 @@ function scheduleInputCompletionCommit(
export function startRecordCapture( export function startRecordCapture(
_requestId: string, _requestId: string,
sendStep: (step: RecordStepPayload) => void, sendStep: (step: RecordStepPayload) => void,
options: { captureNavigation?: boolean } = {},
): RecordCaptureController { ): RecordCaptureController {
const emitStep = (step: RecordStepPayload) => { const emitStep = (step: RecordStepPayload) => {
sendStep({ page_url: location.href, ...step }); sendStep({ page_url: location.href, ...step });
@@ -896,33 +944,26 @@ export function startRecordCapture(
document.addEventListener("change", onChange, true); document.addEventListener("change", onChange, true);
document.addEventListener("keydown", onKeyDown, true); document.addEventListener("keydown", onKeyDown, true);
const captureNavigation = options.captureNavigation ?? true; const urlObserver = new MutationObserver(() => emitNavigateIfChanged());
const urlObserver = captureNavigation urlObserver.observe(document, { subtree: true, childList: true });
? new MutationObserver(() => emitNavigateIfChanged())
: null;
urlObserver?.observe(document, { subtree: true, childList: true });
const onUrlEvent = () => emitNavigateIfChanged(); const onUrlEvent = () => emitNavigateIfChanged();
// Wrap: passing commitFillSession directly would forward the DOM event as // Wrap: passing commitFillSession directly would forward the DOM event as
// the `commit` argument and stamp it onto the recorded fill step. // the `commit` argument and stamp it onto the recorded fill step.
const onPageHide = () => commitFillSession(); const onPageHide = () => commitFillSession();
if (captureNavigation) { window.addEventListener("hashchange", onUrlEvent);
window.addEventListener("hashchange", onUrlEvent); window.addEventListener("popstate", onUrlEvent);
window.addEventListener("popstate", onUrlEvent);
}
window.addEventListener("pagehide", onPageHide); window.addEventListener("pagehide", onPageHide);
const originalPushState = history.pushState; const originalPushState = history.pushState;
const originalReplaceState = history.replaceState; const originalReplaceState = history.replaceState;
if (captureNavigation) { history.pushState = function (...args: Parameters<History["pushState"]>) {
history.pushState = function (...args: Parameters<History["pushState"]>) { originalPushState.apply(this, args);
originalPushState.apply(this, args); emitNavigateIfChanged(navigationActionPending ? true : undefined);
emitNavigateIfChanged(navigationActionPending ? true : undefined); };
}; history.replaceState = function (...args: Parameters<History["replaceState"]>) {
history.replaceState = function (...args: Parameters<History["replaceState"]>) { originalReplaceState.apply(this, args);
originalReplaceState.apply(this, args); emitNavigateIfChanged(navigationActionPending ? true : undefined);
emitNavigateIfChanged(navigationActionPending ? true : undefined); };
};
}
return { return {
dispose() { dispose() {
@@ -936,16 +977,118 @@ export function startRecordCapture(
document.removeEventListener("compositionend", onCompositionEnd, true); document.removeEventListener("compositionend", onCompositionEnd, true);
document.removeEventListener("change", onChange, true); document.removeEventListener("change", onChange, true);
document.removeEventListener("keydown", onKeyDown, true); document.removeEventListener("keydown", onKeyDown, true);
urlObserver?.disconnect(); urlObserver.disconnect();
if (captureNavigation) { window.removeEventListener("hashchange", onUrlEvent);
window.removeEventListener("hashchange", onUrlEvent); window.removeEventListener("popstate", onUrlEvent);
window.removeEventListener("popstate", onUrlEvent);
}
window.removeEventListener("pagehide", onPageHide); window.removeEventListener("pagehide", onPageHide);
if (captureNavigation) { history.pushState = originalPushState;
history.pushState = originalPushState; history.replaceState = originalReplaceState;
history.replaceState = originalReplaceState;
}
}, },
}; };
} }
export type RecordContentMessage = RecordStartMessage | RecordStopMessage | RecordCancelMessage;
export function isRecordContentMessage(msg: unknown): msg is RecordContentMessage {
return isRecordStartMessage(msg) || isRecordStopMessage(msg) || isRecordCancelMessage(msg);
}
export function handleRecordContentMessage(
message: RecordContentMessage,
state: {
activeRequestId: string | null;
capture: RecordCaptureController | null;
setActiveRequestId(id: string | null): void;
setCapture(capture: RecordCaptureController | null): void;
onStart(requestId: string, startedAtMs?: number): void;
onStop(): void;
},
sendResponse?: (response: RecordStartAck | RecordStopAck) => void,
): boolean {
if (isRecordStartMessage(message)) {
if (!knownRecordRequests.has(message.requestId)) {
knownRecordRequests.add(message.requestId);
failedStepDeliveries.delete(message.requestId);
failedStepPayloads.delete(message.requestId);
}
state.capture?.dispose();
state.setCapture(
startRecordCapture(message.requestId, (step) => {
sendRecordStep(message.requestId, step);
}),
);
state.setActiveRequestId(message.requestId);
state.onStart(message.requestId, message.startedAtMs);
sendResponse?.({ ok: true });
return sendResponse !== undefined;
}
if (isRecordStopMessage(message) || isRecordCancelMessage(message)) {
// Require an active recording that matches this requestId — otherwise a
// stray STOP/CANCEL (e.g. after teardown) would still run finishStop and
// clear overlay state even though nothing was capturing.
if (!state.activeRequestId || state.activeRequestId !== message.requestId) {
return false;
}
state.capture?.dispose();
state.setCapture(null);
const finishStop = () => {
state.onStop();
state.setActiveRequestId(null);
};
if (isRecordStopMessage(message) && sendResponse) {
const existingFlush = pendingStopFlushes.get(message.requestId);
if (existingFlush) {
void existingFlush.then(sendResponse);
return true;
}
const failedBeforeStop = [...(failedStepPayloads.get(message.requestId) ?? [])];
if (failedBeforeStop.length > 0) {
failedStepPayloads.delete(message.requestId);
failedStepDeliveries.delete(message.requestId);
}
const pending = [...(pendingStepSends.get(message.requestId) ?? [])];
const flush = Promise.all(pending).then(async (delivered): Promise<RecordStopAck> => {
const retried = await Promise.all(
failedBeforeStop.map((step) => deliverRecordStep(message.requestId, step)),
);
const succeeded =
delivered.every(Boolean) &&
retried.every(Boolean) &&
!failedStepDeliveries.has(message.requestId);
if (succeeded) {
failedStepDeliveries.delete(message.requestId);
failedStepPayloads.delete(message.requestId);
knownRecordRequests.delete(message.requestId);
finishStop();
}
return succeeded
? { ok: true }
: {
ok: false,
error: "failed to deliver one or more recorded steps",
};
});
pendingStopFlushes.set(message.requestId, flush);
void flush.then(sendResponse).finally(() => {
if (pendingStopFlushes.get(message.requestId) === flush) {
pendingStopFlushes.delete(message.requestId);
}
});
return true;
}
if (isRecordCancelMessage(message)) {
failedStepDeliveries.delete(message.requestId);
failedStepPayloads.delete(message.requestId);
pendingStopFlushes.delete(message.requestId);
knownRecordRequests.delete(message.requestId);
}
finishStop();
return false;
}
return false;
}
export { RECORD_CANCEL, RECORD_START, RECORD_STEP, RECORD_STOP };
@@ -1,65 +0,0 @@
import {
isAcceptedRecordStepAck,
RECORD_STEP,
type RecordStepMessage,
type RecordStepPayload,
} from "@/lib/record-bridge";
export type SendRecordStepMessage = (message: RecordStepMessage) => Promise<unknown>;
export class RecordStepDelivery {
readonly #requestId: string;
readonly #producerId: string;
readonly #send: SendRecordStepMessage;
readonly #pending = new Map<number, RecordStepPayload>();
#nextSequence = 1;
#tail = Promise.resolve();
constructor(
requestId: string,
send: SendRecordStepMessage = (message) => chrome.runtime.sendMessage(message),
producerId: string = crypto.randomUUID(),
) {
this.#requestId = requestId;
this.#producerId = producerId;
this.#send = send;
}
enqueue(step: RecordStepPayload): void {
const sequence = this.#nextSequence;
this.#nextSequence += 1;
this.#pending.set(sequence, step);
void this.#schedule();
}
async flush(): Promise<boolean> {
await this.#tail;
await this.#schedule();
return this.#pending.size === 0;
}
#schedule(): Promise<void> {
const drain = () => this.#drain();
this.#tail = this.#tail.then(drain, drain);
return this.#tail;
}
async #drain(): Promise<void> {
for (const [sequence, step] of this.#pending) {
const message: RecordStepMessage = {
type: RECORD_STEP,
requestId: this.#requestId,
producerId: this.#producerId,
sequence,
step,
};
try {
const response = await this.#send(message);
if (!isAcceptedRecordStepAck(response, sequence)) return;
this.#pending.delete(sequence);
} catch {
return;
}
}
}
}
@@ -1,209 +0,0 @@
import {
markRecordingDocument,
type RecordingDocumentMarker,
waitForDocumentElement,
} from "@/lib/recording/document-marker";
import {
isRecordFrameStartMessage,
RECORD_FRAME_PORT,
RECORD_FRAME_QUERY,
RECORD_FRAME_START,
type RecordFramePortMessage,
type RecordFrameQueryResponse,
type RecordFrameStartMessage,
} from "@/lib/recording/frame-bridge";
import { type RecordCaptureController, startRecordCapture } from "../record-capture";
import { RecordStepDelivery } from "../record-step-delivery";
interface ActiveFrameRecording {
requestId: string;
producerId: string;
port: chrome.runtime.Port;
delivery: RecordStepDelivery;
capture: RecordCaptureController;
marker: RecordingDocumentMarker;
}
const RECORD_FRAME_REGISTER_TIMEOUT_MS = 5_000;
export class RecordFrameAgent {
#active: ActiveFrameRecording | null = null;
#startPromise: Promise<{ ok: true } | { ok: false; error: string }> | null = null;
#disposed = false;
async start(
message: RecordFrameStartMessage,
): Promise<{ ok: true } | { ok: false; error: string }> {
if (this.#active?.requestId === message.requestId) return { ok: true };
if (this.#disposed) return { ok: false, error: "recording document is disposed" };
if (this.#startPromise) {
await this.#startPromise;
return this.start(message);
}
if (this.#active) this.cancel(this.#active.requestId);
const starting = this.#start(message);
this.#startPromise = starting;
try {
return await starting;
} finally {
if (this.#startPromise === starting) this.#startPromise = null;
}
}
cancel(requestId: string): void {
const active = this.#active;
if (!active || active.requestId !== requestId) return;
active.capture.dispose();
active.marker.restore();
active.port.disconnect();
this.#active = null;
}
dispose(): void {
this.#disposed = true;
if (this.#active) this.cancel(this.#active.requestId);
}
async #start(
message: RecordFrameStartMessage,
): Promise<{ ok: true } | { ok: false; error: string }> {
const producerId = crypto.randomUUID();
const root = await waitForDocumentElement();
if (this.#disposed) return { ok: false, error: "recording document is disposed" };
const marker = markRecordingDocument(producerId, root);
const port = chrome.runtime.connect({ name: RECORD_FRAME_PORT });
try {
await registerPort(port, message.requestId, producerId);
} catch {
marker.restore();
port.disconnect();
return { ok: false, error: "failed to register recording document" };
}
if (this.#disposed) {
marker.restore();
port.disconnect();
return { ok: false, error: "recording document is disposed" };
}
const delivery = new RecordStepDelivery(message.requestId, undefined, producerId);
const capture = startRecordCapture(
message.requestId,
(step) => {
marker.ensure();
delivery.enqueue(step);
},
{ captureNavigation: window.top === window },
);
const active: ActiveFrameRecording = {
requestId: message.requestId,
producerId,
port,
delivery,
capture,
marker,
};
this.#active = active;
port.onMessage.addListener((raw: unknown) => {
const command = raw as Partial<RecordFramePortMessage>;
if (command.type === "stop" && command.requestId === active.requestId) {
void this.#stop(active, command.commandId);
} else if (command.type === "cancel" && command.requestId === active.requestId) {
this.cancel(active.requestId);
}
});
port.onDisconnect.addListener(() => {
if (this.#active !== active) return;
active.capture.dispose();
active.marker.restore();
this.#active = null;
});
return { ok: true };
}
async #stop(active: ActiveFrameRecording, commandId: string | undefined): Promise<void> {
if (!commandId || this.#active !== active) return;
active.capture.dispose();
const succeeded = await active.delivery.flush();
try {
active.port.postMessage({
type: "stopped",
requestId: active.requestId,
commandId,
ok: succeeded,
...(succeeded ? {} : { error: "failed to deliver one or more recorded steps" }),
} satisfies RecordFramePortMessage);
} catch {
return;
}
if (!succeeded || this.#active !== active) return;
active.marker.restore();
this.#active = null;
active.port.disconnect();
}
}
function registerPort(
port: chrome.runtime.Port,
requestId: string,
producerId: string,
): Promise<void> {
return new Promise((resolve, reject) => {
const timer = setTimeout(() => {
cleanup();
reject(new Error("recording document registration timed out"));
}, RECORD_FRAME_REGISTER_TIMEOUT_MS);
const cleanup = () => {
clearTimeout(timer);
port.onMessage.removeListener(onMessage);
port.onDisconnect.removeListener(onDisconnect);
};
const onMessage = (raw: unknown) => {
const message = raw as Partial<RecordFramePortMessage>;
if (
message.type !== "ready_ack" ||
message.requestId !== requestId ||
message.producerId !== producerId
) {
return;
}
cleanup();
resolve();
};
const onDisconnect = () => {
cleanup();
reject(new Error("recording document port disconnected"));
};
port.onMessage.addListener(onMessage);
port.onDisconnect.addListener(onDisconnect);
port.postMessage({ type: "ready", requestId, producerId } satisfies RecordFramePortMessage);
});
}
export function attachRecordFrameAgent(): () => void {
const agent = new RecordFrameAgent();
const onMessage = (
message: unknown,
_sender: chrome.runtime.MessageSender,
sendResponse: (response: { ok: true } | { ok: false; error: string }) => void,
) => {
if (!isRecordFrameStartMessage(message)) return false;
void agent.start(message).then(sendResponse);
return true;
};
chrome.runtime.onMessage.addListener(onMessage);
void chrome.runtime
.sendMessage({ type: RECORD_FRAME_QUERY })
.then((response: RecordFrameQueryResponse | undefined) => {
if (!response?.active || !response.requestId || response.startedAtMs === undefined) return;
return agent.start({
type: RECORD_FRAME_START,
requestId: response.requestId,
startedAtMs: response.startedAtMs,
});
})
.catch(() => {});
return () => {
chrome.runtime.onMessage.removeListener(onMessage);
agent.dispose();
};
}
+22 -25
View File
@@ -21,7 +21,6 @@ import {
type OverlayMode, type OverlayMode,
} from "@/lib/overlay-bridge"; } from "@/lib/overlay-bridge";
import { POPUP_PORT_NAME, type PopupInbound, type PopupOutbound } from "@/lib/popup-bridge"; import { POPUP_PORT_NAME, type PopupInbound, type PopupOutbound } from "@/lib/popup-bridge";
import { recordFrameCoordinator } from "@/lib/recording/frame-coordinator";
import { attachSessionsLiveFlag } from "@/lib/sessions-live-flag"; import { attachSessionsLiveFlag } from "@/lib/sessions-live-flag";
import { createDisconnectCleanup } from "@/session-manager/disconnect-cleanup"; import { createDisconnectCleanup } from "@/session-manager/disconnect-cleanup";
import { attachSessionEventHandler } from "@/session-manager/event-handler"; import { attachSessionEventHandler } from "@/session-manager/event-handler";
@@ -39,7 +38,6 @@ import {
attachRecordFinishListener, attachRecordFinishListener,
attachRecordQueryListener, attachRecordQueryListener,
attachRecordStepListener, attachRecordStepListener,
type RecordRuntimeDeps,
} from "@/tools/record"; } from "@/tools/record";
import { detectBrowserMeta } from "@/transport/handshake"; import { detectBrowserMeta } from "@/transport/handshake";
import type { Transport } from "@/transport/transport"; import type { Transport } from "@/transport/transport";
@@ -147,33 +145,10 @@ export default defineBackground(() => {
void sessionsLive.syncFromManager(); void sessionsLive.syncFromManager();
}, },
}); });
const recordDeps = {
tabsApi: chrome.tabs,
cdp,
frameCoordinator: recordFrameCoordinator,
sendToTab: (tabId: number, msg: Parameters<typeof chrome.tabs.sendMessage>[1]) =>
chrome.tabs.sendMessage(tabId, msg),
bypassOverlay: async (tabId: number, enabled: boolean) => {
try {
await chrome.tabs.sendMessage(tabId, {
type: OVERLAY_AUTOMATION_BYPASS,
enabled,
});
} catch {
// Content script may be unavailable on restricted pages.
}
},
} satisfies RecordRuntimeDeps;
recordFrameCoordinator.attach();
attachRecordStepListener(recordDeps);
attachRecordFinishListener(recordDeps);
attachRecordQueryListener(recordDeps);
const dispatcher = new ToolDispatcher({ const dispatcher = new ToolDispatcher({
transport, transport,
sessions, sessions,
cdp, cdp,
recording: recordDeps,
onSessionsChanged: onOverlaySessionStateChanged, onSessionsChanged: onOverlaySessionStateChanged,
onBrowserControlResumed, onBrowserControlResumed,
approveBorrow: (ctx) => approveBorrow: (ctx) =>
@@ -195,6 +170,28 @@ export default defineBackground(() => {
}), }),
}); });
dispatcher.start(); dispatcher.start();
const recordDeps = {
tabsApi: chrome.tabs,
cdp,
sendToTab: (tabId: number, msg: Parameters<typeof chrome.tabs.sendMessage>[1]) =>
chrome.tabs.sendMessage(tabId, msg),
bypassOverlay: async (tabId: number, enabled: boolean) => {
try {
await chrome.tabs.sendMessage(tabId, {
type: OVERLAY_AUTOMATION_BYPASS,
enabled,
});
} catch {
// Content script may be unavailable on restricted pages.
}
},
};
// Message listeners stay up (cheap; fire only for record message types).
// Tab / webNavigation observation attaches lazily while a recording is
// active — see ensureBrowserObservationListeners in tools/record.ts.
attachRecordStepListener(recordDeps);
attachRecordFinishListener(recordDeps);
attachRecordQueryListener(recordDeps);
if (typeof chrome.notifications?.onClicked?.addListener === "function") { if (typeof chrome.notifications?.onClicked?.addListener === "function") {
attachBorrowNotificationClickHandler({ attachBorrowNotificationClickHandler({
onClicked: chrome.notifications.onClicked, onClicked: chrome.notifications.onClicked,
+66 -48
View File
@@ -7,10 +7,14 @@ import { BorrowConfirmationOverlay } from "@/content/BorrowConfirmationOverlay";
import { ControlOverlay } from "@/content/ControlOverlay"; import { ControlOverlay } from "@/content/ControlOverlay";
import { createCaptureSuppressController } from "@/content/capture-suppress"; import { createCaptureSuppressController } from "@/content/capture-suppress";
import { HelpRequestOverlay } from "@/content/HelpRequestOverlay"; import { HelpRequestOverlay } from "@/content/HelpRequestOverlay";
import { createHelpRequestData } from "@/content/help-request";
import overlayCss from "@/content/overlay.css?inline"; import overlayCss from "@/content/overlay.css?inline";
import { OverlayController, shouldShowAgentControlOverlay } from "@/content/overlay-controller"; import { OverlayController, shouldShowAgentControlOverlay } from "@/content/overlay-controller";
import { RecordOverlay } from "@/content/RecordOverlay"; import { RecordOverlay } from "@/content/RecordOverlay";
import {
handleRecordContentMessage,
isRecordContentMessage,
type RecordCaptureController,
} from "@/content/record-capture";
import { import {
type CaptureSuppressAck, type CaptureSuppressAck,
type CaptureSuppressMessage, type CaptureSuppressMessage,
@@ -40,17 +44,11 @@ import {
} from "@/lib/overlay-bridge"; } from "@/lib/overlay-bridge";
import { sendInterrupt } from "@/lib/overlay-interrupt-client"; import { sendInterrupt } from "@/lib/overlay-interrupt-client";
import { import {
isRecordCancelMessage,
isRecordStartMessage,
isRecordStopMessage,
RECORD_FINISH, RECORD_FINISH,
RECORD_QUERY, RECORD_QUERY,
type RecordCancelMessage,
type RecordQueryResponse, type RecordQueryResponse,
type RecordStartAck, type RecordStartAck,
type RecordStartMessage,
type RecordStopAck, type RecordStopAck,
type RecordStopMessage,
} from "@/lib/record-bridge"; } from "@/lib/record-bridge";
import type { import type {
BorrowCancelMessage, BorrowCancelMessage,
@@ -70,6 +68,7 @@ export default defineContentScript({
if (window.top !== window) return; if (window.top !== window) return;
const overlays = new OverlayController(); const overlays = new OverlayController();
let recordCapture: RecordCaptureController | null = null;
let activeRecordRequestId: string | null = null; let activeRecordRequestId: string | null = null;
let reactRoot: ReactDOM.Root | null = null; let reactRoot: ReactDOM.Root | null = null;
let overlayHost: HTMLElement | null = null; let overlayHost: HTMLElement | null = null;
@@ -228,6 +227,8 @@ export default defineContentScript({
if (previousHelp) { if (previousHelp) {
void sendHelpFinish(previousHelp.id, "cancelled"); void sendHelpFinish(previousHelp.id, "cancelled");
} }
recordCapture?.dispose();
recordCapture = null;
activeRecordRequestId = null; activeRecordRequestId = null;
renderAll(); renderAll();
} }
@@ -257,9 +258,6 @@ export default defineContentScript({
| HelpRequestMessage | HelpRequestMessage
| HelpCancelMessage | HelpCancelMessage
| CaptureSuppressMessage | CaptureSuppressMessage
| RecordStartMessage
| RecordStopMessage
| RecordCancelMessage
| OverlayAgentOverlayResetMessage | OverlayAgentOverlayResetMessage
| OverlayAgentStateMessage | OverlayAgentStateMessage
| OverlayAutomationBypassMessage, | OverlayAutomationBypassMessage,
@@ -270,32 +268,41 @@ export default defineContentScript({
return captureSuppress.handleMessage(message, sendResponse); return captureSuppress.handleMessage(message, sendResponse);
} }
if (isRecordStartMessage(message)) { if (isRecordContentMessage(message)) {
activeRecordRequestId = message.requestId; const needsAsync = handleRecordContentMessage(
overlays.setAgentRecordRequest({ message,
id: message.requestId, {
...(typeof message.startedAtMs === "number" ? { startedAtMs: message.startedAtMs } : {}), activeRequestId: activeRecordRequestId,
onFinish: () => { capture: recordCapture,
void chrome.runtime.sendMessage({ setActiveRequestId: (id) => {
type: RECORD_FINISH, activeRecordRequestId = id;
requestId: message.requestId, },
}); setCapture: (capture) => {
recordCapture = capture;
},
onStart: (requestId, startedAtMs) => {
overlays.setAgentRecordRequest({
id: requestId,
...(typeof startedAtMs === "number" ? { startedAtMs } : {}),
onFinish: () => {
void chrome.runtime.sendMessage({
type: RECORD_FINISH,
requestId,
});
},
});
renderAll();
},
onStop: () => {
overlays.clearAgentRecordRequest(activeRecordRequestId ?? undefined);
renderAll();
},
}, },
}); sendResponse as unknown as
renderAll(); | ((response: RecordStartAck | RecordStopAck) => void)
(sendResponse as unknown as (response: RecordStartAck) => void)({ ok: true }); | undefined,
return false; );
} return needsAsync;
if (isRecordStopMessage(message) || isRecordCancelMessage(message)) {
if (activeRecordRequestId !== message.requestId) return false;
overlays.clearAgentRecordRequest(message.requestId);
activeRecordRequestId = null;
renderAll();
if (isRecordStopMessage(message)) {
(sendResponse as unknown as (response: RecordStopAck) => void)({ ok: true });
}
return false;
} }
if ( if (
@@ -337,7 +344,16 @@ export default defineContentScript({
if (isHelpRequestMessage(message)) { if (isHelpRequestMessage(message)) {
const helpMsg = message as HelpRequestMessage; const helpMsg = message as HelpRequestMessage;
const previousHelp = mountHelpRequest(helpMsg); const previousHelp = overlays.setAgentHelpRequest({
id: helpMsg.requestId,
prompt: helpMsg.prompt,
...(helpMsg.title ? { title: helpMsg.title } : {}),
...(helpMsg.displayMode ? { displayMode: helpMsg.displayMode } : {}),
selectors: helpMsg.selectors,
onContinue: (note: string) =>
void sendHelpFinish(helpMsg.requestId, "continued", note.trim() ? note : undefined),
onCancel: () => void sendHelpFinish(helpMsg.requestId, "cancelled"),
});
if (previousHelp && previousHelp.id !== helpMsg.requestId) { if (previousHelp && previousHelp.id !== helpMsg.requestId) {
void sendHelpFinish(previousHelp.id, "cancelled"); void sendHelpFinish(previousHelp.id, "cancelled");
} }
@@ -390,18 +406,17 @@ export default defineContentScript({
}); });
} }
function mountHelpRequest(helpMsg: Omit<HelpRequestMessage, "type">) { function mountHelpRequest(helpMsg: Omit<HelpRequestMessage, "type">): void {
return overlays.setAgentHelpRequest( overlays.setAgentHelpRequest({
createHelpRequestData(helpMsg, { id: helpMsg.requestId,
finish: (requestId, outcome, note) => { prompt: helpMsg.prompt,
void sendHelpFinish(requestId, outcome, note); ...(helpMsg.title ? { title: helpMsg.title } : {}),
}, ...(helpMsg.displayMode ? { displayMode: helpMsg.displayMode } : {}),
query: () => selectors: helpMsg.selectors,
chrome.runtime.sendMessage({ type: HELP_QUERY }) as Promise< onContinue: (note: string) =>
HelpQueryResponse | undefined void sendHelpFinish(helpMsg.requestId, "continued", note.trim() ? note : undefined),
>, onCancel: () => void sendHelpFinish(helpMsg.requestId, "cancelled"),
}), });
);
} }
async function queryActiveHelpWithRetry(): Promise<void> { async function queryActiveHelpWithRetry(): Promise<void> {
@@ -518,6 +533,9 @@ export default defineContentScript({
chrome.runtime.onMessage.removeListener(onMessage); chrome.runtime.onMessage.removeListener(onMessage);
chrome.storage.onChanged.removeListener(onStorageChange); chrome.storage.onChanged.removeListener(onStorageChange);
window.removeEventListener("pageshow", onPageShow); window.removeEventListener("pageshow", onPageShow);
// Restore history hooks / remove capture listeners before the CS unloads.
recordCapture?.dispose();
recordCapture = null;
activeRecordRequestId = null; activeRecordRequestId = null;
}); });
}, },
@@ -100,6 +100,24 @@ describe("App", () => {
expect(screen.queryByText("录制你的操作,供 Agent 参考")).toBeNull(); expect(screen.queryByText("录制你的操作,供 Agent 参考")).toBeNull();
}); });
it("renders localized copy for a version failure instead of protocol details", () => {
mockUseConnectionState.mockReturnValue({
snapshot: {
...baseSnapshot,
lastError:
"[handshake] daemon rejected handshake: version_too_old — peer protocol 1.0 is below local min_compatible_protocol 1.0",
},
statusState: "disconnected",
setLabel,
setConnectionEnabled,
});
render(<App />);
const error = screen.getByText(/bsk 与 BrowserSkill 扩展版本不兼容/);
expect(error.textContent).not.toContain("min_compatible_protocol");
});
it("shows single-line compact metadata and copies the instance id", async () => { it("shows single-line compact metadata and copies the instance id", async () => {
mockUseConnectionState.mockReturnValue({ mockUseConnectionState.mockReturnValue({
snapshot: { snapshot: {
@@ -202,7 +220,11 @@ describe("App", () => {
expect(copied).not.toMatch(/bsk record start[^\n]*--url/); expect(copied).not.toMatch(/bsk record start[^\n]*--url/);
expect(copied).toContain("./trace"); expect(copied).toContain("./trace");
expect(copied).toContain("trace.json"); expect(copied).toContain("trace.json");
expect(copied).toContain("states/"); expect(copied).toContain("pages/");
expect(copied).toContain("总结我做了什么");
expect(copied).not.toContain("不要重跑");
expect(copied).not.toContain("pages + steps");
expect(copied).not.toContain("@eN");
// Button label stays static; a transient toast confirms the copy. // Button label stays static; a transient toast confirms the copy.
expect(copyButton.textContent).toContain("复制录制指令"); expect(copyButton.textContent).toContain("复制录制指令");
await waitFor(() => expect(screen.getByRole("status").textContent).toContain("已复制")); await waitFor(() => expect(screen.getByRole("status").textContent).toContain("已复制"));
@@ -247,41 +269,6 @@ describe("App", () => {
expect(copyButton.getAttribute("disabled")).not.toBeNull(); expect(copyButton.getAttribute("disabled")).not.toBeNull();
expect(screen.getByText("连接后可用")).toBeTruthy(); expect(screen.getByText("连接后可用")).toBeTruthy();
}); });
it("treats protocol drift as still connected and prompts an upgrade", async () => {
mockUseConnectionState.mockReturnValue({
snapshot: {
...baseSnapshot,
state: "version_skew",
instanceId: "03c3e47f",
handshake: {
server: "bh",
version: mockDaemonVersion,
protocol_version: "1.0",
},
},
statusState: "version_skew",
setLabel,
setConnectionEnabled,
});
render(<App />);
expect(screen.getByText("已连接")).toBeTruthy();
expect(screen.getByText("可升级")).toBeTruthy();
expect(screen.queryByText("协议不一致")).toBeNull();
expect(screen.queryByText("Action needed")).toBeNull();
expect(screen.queryByText("兼容")).toBeNull();
const warning = screen.getByText(/协议版本不同,请及时升级/);
expect(warning.textContent).toContain("CLI 协议");
expect(warning.textContent).toContain("扩展协议");
openRecordView();
const copyButton = screen.getByRole("button", { name: "复制录制指令" });
expect(copyButton.getAttribute("disabled")).toBeNull();
fireEvent.click(copyButton);
expect(navigator.clipboard.writeText).toHaveBeenCalledTimes(1);
});
}); });
describe("control hints toggle", () => { describe("control hints toggle", () => {
+8 -6
View File
@@ -79,11 +79,13 @@ export function App() {
}, [copiedTick]); }, [copiedTick]);
const isSkewed = statusState === "version_skew"; const isSkewed = statusState === "version_skew";
const connectionLive = statusState === "connected" || isSkewed;
const daemonVersion = snapshot.handshake?.version ?? "—"; const daemonVersion = snapshot.handshake?.version ?? "—";
const daemonProtocol = snapshot.handshake?.protocol_version ?? "—"; const daemonProtocol = snapshot.handshake?.protocol_version ?? "—";
const extensionVersion = snapshot.extensionVersion || "—"; const extensionVersion = snapshot.extensionVersion || "—";
const instanceId = snapshot.instanceId || "—"; const instanceId = snapshot.instanceId || "—";
const errorMessage = snapshot.lastError?.includes("version_too_old")
? t("popup.versionIncompatibleError")
: snapshot.lastError;
const copyInstanceId = async () => { const copyInstanceId = async () => {
if (!snapshot.instanceId || !navigator.clipboard?.writeText) return; if (!snapshot.instanceId || !navigator.clipboard?.writeText) return;
@@ -91,7 +93,7 @@ export function App() {
setCopiedInstanceId(true); setCopiedInstanceId(true);
}; };
const recordReady = connectionLive && Boolean(snapshot.instanceId); const recordReady = statusState === "connected" && Boolean(snapshot.instanceId);
const recordPurpose = purposeDraft.trim(); const recordPurpose = purposeDraft.trim();
const recordStartUrl = startUrlDraft.trim(); const recordStartUrl = startUrlDraft.trim();
const recordCommand = snapshot.instanceId const recordCommand = snapshot.instanceId
@@ -207,12 +209,12 @@ export function App() {
</div> </div>
{isSkewed && ( {isSkewed && (
<p <p
className="mt-2 text-xs leading-snug text-muted-foreground" className="mt-2 text-xs leading-snug text-amber-600 dark:text-amber-400"
data-slot="popup-version-skew-warning" data-slot="popup-version-skew-warning"
> >
{t("popup.versionSkewWarning", { {t("popup.versionSkewWarning", {
extensionProtocol: PROTOCOL_VERSION, extensionProtocol: PROTOCOL_VERSION,
cliProtocol: daemonProtocol, daemonProtocol,
})} })}
</p> </p>
)} )}
@@ -253,12 +255,12 @@ export function App() {
</div> </div>
</section> </section>
{snapshot.lastError && ( {errorMessage && (
<div <div
className="rounded-lg border border-destructive/25 bg-destructive/10 px-3 py-2 text-xs leading-snug text-destructive" className="rounded-lg border border-destructive/25 bg-destructive/10 px-3 py-2 text-xs leading-snug text-destructive"
data-slot="popup-error" data-slot="popup-error"
> >
{snapshot.lastError} {errorMessage}
</div> </div>
)} )}
@@ -1,14 +0,0 @@
import { attachRecordFrameAgent } from "@/content/recording/frame-agent";
export default defineContentScript({
matches: ["<all_urls>"],
runAt: "document_start",
allFrames: true,
matchAboutBlank: true,
matchOriginAsFallback: true,
main(ctx) {
const dispose = attachRecordFrameAgent();
ctx.onInvalidated(dispose);
},
});
@@ -26,20 +26,20 @@ function handshake(
} }
describe("computeConnectedState (protocol-based compat)", () => { describe("computeConnectedState (protocol-based compat)", () => {
it("returns connected when daemon protocol equals extension protocol", () => { it("returns connected when protocol strings match", () => {
expect(computeConnectedState(handshake("1.1", "1.0"), MIN_COMPATIBLE_PROTOCOL)).toEqual({ expect(computeConnectedState(handshake("1.1", "1.1"), MIN_COMPATIBLE_PROTOCOL)).toEqual({
kind: "connected", kind: "connected",
}); });
}); });
it("returns version_skew when daemon protocol minor is newer", () => { it("returns version_skew when daemon protocol minor is newer", () => {
expect(computeConnectedState(handshake("1.2", "1.0"))).toEqual({ expect(computeConnectedState(handshake("1.2", "1.1"))).toEqual({
kind: "version_skew", kind: "version_skew",
}); });
}); });
it("returns version_skew when daemon protocol string differs but floor is satisfied", () => { it("returns version_skew when daemon protocol string differs but floor is satisfied", () => {
expect(computeConnectedState(handshake("1.1.0", "1.0"))).toEqual({ expect(computeConnectedState(handshake("1.1.0", "1.1"))).toEqual({
kind: "version_skew", kind: "version_skew",
}); });
}); });
@@ -48,7 +48,9 @@ describe("computeConnectedState (protocol-based compat)", () => {
const result = computeConnectedState(handshake("2.0", "1.0")); const result = computeConnectedState(handshake("2.0", "1.0"));
expect(result.kind).toBe("rejected"); expect(result.kind).toBe("rejected");
if (result.kind === "rejected") { if (result.kind === "rejected") {
expect(result.reason).toContain("protocol-major mismatch"); expect(result.reason).toContain("bsk CLI");
expect(result.reason).toContain("BrowserSkill extension");
expect(result.reason).not.toContain("protocol");
} }
}); });
@@ -56,8 +58,9 @@ describe("computeConnectedState (protocol-based compat)", () => {
const result = computeConnectedState(handshake("1.1", "1.5")); const result = computeConnectedState(handshake("1.1", "1.5"));
expect(result.kind).toBe("rejected"); expect(result.kind).toBe("rejected");
if (result.kind === "rejected") { if (result.kind === "rejected") {
expect(result.reason).toContain("min_compatible_protocol"); expect(result.reason).toContain("BrowserSkill extension is out of date");
expect(result.reason).toContain("1.5"); expect(result.reason).toContain("Update and reload the extension");
expect(result.reason).not.toContain("min_compatible_protocol");
} }
}); });
@@ -75,16 +78,24 @@ describe("computeConnectedState (protocol-based compat)", () => {
const result = computeConnectedState(handshake("1.0", "1.0"), "1.1"); const result = computeConnectedState(handshake("1.0", "1.0"), "1.1");
expect(result.kind).toBe("rejected"); expect(result.kind).toBe("rejected");
if (result.kind === "rejected") { if (result.kind === "rejected") {
expect(result.reason).toContain("below extension min_compatible_protocol"); expect(result.reason).toContain("bsk CLI is out of date");
expect(result.reason).toContain("bsk daemon restart");
expect(result.reason).not.toContain("min_compatible_protocol");
} }
}); });
it("returns version_skew when daemon protocol is 1.0 and floor is satisfied", () => {
expect(computeConnectedState(handshake("1.0", "1.0"))).toEqual({
kind: "version_skew",
});
});
it("rejects malformed daemon min_compatible_protocol with a daemon-floor reason", () => { it("rejects malformed daemon min_compatible_protocol with a daemon-floor reason", () => {
const result = computeConnectedState(handshake("1.1", "not-a-protocol")); const result = computeConnectedState(handshake("1.1", "not-a-protocol"));
expect(result.kind).toBe("rejected"); expect(result.kind).toBe("rejected");
if (result.kind === "rejected") { if (result.kind === "rejected") {
expect(result.reason).toContain("daemon min_compatible_protocol"); expect(result.reason).toContain("could not verify version compatibility");
expect(result.reason).toContain("not-a-protocol"); expect(result.reason).not.toContain("min_compatible_protocol");
} }
}); });
@@ -242,11 +253,11 @@ describe("ConnectionController connectionEnabled", () => {
const second = transport.send.mock.calls[1]?.[0] as { id: string }; const second = transport.send.mock.calls[1]?.[0] as { id: string };
expect(second.id).not.toBe(first.id); expect(second.id).not.toBe(first.id);
transport.emitMessage({ id: first.id, result: handshake("1.1", "1.0") }); transport.emitMessage({ id: first.id, result: handshake("1.1", "1.1") });
await Promise.resolve(); await Promise.resolve();
expect(controller.snapshot().state).not.toBe("connected"); expect(controller.snapshot().state).not.toBe("connected");
transport.emitMessage({ id: second.id, result: handshake("1.1", "1.0") }); transport.emitMessage({ id: second.id, result: handshake("1.1", "1.1") });
await vi.waitFor(() => expect(controller.snapshot().state).toBe("connected")); await vi.waitFor(() => expect(controller.snapshot().state).toBe("connected"));
}); });
}); });
@@ -1,101 +0,0 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import type { CdpRunner } from "@/tools/shared";
import { waitForDocumentSettled } from "../recording/document-settle";
function quietCdp(): { cdp: CdpRunner; send: ReturnType<typeof vi.fn> } {
const send = vi.fn(async () => ({
result: { value: { idleMs: 1_000, readyState: "complete" } },
}));
return {
cdp: { send: send as unknown as CdpRunner["send"] },
send,
};
}
describe("waitForDocumentSettled", () => {
afterEach(() => {
vi.useRealTimers();
});
it("waits through the minimum observation floor before accepting a quiet page", async () => {
vi.useFakeTimers();
const { cdp, send } = quietCdp();
const settled = waitForDocumentSettled(cdp, { target: { tabId: 7 } });
await vi.advanceTimersByTimeAsync(180);
await expect(settled).resolves.toBe("quiet");
expect(send).toHaveBeenCalledTimes(3);
});
it("cancels before probing a superseded observation", async () => {
vi.useFakeTimers();
const { cdp, send } = quietCdp();
const controller = new AbortController();
const settled = waitForDocumentSettled(
cdp,
{ target: { tabId: 7 } },
{
signal: controller.signal,
},
);
controller.abort();
await vi.advanceTimersByTimeAsync(60);
await expect(settled).resolves.toBe("cancelled");
expect(send).not.toHaveBeenCalled();
});
it("probes a same-process iframe through its isolated execution context", async () => {
vi.useFakeTimers();
const send = vi.fn(async (_tabId: number, method: string) => {
if (method === "Page.createIsolatedWorld") return { executionContextId: 91 };
return { result: { value: { idleMs: 1_000, readyState: "complete" } } };
});
const cdp: CdpRunner = { send: send as unknown as CdpRunner["send"] };
const settled = waitForDocumentSettled(cdp, {
target: { tabId: 7 },
frameId: "child-frame",
});
await vi.advanceTimersByTimeAsync(180);
await expect(settled).resolves.toBe("quiet");
expect(send).toHaveBeenCalledWith(
7,
"Page.createIsolatedWorld",
expect.objectContaining({ frameId: "child-frame" }),
);
expect(send).toHaveBeenCalledWith(
7,
"Runtime.evaluate",
expect.objectContaining({ contextId: 91 }),
);
});
it("routes an OOPIF probe through its CDP target session", async () => {
vi.useFakeTimers();
const send = vi.fn();
const sendToTarget = vi.fn(async (_target, method: string) => {
if (method === "Page.createIsolatedWorld") return { executionContextId: 27 };
return { result: { value: { idleMs: 1_000, readyState: "complete" } } };
});
const cdp: CdpRunner = {
send: send as unknown as CdpRunner["send"],
sendToTarget: sendToTarget as unknown as NonNullable<CdpRunner["sendToTarget"]>,
};
const target = { tabId: 7, sessionId: "oopif-session" };
const settled = waitForDocumentSettled(cdp, { target, frameId: "oopif-frame" });
await vi.advanceTimersByTimeAsync(180);
await expect(settled).resolves.toBe("quiet");
expect(send).not.toHaveBeenCalled();
expect(sendToTarget).toHaveBeenCalledWith(
target,
"Runtime.evaluate",
expect.objectContaining({ contextId: 27 }),
);
});
});
@@ -32,7 +32,6 @@ describe("help-bridge", () => {
prompt: "log in", prompt: "log in",
displayMode: "compact", displayMode: "compact",
selectors: ["#login"], selectors: ["#login"],
rects: [{ top: 10, left: 20, width: 100, height: 40 }],
timeoutMs: 1000, timeoutMs: 1000,
}), }),
).toBe(true); ).toBe(true);
@@ -0,0 +1,116 @@
import type { RenderedRef } from "@browser-skill/vom";
import { describe, expect, it } from "vitest";
import type { CapturedNode } from "@/tools/vom/capture";
import { matchTarget } from "../match-target";
function node(backendNodeId: number, overrides: Partial<CapturedNode> = {}): CapturedNode {
return {
backendNodeId,
parentBackendNodeId: null,
tag: "button",
attrs: {},
rect: { x: 10, y: 20, w: 100, h: 30 },
paintOrder: 1,
position: "static",
pointerEvents: "auto",
...overrides,
};
}
describe("matchTarget", () => {
const refs: RenderedRef[] = [
{ ref: "e1", backendNodeId: 42, role: "button", name: "发布", line: 3 },
];
it("matches a unique node by viewport coordinates", () => {
const target = matchTarget({
geometry: {
rect: { x: 10, y: 20, w: 100, h: 30 },
scrollX: 0,
scrollY: 0,
position: "static",
tag: "button",
},
captured: [node(42)],
refs,
fallback: { tag: "button", name: "发布" },
});
expect(target).toEqual({ ref: "e1", role: "button", name: "发布" });
});
it("matches a non-fixed node after the top frame scrolls", () => {
const target = matchTarget({
geometry: {
rect: { x: 10, y: 20, w: 100, h: 30 },
scrollX: 40,
scrollY: 300,
position: "static",
tag: "button",
},
captured: [
node(42, {
documentRect: { x: 50, y: 320, w: 100, h: 30 },
localRect: { x: 10, y: 20, w: 100, h: 30 },
rect: { x: 10, y: 20, w: 100, h: 30 },
}),
],
refs,
fallback: { tag: "button", name: "发布" },
});
expect(target).toEqual({ ref: "e1", role: "button", name: "发布" });
});
it("matches a static node by document coordinates when scroll changed after observation", () => {
const target = matchTarget({
geometry: {
rect: { x: 10, y: 50, w: 100, h: 30 },
scrollX: 0,
scrollY: 250,
position: "static",
tag: "button",
},
captured: [
node(42, {
documentRect: { x: 10, y: 300, w: 100, h: 30 },
localRect: { x: 10, y: 200, w: 100, h: 30 },
rect: { x: 10, y: 200, w: 100, h: 30 },
}),
],
refs,
fallback: { tag: "button", name: "发布" },
});
expect(target).toEqual({ ref: "e1", role: "button", name: "发布" });
});
it("returns unmatched when multiple candidates match", () => {
const target = matchTarget({
geometry: {
rect: { x: 10, y: 20, w: 100, h: 30 },
scrollX: 0,
scrollY: 0,
position: "static",
tag: "button",
},
captured: [node(42), node(43)],
refs,
});
expect(target.unmatched).toBe(true);
});
it("uses viewport coordinates for fixed elements", () => {
const target = matchTarget({
geometry: {
rect: { x: 5, y: 5, w: 80, h: 24 },
scrollX: 100,
scrollY: 200,
position: "fixed",
tag: "button",
},
captured: [node(42, { position: "fixed", rect: { x: 5, y: 5, w: 80, h: 24 } })],
refs,
});
expect(target.ref).toBe("e1");
});
});
@@ -1,32 +0,0 @@
import { describe, expect, it } from "vitest";
import {
isAcceptedRecordStepAck,
isRecordStepMessage,
RECORD_STEP,
type RecordStepMessage,
} from "../record-bridge";
const message: RecordStepMessage = {
type: RECORD_STEP,
requestId: "rec-1",
producerId: "document-1",
sequence: 1,
step: { op: "click", target: { tag: "button", name: "Submit" } },
};
describe("record step delivery protocol", () => {
it("requires a producer and a positive integer sequence", () => {
expect(isRecordStepMessage(message)).toBe(true);
expect(isRecordStepMessage({ ...message, producerId: "" })).toBe(false);
expect(isRecordStepMessage({ ...message, sequence: 0 })).toBe(false);
expect(isRecordStepMessage({ ...message, sequence: 1.5 })).toBe(false);
});
it("accepts only the acknowledgement for the sent sequence", () => {
expect(isAcceptedRecordStepAck({ ok: true, sequence: 2 }, 2)).toBe(true);
expect(isAcceptedRecordStepAck({ ok: true, sequence: 1 }, 2)).toBe(false);
expect(isAcceptedRecordStepAck({ ok: false, expectedSequence: 2, error: "gap" }, 2)).toBe(
false,
);
});
});
@@ -1,213 +0,0 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
import { RECORD_FRAME_PORT, RECORD_FRAME_QUERY } from "../recording/frame-bridge";
import { RecordFrameCoordinator } from "../recording/frame-coordinator";
class ListenerSet<T extends (...args: never[]) => unknown> {
readonly listeners = new Set<T>();
addListener = (listener: T) => this.listeners.add(listener);
removeListener = (listener: T) => this.listeners.delete(listener);
}
function fakePort(sender: chrome.runtime.MessageSender) {
const messages: unknown[] = [];
const onMessage = new ListenerSet<(message: unknown) => void>();
const onDisconnect = new ListenerSet<() => void>();
return {
port: {
name: RECORD_FRAME_PORT,
sender,
onMessage,
onDisconnect,
postMessage: vi.fn((message: unknown) => messages.push(message)),
disconnect: vi.fn(),
} as unknown as chrome.runtime.Port,
messages,
receive(message: unknown) {
for (const listener of onMessage.listeners) listener(message);
},
};
}
describe("RecordFrameCoordinator", () => {
const onConnect = new ListenerSet<(port: chrome.runtime.Port) => void>();
const onMessage = new ListenerSet<
(
message: unknown,
sender: chrome.runtime.MessageSender,
sendResponse: (response: unknown) => void,
) => boolean
>();
beforeEach(() => {
onConnect.listeners.clear();
onMessage.listeners.clear();
vi.stubGlobal("chrome", {
runtime: { onConnect, onMessage },
});
});
it("answers child-document queries for the initial tab before it is armed", () => {
const coordinator = new RecordFrameCoordinator({
getAllFrames: async () => [],
sendToDocument: async () => ({ ok: true }),
});
coordinator.attach();
coordinator.begin("rec-1", 10, 3);
const query = (tabId: number) => {
let response: unknown;
for (const listener of onMessage.listeners) {
listener(
{ type: RECORD_FRAME_QUERY },
{
tab: { id: tabId },
frameId: 7,
documentId: "child-document",
} as chrome.runtime.MessageSender,
(value) => {
response = value;
},
);
}
return response;
};
expect(query(3)).toEqual({ active: true, requestId: "rec-1", startedAtMs: 10 });
expect(query(4)).toEqual({ active: false });
});
it("starts a child Document that appears after the initial frame snapshot", async () => {
let notifyFrameNavigation:
| ((frame: {
tabId: number;
frameId: number;
documentId?: string;
lifecycle: "committed" | "completed";
}) => void)
| undefined;
const sendToDocument = vi.fn(async () => ({ ok: true }));
const coordinator = new RecordFrameCoordinator({
getAllFrames: async () => [{ frameId: 0, documentId: "top-document" }],
sendToDocument,
subscribeFrameNavigation(listener) {
notifyFrameNavigation = listener;
return () => {};
},
});
coordinator.attach();
const onDocumentReady = vi.fn();
coordinator.begin("rec-1", 10, 3, onDocumentReady);
await coordinator.armTab("rec-1", 3);
sendToDocument.mockClear();
notifyFrameNavigation?.({
tabId: 3,
frameId: 7,
documentId: "late-child",
lifecycle: "completed",
});
await vi.waitFor(() => {
expect(sendToDocument).toHaveBeenCalledWith(
3,
{ type: "bsk-record-frame-start", requestId: "rec-1", startedAtMs: 10 },
{ documentId: "late-child" },
);
});
expect(onDocumentReady).not.toHaveBeenCalled();
const child = fakePort({
tab: { id: 3 },
frameId: 7,
documentId: "late-child",
} as chrome.runtime.MessageSender);
for (const listener of onConnect.listeners) listener(child.port);
child.receive({ type: "ready", requestId: "rec-1", producerId: "producer-1" });
notifyFrameNavigation?.({
tabId: 3,
frameId: 7,
documentId: "late-child",
lifecycle: "completed",
});
await vi.waitFor(() => {
expect(onDocumentReady).toHaveBeenCalledWith({
tabId: 3,
documentId: "late-child",
browserFrameId: 7,
producerId: "producer-1",
});
});
});
it("binds a producer to its sender Document and keeps final steps valid while stopping", async () => {
const sendToDocument = vi.fn(async () => ({ ok: true }));
const coordinator = new RecordFrameCoordinator({
getAllFrames: async () => [
{ frameId: 0, documentId: "top-document" },
{ frameId: 7, documentId: "child-document" },
],
sendToDocument,
});
coordinator.attach();
coordinator.begin("rec-1", 10, 3);
await expect(coordinator.armTab("rec-1", 3)).resolves.toBe(true);
const sender = {
tab: { id: 3 },
frameId: 7,
documentId: "child-document",
} as chrome.runtime.MessageSender;
const frame = fakePort(sender);
for (const listener of onConnect.listeners) listener(frame.port);
frame.receive({ type: "ready", requestId: "rec-1", producerId: "producer-1" });
expect(coordinator.sourceFor("rec-1", "producer-1", sender)).toEqual({
tabId: 3,
documentId: "child-document",
browserFrameId: 7,
producerId: "producer-1",
});
const stopping = coordinator.stop("rec-1");
expect(coordinator.sourceFor("rec-1", "producer-1", sender)).not.toBeNull();
const stop = frame.messages.find(
(message): message is { type: "stop"; commandId: string } =>
typeof message === "object" &&
message !== null &&
(message as { type?: string }).type === "stop",
);
expect(stop).toBeDefined();
frame.receive({
type: "stopped",
requestId: "rec-1",
commandId: stop!.commandId,
ok: true,
});
await expect(stopping).resolves.toBe(true);
expect(coordinator.sourceFor("rec-1", "producer-1", sender)).toBeNull();
});
it("rejects a producer used from another Document", async () => {
const coordinator = new RecordFrameCoordinator({
getAllFrames: async () => [{ frameId: 0, documentId: "top-document" }],
sendToDocument: async () => ({ ok: true }),
});
coordinator.attach();
coordinator.begin("rec-1", 10, 3);
await coordinator.armTab("rec-1", 3);
const sender = {
tab: { id: 3 },
frameId: 0,
documentId: "top-document",
} as chrome.runtime.MessageSender;
const frame = fakePort(sender);
for (const listener of onConnect.listeners) listener(frame.port);
frame.receive({ type: "ready", requestId: "rec-1", producerId: "producer-1" });
expect(
coordinator.sourceFor("rec-1", "producer-1", {
...sender,
documentId: "other-document",
}),
).toBeNull();
});
});
@@ -0,0 +1,114 @@
import { beforeEach, describe, expect, it } from "vitest";
import type { CapturedNode } from "@/tools/vom/capture";
import type { DraftTraceStep } from "@/transport/types";
import { resetStateIdCounterForTests } from "../record-constants";
import {
applyTargetMatching,
buildTraceV3,
createObservationState,
rememberStepOnPage,
} from "../record-observation";
import { registerObservation } from "../trace-reducer";
const URL = "https://example.com/login";
function capturedInput(): CapturedNode {
return {
backendNodeId: 42,
parentBackendNodeId: null,
tag: "input",
attrs: {},
rect: { x: 20, y: 40, w: 200, h: 30 },
localRect: { x: 20, y: 40, w: 200, h: 30 },
paintOrder: 1,
position: "static",
pointerEvents: "auto",
};
}
function finalizedFillBody(value: string, redactValues: boolean): string {
const obs = createObservationState({ redactValues });
const rawVomText = '@vom 1\ntextbox "Password" value="••••••" [ref=e1]';
const stateId = registerObservation(obs.stateRegistry, { url: URL, rawVomText });
obs.lastSettled = {
stateId,
captured: [capturedInput()],
refs: [{ ref: "e1", backendNodeId: 42, role: "textbox", name: "Password", line: 1 }],
url: URL,
vomText: rawVomText,
};
const draft: DraftTraceStep = {
op: "fill",
target: { unmatched: true },
value,
geometry: {
rect: { x: 20, y: 40, w: 200, h: 30 },
scrollX: 0,
scrollY: 0,
position: "static",
tag: "input",
},
};
applyTargetMatching(obs, draft, 1);
draft.postStateId = stateId;
rememberStepOnPage(obs, draft, 1);
return (
buildTraceV3({
obs,
steps: [draft],
startedAt: "2026-08-12T00:00:00.000Z",
stoppedBy: "user_finish",
bskVersion: "test",
}).states[0]?.body ?? ""
);
}
describe("record observation annotations", () => {
beforeEach(() => {
resetStateIdCounterForTests();
});
it("omits a fill literal from finalized state bodies when values are redacted", () => {
const secret = "hunter2-private";
const body = finalizedFillBody(secret, true);
expect(body).toContain("step 1: fill");
expect(body).not.toContain(secret);
});
it("keeps fill details in ordinary recording annotations", () => {
const value = "ordinary text";
const body = finalizedFillBody(value, false);
expect(body).toContain(`step 1: fill: ${JSON.stringify(value)}`);
});
});
describe("applyTargetMatching without a settled observation", () => {
it("keeps the capture role/name instead of a bare unmatched target", () => {
const obs = createObservationState();
const draft: DraftTraceStep = {
op: "hover",
target: { unmatched: true },
captureTarget: { tag: "button", role: "button", name: "新建" },
geometry: {
rect: { x: 900, y: 8, w: 60, h: 32 },
scrollX: 0,
scrollY: 0,
position: "static",
tag: "button",
},
};
applyTargetMatching(obs, draft, 1);
expect(draft.preStateId).toBeUndefined();
expect(draft.target).toEqual({
role: "button",
name: "新建",
unmatched: true,
});
});
});
@@ -1,70 +0,0 @@
import { describe, expect, it } from "vitest";
import {
RECORD_DOCUMENT_ATTRIBUTE,
recordingDocumentMarkerValue,
} from "@/shared/recording-document-identity";
import { markRecordingDocument } from "../recording/document-marker";
import { ObservationNodeIndex } from "../recording/observation-capture";
describe("recording document marker", () => {
it("rejects a non-random Document identity", () => {
expect(() => markRecordingDocument("user@example.com")).toThrow(
"recording Document identity must be a random UUID",
);
});
it("restores the Document attribute exactly", () => {
document.documentElement.setAttribute(RECORD_DOCUMENT_ATTRIBUTE, "page-value");
const producerId = "123e4567-e89b-42d3-a456-426614174000";
const marker = markRecordingDocument(producerId);
expect(document.documentElement.getAttribute(RECORD_DOCUMENT_ATTRIBUTE)).toBe(
recordingDocumentMarkerValue(producerId),
);
marker.restore();
expect(document.documentElement.getAttribute(RECORD_DOCUMENT_ATTRIBUTE)).toBe("page-value");
document.documentElement.removeAttribute(RECORD_DOCUMENT_ATTRIBUTE);
});
it("indexes the CDP scope from the safe frame identity", () => {
const index = new ObservationNodeIndex({
rootFrameId: "root",
frames: [
{
frameId: "child",
target: { tabId: 3, sessionId: "oopif-session" },
recordingDocumentId: "producer-1",
},
],
matchNodes: [],
refs: [],
});
expect(index.documentScope("producer-1")).toEqual({
frameId: "child",
target: { tabId: 3, sessionId: "oopif-session" },
});
});
it("fails closed when a recording identity maps to multiple Documents", () => {
const index = new ObservationNodeIndex({
rootFrameId: "root",
frames: [
{
frameId: "left",
target: { tabId: 3, sessionId: "left-session" },
recordingDocumentId: "duplicate",
},
{
frameId: "right",
target: { tabId: 3, sessionId: "right-session" },
recordingDocumentId: "duplicate",
},
],
matchNodes: [],
refs: [],
});
expect(index.documentScope("duplicate")).toBeUndefined();
});
});
@@ -1,146 +0,0 @@
import { describe, expect, it } from "vitest";
import { ObservationNodeIndex } from "../recording/observation-capture";
import { RecordingObservationSession } from "../recording/observation-session";
import { RecordingStateRegistry } from "../recording/state-registry";
import { buildTraceV3 } from "../recording/trace-builder-v3";
import type { RecordingDraftStep } from "../recording/types";
const URL = "https://example.com/login";
function sessionWithInput(redactValues = false): RecordingObservationSession {
const session = new RecordingObservationSession({ redactValues });
const state = session.registry.register({
url: URL,
vomText: '@vom 1\ntextbox "Password" value="••••••" [ref=e1]',
});
session.cursor.lastSettled = {
stateId: state.id,
rootFrameId: "root",
index: new ObservationNodeIndex({
rootFrameId: "root",
matchNodes: [
{
frameId: "root",
backendNodeId: 42,
tag: "input",
rect: { x: 20, y: 40, w: 200, h: 30 },
localRect: { x: 20, y: 40, w: 200, h: 30 },
},
],
refs: [{ ref: "e1", backendNodeId: 42, role: "textbox", name: "Password", line: 1 }],
}),
url: URL,
};
return session;
}
function finalizedFillTrace(value: string, redactValues: boolean) {
const session = sessionWithInput(redactValues);
const draft: RecordingDraftStep = {
op: "fill",
captureTarget: { tag: "input", role: "textbox", name: "Password" },
value,
targetHint: {
geometry: { rect: { x: 20, y: 40, w: 200, h: 30 }, tag: "input" },
},
};
session.bindDraft(draft, 1);
draft.postStateId = draft.preStateId;
return buildTraceV3({
registry: session.registry,
drafts: [draft],
annotations: session.annotations,
startedAt: "2026-08-12T00:00:00.000Z",
stoppedBy: "user_finish",
bskVersion: "test",
redactValues,
});
}
function finalizedFillBody(value: string, redactValues: boolean): string {
return finalizedFillTrace(value, redactValues).states[0]!.body;
}
describe("record observation annotations", () => {
it("omits a fill literal when values are redacted", () => {
const secret = "hunter2-private";
const dumped = JSON.stringify(finalizedFillTrace(secret, true));
expect(dumped).toContain("step 1: fill");
expect(dumped).not.toContain(secret);
});
it("keeps ordinary fill details", () => {
expect(finalizedFillBody("ordinary text", false)).toContain('step 1: fill: "ordinary text"');
});
it("encodes title and URL so line breaks cannot corrupt state metadata", () => {
const registry = new RecordingStateRegistry();
const state = registry.register({
url: "https://example.com/a\nb",
title: "hello\nworld",
vomText: "@vom 1",
});
const trace = buildTraceV3({
registry,
drafts: [{ op: "scroll", preStateId: state.id, postStateId: state.id }],
startedAt: "2026-08-12T00:00:00.000Z",
stoppedBy: "user_finish",
bskVersion: "test",
});
expect(trace.states[0]?.body).toContain('url: "https://example.com/a\\nb"');
expect(trace.states[0]?.body).toContain('title: "hello\\nworld"');
});
});
describe("recording state ownership", () => {
it("deduplicates within one recording and isolates ids between recordings", () => {
const first = new RecordingStateRegistry();
const second = new RecordingStateRegistry();
expect(first.register({ url: URL, vomText: "same" }).id).toBe("s1");
expect(first.register({ url: URL, vomText: "same" }).id).toBe("s1");
expect(second.register({ url: URL, vomText: "other" }).id).toBe("s1");
});
it("enriches metadata when a deduplicated observation becomes more complete", () => {
const registry = new RecordingStateRegistry();
registry.register({ url: URL, vomText: "same" });
const state = registry.register({
url: URL,
title: "Login",
vomText: "same",
truncated: true,
});
expect(state).toMatchObject({ id: "s1", title: "Login", truncated: true });
});
});
describe("draft binding", () => {
it("keeps capture semantics when no observation exists", () => {
const session = new RecordingObservationSession();
const draft: RecordingDraftStep = {
op: "hover",
captureTarget: { tag: "button", role: "button", name: "新建" },
};
session.bindDraft(draft, 1);
expect(draft.matchedTarget).toEqual({ role: "button", name: "新建", unmatched: true });
expect(draft.preStateId).toBeUndefined();
});
it("does not bind an unmatched new action to a stale observation", () => {
const session = sessionWithInput();
const draft: RecordingDraftStep = {
op: "click",
captureTarget: { tag: "button", role: "button", name: "Confirm" },
targetHint: {
geometry: { rect: { x: 400, y: 300, w: 80, h: 30 }, tag: "button" },
},
};
session.bindDraft(draft, 2, true);
expect(draft.preStateId).toBeUndefined();
expect("matchedTarget" in draft ? draft.matchedTarget : undefined).toEqual({
role: "button",
name: "Confirm",
unmatched: true,
});
});
});
@@ -1,289 +0,0 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
const captureRecordingObservation = vi.hoisted(() => vi.fn());
const waitForDocumentSettled = vi.hoisted(() => vi.fn(async () => "quiet" as const));
vi.mock("../recording/observation-capture", async (importOriginal) => {
const actual = await importOriginal<typeof import("../recording/observation-capture")>();
return { ...actual, captureRecordingObservation };
});
vi.mock("../recording/document-settle", async (importOriginal) => {
const actual = await importOriginal<typeof import("../recording/document-settle")>();
return { ...actual, waitForDocumentSettled };
});
import { ObservationNodeIndex } from "../recording/observation-capture";
import { RecordingObservationRuntime } from "../recording/recording-runtime";
import type { RecordingDraftStep } from "../recording/types";
function observation(url = "https://example.com/") {
return {
rootFrameId: "root",
index: new ObservationNodeIndex({ rootFrameId: "root", matchNodes: [], refs: [] }),
url,
title: "Example",
vomText: '@vom 1\nRootWebArea "Example"',
truncated: false,
};
}
function runtime(): RecordingObservationRuntime {
return new RecordingObservationRuntime({
cdp: { send: vi.fn() as unknown as CdpRunner["send"] },
tabsApi: { get: vi.fn(), query: vi.fn() } as unknown as ChromeTabsApi,
});
}
describe("RecordingObservationRuntime", () => {
afterEach(() => {
captureRecordingObservation.mockReset();
waitForDocumentSettled.mockClear();
});
it("shares one initial capture and includes it in flush", async () => {
let release!: (value: ReturnType<typeof observation>) => void;
captureRecordingObservation.mockReturnValueOnce(
new Promise((resolve) => {
release = resolve;
}),
);
const recording = runtime();
const first = recording.captureInitial(7);
const second = recording.captureInitial(7);
const flushed = recording.flush();
let flushCompleted = false;
void flushed.then(() => {
flushCompleted = true;
});
await Promise.resolve();
expect(captureRecordingObservation).toHaveBeenCalledTimes(1);
expect(flushCompleted).toBe(false);
release(observation());
await Promise.all([first, second, flushed]);
expect(flushCompleted).toBe(true);
});
it("cancels an in-flight initial capture", async () => {
captureRecordingObservation.mockImplementationOnce(
(input: { signal?: AbortSignal }) =>
new Promise((_, reject) => {
input.signal?.addEventListener(
"abort",
() => reject(new DOMException("observation aborted", "AbortError")),
{ once: true },
);
}),
);
const recording = runtime();
const pending = recording.captureInitial(7);
recording.cancel();
await expect(pending).rejects.toMatchObject({ name: "AbortError" });
});
it("retries a failed initial capture when the recording settles at stop", async () => {
captureRecordingObservation
.mockRejectedValueOnce(new Error("document swapped"))
.mockResolvedValueOnce(observation());
const recording = runtime();
await expect(recording.captureInitial(7)).rejects.toThrow("document swapped");
await recording.settleTrailing(7, []);
const trace = recording.buildTrace({
drafts: [],
startedAt: "2026-08-19T00:00:00.000Z",
stoppedBy: "user_finish",
bskVersion: "0.1.6",
});
expect(captureRecordingObservation).toHaveBeenCalledTimes(2);
expect(trace.states).toHaveLength(1);
});
it("keeps runtime matching data out of the serialized trace", async () => {
const captured = observation();
captured.index = new ObservationNodeIndex({
rootFrameId: "root",
matchNodes: [
{
frameId: "root",
backendNodeId: 42,
tag: "button",
rect: { x: 10, y: 20, w: 100, h: 30 },
localRect: { x: 10, y: 20, w: 100, h: 30 },
},
],
refs: [{ ref: "e1", backendNodeId: 42, role: "button", name: "Submit", line: 1 }],
});
captureRecordingObservation.mockResolvedValueOnce(captured);
const recording = runtime();
await recording.captureInitial(7);
const dumped = JSON.stringify(
recording.buildTrace({
drafts: [],
startedAt: "2026-08-19T00:00:00.000Z",
stoppedBy: "user_finish",
bskVersion: "0.1.6",
}),
);
expect(dumped).toContain("RootWebArea");
expect(dumped).not.toContain("matchNodes");
expect(dumped).not.toContain("localRect");
expect(dumped).not.toContain("backendNodeId");
expect(dumped).not.toContain("ObservationNodeIndex");
});
it("captures a tab transition without sharing observation cursors", async () => {
captureRecordingObservation
.mockResolvedValueOnce(observation("https://example.com/first"))
.mockResolvedValueOnce(observation("https://example.com/second"));
const recording = runtime();
await recording.captureInitial(4);
const transition = await recording.captureTabTransition(4, 5);
await recording.captureInitial(4);
await recording.captureInitial(5);
expect(transition).toEqual({
preStateId: "s1",
postStateId: "s2",
targetUrl: "https://example.com/second",
});
expect(captureRecordingObservation.mock.calls.map(([input]) => input.tabId)).toEqual([4, 5]);
});
it("binds an iframe draft to its marked CDP Document scope", async () => {
captureRecordingObservation.mockResolvedValueOnce({
...observation(),
index: new ObservationNodeIndex({
rootFrameId: "root",
frames: [
{
frameId: "child",
target: { tabId: 7, sessionId: "oopif-session" },
recordingDocumentId: "producer-1",
},
],
matchNodes: [
{
backendNodeId: 42,
frameId: "child",
tag: "button",
rect: { x: 410, y: 20, w: 100, h: 30 },
localRect: { x: 10, y: 20, w: 100, h: 30 },
},
],
refs: [{ ref: "e1", backendNodeId: 42, frameId: "child", line: 1 }],
}),
});
const recording = runtime();
const drafts: RecordingDraftStep[] = [
{
op: "click" as const,
captureTarget: { tag: "button", role: "button", name: "Save" },
targetHint: {
geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" },
},
},
];
await recording.captureInitial(7);
await recording.processDraft(7, drafts, 0, "producer-1");
recording.cancel();
const draft = drafts[0];
expect(draft?.op).toBe("click");
if (!draft || draft.op !== "click") throw new Error("expected click draft");
expect(draft.targetHint).toMatchObject({
frameId: "child",
geometrySpace: "local",
});
expect(draft.matchedTarget?.ref).toBe("e1");
});
it("refreshes a late iframe before matching its first action", async () => {
const iframeObservation = {
...observation(),
index: new ObservationNodeIndex({
rootFrameId: "root",
frames: [
{
frameId: "child",
target: { tabId: 7, sessionId: "oopif-session" },
recordingDocumentId: "producer-1",
},
],
matchNodes: [
{
backendNodeId: 42,
frameId: "child",
tag: "button",
rect: { x: 410, y: 20, w: 100, h: 30 },
localRect: { x: 10, y: 20, w: 100, h: 30 },
},
],
refs: [
{
ref: "e1",
backendNodeId: 42,
frameId: "child",
role: "button",
name: "表格视图",
line: 1,
},
],
}),
};
captureRecordingObservation
.mockResolvedValueOnce(observation())
.mockResolvedValueOnce(iframeObservation)
.mockResolvedValueOnce(iframeObservation);
const recording = runtime();
await recording.captureInitial(7);
await recording.refreshDocument(7, "producer-1");
const drafts: RecordingDraftStep[] = [
{
op: "click",
captureTarget: { tag: "button", role: "button", name: "表格视图" },
targetHint: {
geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" },
},
},
];
await recording.processDraft(7, drafts, 0, "producer-1");
recording.cancel();
expect(captureRecordingObservation).toHaveBeenCalledTimes(3);
expect(waitForDocumentSettled).toHaveBeenCalledWith(
expect.anything(),
{ frameId: "child", target: { tabId: 7, sessionId: "oopif-session" } },
{ signal: expect.any(AbortSignal) },
);
const draft = drafts[0];
expect(draft?.op).toBe("click");
if (!draft || draft.op !== "click") throw new Error("expected click draft");
expect(draft.matchedTarget).toMatchObject({ ref: "e1", name: "表格视图" });
});
it("does not poison final flush when a frame readiness refresh fails", async () => {
captureRecordingObservation
.mockResolvedValueOnce(observation())
.mockRejectedValueOnce(new Error("child Document replaced"));
const recording = runtime();
await recording.captureInitial(7);
await expect(recording.refreshDocument(7, "producer-1")).rejects.toThrow(
"child Document replaced",
);
await expect(recording.flush()).resolves.toBeUndefined();
});
});
@@ -1,9 +1,9 @@
import { describe, expect, it } from "vitest"; import { describe, expect, it } from "vitest";
import { appendRecordedPayload, observeRecordedNavigation } from "../recording/step-buffer"; import { appendRecordedPayload, observeRecordedNavigation } from "../recording-step-buffer";
describe("recording-step-buffer", () => { describe("recording-step-buffer", () => {
it("stores semantic click without summary", () => { it("stores semantic click without summary", () => {
const buffer = { steps: [], navigation: { pendingNavigation: false } }; const buffer = { steps: [], pendingNavigation: false };
appendRecordedPayload(buffer, { appendRecordedPayload(buffer, {
op: "click", op: "click",
target: { tag: "button", role: "button", name: "发布" }, target: { tag: "button", role: "button", name: "发布" },
@@ -12,40 +12,37 @@ describe("recording-step-buffer", () => {
expect(buffer.steps).toEqual([ expect(buffer.steps).toEqual([
{ {
op: "click", op: "click",
target: { unmatched: true },
captureTarget: { tag: "button", role: "button", name: "发布" }, captureTarget: { tag: "button", role: "button", name: "发布" },
}, },
]); ]);
expect(buffer.navigation.pendingNavigation).toBe(true); expect(buffer.pendingNavigation).toBe(true);
}); });
it("keeps the hovered element description as a capture fallback", () => { it("keeps the hovered element description as a capture fallback", () => {
const buffer = { steps: [], navigation: { pendingNavigation: false } }; const buffer = { steps: [], pendingNavigation: false };
appendRecordedPayload( appendRecordedPayload(buffer, {
buffer, op: "hover",
{ target: { tag: "button", role: "button", name: "新建" },
op: "hover", geometry: {
target: { tag: "button", role: "button", name: "新建" }, rect: { x: 900, y: 8, w: 60, h: 32 },
geometry: { scrollX: 0,
rect: { x: 900, y: 8, w: 60, h: 32 }, scrollY: 0,
tag: "button", position: "static",
}, tag: "button",
}, },
{ });
geometry: {
rect: { x: 900, y: 8, w: 60, h: 32 },
tag: "button",
},
},
);
expect(buffer.steps).toEqual([ expect(buffer.steps).toEqual([
{ {
op: "hover", op: "hover",
target: { unmatched: true },
captureTarget: { tag: "button", role: "button", name: "新建" }, captureTarget: { tag: "button", role: "button", name: "新建" },
targetHint: { geometry: {
geometry: { rect: { x: 900, y: 8, w: 60, h: 32 },
rect: { x: 900, y: 8, w: 60, h: 32 }, scrollX: 0,
tag: "button", scrollY: 0,
}, position: "static",
tag: "button",
}, },
}, },
]); ]);
@@ -56,21 +53,19 @@ describe("recording-step-buffer", () => {
steps: [ steps: [
{ {
op: "click" as const, op: "click" as const,
captureTarget: { tag: "button", role: "button", name: "发布" }, target: { tag: "button", role: "button", name: "发布" },
}, },
], ],
navigation: { currentUrl: "https://example.com/a",
currentUrl: "https://example.com/a", pendingNavigation: true,
pendingNavigation: true, pendingNavigationDeadline: Date.now() + 5_000,
pendingNavigationDeadline: Date.now() + 5_000,
},
}; };
observeRecordedNavigation(buffer, "https://example.com/b", true); observeRecordedNavigation(buffer, "https://example.com/b", true);
expect(buffer.steps).toEqual([ expect(buffer.steps).toEqual([
{ {
op: "click", op: "click",
captureTarget: { tag: "button", role: "button", name: "发布" }, target: { tag: "button", role: "button", name: "发布" },
navigatedTo: "https://example.com/b", navigated_to: "https://example.com/b",
}, },
]); ]);
expect(JSON.stringify(buffer.steps)).not.toContain("wait_for_navigation"); expect(JSON.stringify(buffer.steps)).not.toContain("wait_for_navigation");
@@ -81,23 +76,21 @@ describe("recording-step-buffer", () => {
steps: [ steps: [
{ {
op: "select" as const, op: "select" as const,
captureTarget: { tag: "select", role: "combobox", name: "分类" }, target: { tag: "select", role: "combobox", name: "分类" },
values: ["tech"], values: ["tech"],
}, },
], ],
navigation: { currentUrl: "https://example.com/list",
currentUrl: "https://example.com/list", pendingNavigation: true,
pendingNavigation: true, pendingNavigationDeadline: Date.now() + 5_000,
pendingNavigationDeadline: Date.now() + 5_000,
},
}; };
observeRecordedNavigation(buffer, "https://example.com/list?cat=tech", true); observeRecordedNavigation(buffer, "https://example.com/list?cat=tech", true);
expect(buffer.steps).toEqual([ expect(buffer.steps).toEqual([
{ {
op: "select", op: "select",
captureTarget: { tag: "select", role: "combobox", name: "分类" }, target: { tag: "select", role: "combobox", name: "分类" },
values: ["tech"], values: ["tech"],
navigatedTo: "https://example.com/list?cat=tech", navigated_to: "https://example.com/list?cat=tech",
}, },
]); ]);
}); });
@@ -105,7 +98,8 @@ describe("recording-step-buffer", () => {
it("emits navigate for uncaused URL changes", () => { it("emits navigate for uncaused URL changes", () => {
const buffer = { const buffer = {
steps: [], steps: [],
navigation: { currentUrl: "https://example.com/a", pendingNavigation: false }, currentUrl: "https://example.com/a",
pendingNavigation: false,
}; };
const result = observeRecordedNavigation(buffer, "https://example.com/b", false); const result = observeRecordedNavigation(buffer, "https://example.com/b", false);
expect(result).toEqual({ kind: "appended", index: 0 }); expect(result).toEqual({ kind: "appended", index: 0 });
@@ -118,10 +112,8 @@ describe("recording-step-buffer", () => {
it("asks the recorder to coalesce redirect hops instead of emitting each one", () => { it("asks the recorder to coalesce redirect hops instead of emitting each one", () => {
const buffer = { const buffer = {
steps: [], steps: [],
navigation: { currentUrl: "https://passport.example/login",
currentUrl: "https://passport.example/login", pendingNavigation: false,
pendingNavigation: false,
},
}; };
const hop1 = observeRecordedNavigation( const hop1 = observeRecordedNavigation(
buffer, buffer,
@@ -144,21 +136,6 @@ describe("recording-step-buffer", () => {
url: "https://app.example/dashboard", url: "https://app.example/dashboard",
}); });
expect(buffer.steps).toEqual([]); expect(buffer.steps).toEqual([]);
expect(buffer.navigation.currentUrl).toBe("https://app.example/dashboard"); expect(buffer.currentUrl).toBe("https://app.example/dashboard");
});
it("does not reuse redirect metadata for a later content-observed URL change", () => {
const buffer = {
steps: [],
navigation: {
currentUrl: "https://example.com/redirect",
pendingNavigation: false,
},
};
const result = observeRecordedNavigation(buffer, "https://example.com/spa");
expect(result).toEqual({ kind: "appended", index: 0 });
expect(buffer.steps[0]).toMatchObject({ op: "navigate", url: "https://example.com/spa" });
}); });
}); });
@@ -1,136 +0,0 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
import { ObservationNodeIndex, type RegisteredObservation } from "../recording/observation-capture";
import { RecordingObservationSession } from "../recording/observation-session";
import { SettleController } from "../recording/settle-controller";
import type { RecordingDraftStep } from "../recording/types";
const OBSERVATION: RegisteredObservation = {
stateId: "s-next",
rootFrameId: "root",
index: new ObservationNodeIndex({ rootFrameId: "root", matchNodes: [], refs: [] }),
url: "https://example.com/next",
};
describe("SettleController", () => {
afterEach(() => vi.useRealTimers());
it("aborts superseded capture work and lands the prior action on the next origin", async () => {
vi.useFakeTimers();
const cdp: CdpRunner = {
send: vi.fn(async () => ({
result: { value: { idleMs: 1_000, readyState: "complete" } },
})) as unknown as CdpRunner["send"],
};
const tabsApi: ChromeTabsApi = {
get: vi.fn(async () => ({ id: 7, url: OBSERVATION.url }) as chrome.tabs.Tab),
query: vi.fn(async () => []),
};
const session = new RecordingObservationSession();
let firstSignal: AbortSignal | undefined;
vi.spyOn(session, "capture")
.mockImplementationOnce(async (_cdp, _tabs, _tabId, signal) => {
firstSignal = signal;
return new Promise<RegisteredObservation>((_resolve, reject) => {
signal?.addEventListener(
"abort",
() => reject(new DOMException("observation aborted", "AbortError")),
{ once: true },
);
});
})
.mockResolvedValueOnce(OBSERVATION);
const drafts: RecordingDraftStep[] = [
{ op: "click", captureTarget: { tag: "button", name: "First" }, preStateId: "s1" },
];
const controller = new SettleController({ session, cdp, tabsApi, tabId: 7 });
controller.schedule(drafts, 0);
await vi.advanceTimersByTimeAsync(180);
drafts.push({
op: "click",
captureTarget: { tag: "button", name: "Second" },
preStateId: "s-next",
});
controller.schedule(drafts, 1);
expect(firstSignal?.aborted).toBe(true);
expect(drafts[0]?.postStateId).toBe("s-next");
await vi.advanceTimersByTimeAsync(500);
await controller.flush();
expect(drafts[1]?.postStateId).toBe("s-next");
});
it("does not consume a newer redirect while reading the prior landing URL", async () => {
vi.useFakeTimers();
const cdp: CdpRunner = {
send: vi.fn(async () => ({
result: { value: { idleMs: 1_000, readyState: "complete" } },
})) as unknown as CdpRunner["send"],
};
let resolveFirstTab!: (tab: chrome.tabs.Tab) => void;
const firstTab = new Promise<chrome.tabs.Tab>((resolve) => {
resolveFirstTab = resolve;
});
const tabsApi: ChromeTabsApi = {
get: vi
.fn()
.mockImplementationOnce(() => firstTab)
.mockResolvedValue({ id: 7, url: OBSERVATION.url }),
query: vi.fn(async () => []),
};
const session = new RecordingObservationSession();
session.cursor.lastSettled = OBSERVATION;
const drafts: RecordingDraftStep[] = [];
const controller = new SettleController({ session, cdp, tabsApi, tabId: 7 });
controller.scheduleRedirect(drafts, "https://example.com/intermediate");
await vi.advanceTimersByTimeAsync(180);
expect(tabsApi.get).toHaveBeenCalledTimes(1);
controller.scheduleRedirect(drafts, OBSERVATION.url);
resolveFirstTab({ id: 7, url: "https://example.com/intermediate" } as chrome.tabs.Tab);
await vi.advanceTimersByTimeAsync(500);
await controller.flushRedirects();
expect(drafts).toEqual([]);
expect(controller.hasPending).toBe(false);
});
it("settles an action in its own OOPIF document scope", async () => {
vi.useFakeTimers();
const send = vi.fn();
const sendToTarget = vi.fn(async (_target, method: string) => {
if (method === "Page.createIsolatedWorld") return { executionContextId: 39 };
return { result: { value: { idleMs: 1_000, readyState: "complete" } } };
});
const cdp: CdpRunner = {
send: send as unknown as CdpRunner["send"],
sendToTarget: sendToTarget as unknown as NonNullable<CdpRunner["sendToTarget"]>,
};
const tabsApi: ChromeTabsApi = {
get: vi.fn(async () => ({ id: 7, url: OBSERVATION.url }) as chrome.tabs.Tab),
query: vi.fn(async () => []),
};
const session = new RecordingObservationSession();
vi.spyOn(session, "capture").mockResolvedValue(OBSERVATION);
const drafts: RecordingDraftStep[] = [
{ op: "click", captureTarget: { tag: "button", name: "Inside frame" } },
];
const controller = new SettleController({ session, cdp, tabsApi, tabId: 7 });
const target = { tabId: 7, sessionId: "oopif-session" };
controller.schedule(drafts, 0, { target, frameId: "oopif-frame" });
await vi.advanceTimersByTimeAsync(180);
await controller.flush();
expect(send).not.toHaveBeenCalled();
expect(sendToTarget).toHaveBeenCalledWith(
target,
"Runtime.evaluate",
expect.objectContaining({ contextId: 39 }),
);
expect(drafts[0]?.postStateId).toBe("s-next");
});
});
@@ -1,39 +0,0 @@
import { describe, expect, it } from "vitest";
import { RecordingTabCoordinator } from "../recording/tab-coordinator";
describe("RecordingTabCoordinator", () => {
it("keeps navigation state isolated per tab", () => {
const tabs = new RecordingTabCoordinator(4, "https://example.com/first");
const first = tabs.navigation(4);
first.pendingNavigation = true;
const second = tabs.navigation(5, "https://example.com/second");
expect(second).toEqual({
currentUrl: "https://example.com/second",
pendingNavigation: false,
});
expect(tabs.navigation(4)).toBe(first);
expect(tabs.navigation(4).pendingNavigation).toBe(true);
});
it("invalidates an earlier activation when a newer tab becomes active", () => {
const tabs = new RecordingTabCoordinator(4);
const second = tabs.noteActivation(5);
const third = tabs.noteActivation(6);
expect(tabs.isLatest(second)).toBe(false);
expect(tabs.isLatest(third)).toBe(true);
tabs.commit(6);
expect(tabs.currentTabId).toBe(6);
});
it("commits the freshly observed URL when returning to an existing tab", () => {
const tabs = new RecordingTabCoordinator(4);
tabs.navigation(5, "https://example.com/old");
tabs.commit(5, "https://example.com/new");
expect(tabs.currentTabId).toBe(5);
expect(tabs.navigation(5).currentUrl).toBe("https://example.com/new");
});
});
@@ -1,159 +0,0 @@
import type { RenderedRef } from "@browser-skill/vom";
import { describe, expect, it } from "vitest";
import type { CaptureVomMatchNode } from "@/tools/capture-vom-observation";
import { ObservationNodeIndex, type RegisteredObservation } from "../recording/observation-capture";
import { matchObservationTarget } from "../recording/target-matcher";
function node(backendNodeId: number, frameId: string, x = 10): CaptureVomMatchNode {
return {
backendNodeId,
frameId,
tag: "button",
rect: { x, y: 20, w: 100, h: 30 },
localRect: { x, y: 20, w: 100, h: 30 },
};
}
function observation(
matchNodes: CaptureVomMatchNode[],
refs: RenderedRef[],
): RegisteredObservation {
return {
stateId: "s1",
rootFrameId: "root",
index: new ObservationNodeIndex({ rootFrameId: "root", matchNodes, refs }),
url: "https://example.com",
};
}
describe("matchObservationTarget", () => {
it("indexes the safe geometry DTO without discarding frame-local geometry", () => {
const geometry = node(42, "child");
geometry.localRect = { x: 5, y: 6, w: 100, h: 30 };
const ref: RenderedRef = {
ref: "e1",
backendNodeId: 42,
frameId: "child",
line: 1,
};
const candidate = observation([geometry], [ref]).index.candidates("child", "button")[0];
expect(candidate).toEqual({ frameId: "child", geometry, ref });
});
it("matches a unique node using canonical top-level viewport geometry", () => {
const target = matchObservationTarget({
observation: observation(
[node(42, "root")],
[{ ref: "e1", backendNodeId: 42, role: "button", name: "发布", line: 1 }],
),
hint: { geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" } },
});
expect(target).toEqual({ ref: "e1", role: "button", name: "发布" });
});
it("uses frame id with backend node id so sibling frames cannot collide", () => {
const target = matchObservationTarget({
observation: observation(
[node(42, "left"), node(42, "right")],
[
{ ref: "e1", backendNodeId: 42, frameId: "left", line: 1 },
{ ref: "e2", backendNodeId: 42, frameId: "right", line: 2 },
],
),
hint: {
frameId: "right",
geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" },
},
});
expect(target.ref).toBe("e2");
});
it("matches iframe capture geometry in the child viewport coordinate space", () => {
const child = node(42, "child", 410);
child.localRect = { x: 10, y: 20, w: 100, h: 30 };
const target = matchObservationTarget({
observation: observation(
[child],
[{ ref: "e1", backendNodeId: 42, frameId: "child", line: 1 }],
),
hint: {
frameId: "child",
geometrySpace: "local",
geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" },
},
});
expect(target.ref).toBe("e1");
});
it("restricts a missing frame hint to the root frame", () => {
const target = matchObservationTarget({
observation: observation(
[node(42, "child")],
[{ ref: "e1", backendNodeId: 42, frameId: "child", line: 1 }],
),
hint: { geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" } },
fallback: { tag: "button", name: "发布" },
});
expect(target).toEqual({ name: "发布", unmatched: true });
});
it("fails closed when the source Document has no VOM frame mapping", () => {
const target = matchObservationTarget({
observation: observation(
[node(42, "root")],
[{ ref: "e1", backendNodeId: 42, role: "button", name: "发布", line: 1 }],
),
hint: {
frameId: null,
geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" },
},
fallback: { tag: "button", role: "button", name: "发布" },
});
expect(target).toEqual({ role: "button", name: "发布", unmatched: true });
});
it("returns unmatched for ambiguous geometry", () => {
const target = matchObservationTarget({
observation: observation(
[node(42, "root"), node(43, "root")],
[
{ ref: "e1", backendNodeId: 42, line: 1 },
{ ref: "e2", backendNodeId: 43, line: 2 },
],
),
hint: { geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" } },
});
expect(target.unmatched).toBe(true);
});
it("uses semantics when geometry is unavailable without crossing frame boundaries", () => {
const target = matchObservationTarget({
observation: observation(
[node(41, "root"), node(42, "child")],
[
{ ref: "e1", backendNodeId: 41, role: "button", name: "保存", line: 1 },
{ ref: "e2", backendNodeId: 42, frameId: "child", role: "button", name: "保存", line: 2 },
],
),
hint: { frameId: "child" },
fallback: { tag: "button", role: "button", name: "保存" },
});
expect(target.ref).toBe("e2");
});
it("uses semantics to disambiguate equal geometry in one frame", () => {
const target = matchObservationTarget({
observation: observation(
[node(42, "root"), node(43, "root")],
[
{ ref: "e1", backendNodeId: 42, role: "button", name: "保存", line: 1 },
{ ref: "e2", backendNodeId: 43, role: "button", name: "取消", line: 2 },
],
),
hint: { geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" } },
fallback: { tag: "button", role: "button", name: "取消" },
});
expect(target.ref).toBe("e2");
});
});
@@ -1,202 +1,70 @@
import { describe, expect, it } from "vitest"; import { describe, expect, it } from "vitest";
import type { RecordingDraftStep } from "@/lib/recording/types"; import { buildTraceV2, shouldRecordPress } from "@/lib/trace-reducer-v2";
import { shouldRecordPress } from "../recording/draft-policy"; import type { DraftTraceStep } from "@/transport/types";
import { buildTraceV2 } from "../recording/trace-reducer-v2";
function reduceTraceSteps(steps: RecordingDraftStep[], startUrl?: string) { describe("trace-reducer-v2", () => {
return buildTraceV2({ it("drops scroll and bare character press steps", () => {
steps, const steps: DraftTraceStep[] = [
startedAt: "2026-01-01T00:00:00.000Z",
...(startUrl ? { startUrl } : {}),
});
}
describe("shouldRecordPress", () => {
it("keeps Enter and Escape", () => {
expect(shouldRecordPress("Enter")).toBe(true);
expect(shouldRecordPress("Escape")).toBe(true);
});
it("drops modifiers, clipboard shortcuts, and bare typing", () => {
expect(shouldRecordPress("Meta")).toBe(false);
expect(shouldRecordPress("c", ["meta"])).toBe(false);
expect(shouldRecordPress("a", ["ctrl"])).toBe(false);
expect(shouldRecordPress("x")).toBe(false);
expect(shouldRecordPress("中")).toBe(false);
});
});
describe("reduceTraceSteps", () => {
it("does not expose internal tab transitions in trace v2", () => {
const trace = reduceTraceSteps([
{ {
op: "switch_tab", op: "scroll",
preStateId: "s1", page_url: "https://example.com/",
postStateId: "s2",
},
]);
expect(trace.steps).toEqual([]);
});
it("builds steps with pages dictionary and page id references", () => {
const drafts: RecordingDraftStep[] = [
{
op: "navigate",
url: "https://example.com/search?q=hello&utm_source=x",
pageUrl: "https://example.com/search?q=hello&utm_source=x",
},
{
op: "fill",
captureTarget: { tag: "input", role: "textbox", name: "搜索", name_attr: "q" },
value: "browser skill",
pageUrl: "https://example.com/search?q=hello&utm_source=x",
},
{
op: "press",
key: "Enter",
captureTarget: { tag: "input", role: "textbox", name: "搜索", name_attr: "q" },
navigatedTo: "https://example.com/results/42",
pageUrl: "https://example.com/search?q=hello&utm_source=x",
},
{
op: "click",
captureTarget: { tag: "button", role: "button", name: "发布" },
navigatedTo: "https://example.com/p/99",
pageUrl: "https://example.com/results/42",
}, },
{ {
op: "press", op: "press",
key: "a", key: "a",
pageUrl: "https://example.com/p/99", page_url: "https://example.com/",
}, },
]; ];
const trace = buildTraceV2({
const { pages, steps } = reduceTraceSteps( steps,
drafts, startedAt: "2026-01-01T00:00:00.000Z",
"https://example.com/search?q=hello&utm_source=x", startUrl: "https://example.com/",
);
expect(JSON.stringify(steps)).not.toContain("parameters");
expect(JSON.stringify(steps)).not.toContain("intent");
expect(JSON.stringify(steps)).not.toContain("summary");
expect(steps.map((s) => s.op)).toEqual(["navigate", "fill", "press", "click"]);
expect(pages.map((p) => p.url)).toEqual([
"https://example.com/search?q=hello&utm_source=x",
"https://example.com/results/42",
"https://example.com/p/99",
]);
expect(steps[0]).toMatchObject({
id: 1,
op: "navigate",
page: "p1",
to: "https://example.com/search?q=hello&utm_source=x",
});
expect(steps[1]).toMatchObject({
op: "fill",
page: "p1",
value: "browser skill",
});
expect(steps[2]).toMatchObject({
op: "press",
key: "Enter",
page: "p1",
effect: { navigated_to: "p2" },
});
expect(steps[3]).toMatchObject({
op: "click",
page: "p2",
effect: { navigated_to: "p3" },
}); });
expect(trace.steps).toHaveLength(0);
expect(trace.pages).toHaveLength(1);
}); });
it("collapses consecutive navigations", () => { it("drops hover steps unsupported by historical v2 clients", () => {
const { steps } = reduceTraceSteps([ const trace = buildTraceV2({
{ op: "navigate", url: "https://a.example/redirect1" }, steps: [
{ op: "navigate", url: "https://a.example/final" },
]);
expect(steps).toHaveLength(1);
expect(steps[0]).toMatchObject({
op: "navigate",
to: "https://a.example/final",
});
});
it("keeps committed empty fill steps", () => {
const { steps } = reduceTraceSteps(
[
{
op: "fill",
captureTarget: { tag: "input", role: "textbox", name: "Search query" },
value: "",
pageUrl: "https://example.com/search",
},
],
"https://example.com/search",
);
expect(steps).toEqual([
expect.objectContaining({
op: "fill",
value: "",
}),
]);
});
it("maps select navigatedTo onto effect.navigatedTo (page id)", () => {
const { pages, steps } = reduceTraceSteps(
[
{
op: "select",
captureTarget: { tag: "select", role: "combobox", name: "分类" },
values: ["tech"],
labels: ["技术"],
navigatedTo: "https://example.com/list?cat=tech",
pageUrl: "https://example.com/list",
},
],
"https://example.com/list",
);
expect(pages.map((p) => p.url)).toEqual([
"https://example.com/list",
"https://example.com/list?cat=tech",
]);
expect(steps[0]).toMatchObject({
op: "select",
page: "p1",
selection: [{ value: "tech", label: "技术" }],
effect: { navigated_to: "p2" },
});
});
it("drops hover steps, which only exist in trace v3", () => {
const { steps } = reduceTraceSteps(
[
{ {
op: "hover", op: "hover",
captureTarget: { tag: "span", role: "button", name: "Account" }, target: { unmatched: true },
pageUrl: "https://example.com/app", captureTarget: { tag: "button", role: "button", name: "结束" },
}, page_url: "https://example.com/",
{
op: "click",
captureTarget: { tag: "a", role: "link", name: "Profile" },
pageUrl: "https://example.com/app",
}, },
], ],
"https://example.com/app", startedAt: "2026-01-01T00:00:00.000Z",
); startUrl: "https://example.com/",
expect(steps.map((s) => s.op)).toEqual(["click"]); });
expect(trace.steps).toHaveLength(0);
}); });
it("resolveTraceStartUrl prefers explicit start URL", () => { it("maps captureTarget to v2 target with required tag", () => {
expect( const steps: DraftTraceStep[] = [
buildTraceV2({ {
steps: [{ op: "navigate", url: "https://example.com/other" }], op: "click",
startedAt: "2026-01-01T00:00:00.000Z", target: { unmatched: true },
startUrl: "https://example.com/start", captureTarget: { tag: "button", role: "button", name: "Submit" },
}).entry.start_url, page_url: "https://example.com/",
).toBe("https://example.com/start"); },
];
const trace = buildTraceV2({
steps,
startedAt: "2026-01-01T00:00:00.000Z",
startUrl: "https://example.com/",
});
expect(trace.steps).toHaveLength(1);
expect(trace.steps[0]?.op).toBe("click");
if (trace.steps[0]?.op === "click") {
expect(trace.steps[0].target.tag).toBe("button");
expect(trace.steps[0].target.name).toBe("Submit");
}
expect("version" in trace).toBe(false);
});
it("records Enter presses", () => {
expect(shouldRecordPress("Enter")).toBe(true);
}); });
}); });
@@ -1,132 +0,0 @@
import { describe, expect, it } from "vitest";
import { TRACE_VERSION_V3, VOM_FORMAT_VERSION } from "@/transport/types";
import { RecordingStateRegistry } from "../recording/state-registry";
import { buildTraceV3 } from "../recording/trace-builder-v3";
import { reduceTraceStepsV3 } from "../recording/trace-reducer-v3";
import type { RecordingDraftStep } from "../recording/types";
describe("trace reducer v3", () => {
it("removes form literals when value redaction is requested", () => {
const reduced = reduceTraceStepsV3(
[
{
op: "fill",
value: "private@example.com",
preStateId: "s1",
postStateId: "s1",
},
{
op: "select",
values: ["private-account-id"],
labels: ["Private account"],
preStateId: "s1",
postStateId: "s1",
},
],
{ redactValues: true },
);
expect(reduced.steps[0]).toMatchObject({ op: "fill", value: "***", redacted: true });
expect(reduced.steps[1]).toMatchObject({ op: "select" });
expect(reduced.steps[1]).not.toHaveProperty("selection");
expect(JSON.stringify(reduced.steps)).not.toContain("private@example.com");
expect(JSON.stringify(reduced.steps)).not.toContain("private-account-id");
});
it("emits tab transitions only for callers that advertised support", () => {
const drafts: RecordingDraftStep[] = [
{
op: "switch_tab",
preStateId: "s1",
postStateId: "s2",
},
];
expect(reduceTraceStepsV3(drafts).steps).toEqual([]);
expect(reduceTraceStepsV3(drafts, { includeTabSwitches: true }).steps).toEqual([
{ op: "switch_tab", id: 1, state: "s1", result: { state: "s2" } },
]);
});
it("keeps hover steps before menu clicks", () => {
const drafts: RecordingDraftStep[] = [
{
op: "hover",
captureTarget: { tag: "span", role: "button", name: "Account" },
preStateId: "s1",
postStateId: "s1",
},
{
op: "click",
captureTarget: { tag: "a", role: "link", name: "Profile" },
preStateId: "s1",
postStateId: "s1",
},
];
expect(reduceTraceStepsV3(drafts).steps.map((s) => s.op)).toEqual(["hover", "click"]);
});
it("does not export a tab transition without both observation endpoints", () => {
const sourceOnly: RecordingDraftStep = { op: "switch_tab", preStateId: "s1" };
const targetOnly: RecordingDraftStep = { op: "switch_tab", postStateId: "s2" };
expect(
reduceTraceStepsV3([sourceOnly, targetOnly], { includeTabSwitches: true }).steps,
).toEqual([]);
});
it("collapses redirect hops while retaining draft-to-step identity", () => {
const drafts: RecordingDraftStep[] = [
{ op: "navigate", url: "https://example.com/start", preStateId: "s1", postStateId: "s2" },
{
op: "navigate",
url: "https://example.com/final",
transitionQualifiers: ["server_redirect"],
preStateId: "s2",
postStateId: "s3",
},
];
const before = structuredClone(drafts);
const reduced = reduceTraceStepsV3(drafts);
expect(drafts).toEqual(before);
expect(reduced.steps).toEqual([
expect.objectContaining({
id: 1,
op: "navigate",
state: "s1",
to: "https://example.com/final",
result: { state: "s3" },
}),
]);
expect(reduced.stepIdByDraftId.get(1)).toBe(1);
expect(reduced.stepIdByDraftId.get(2)).toBe(1);
});
it("builds the wire model from protocol constants", () => {
const registry = new RecordingStateRegistry();
const state = registry.register({ url: "https://example.com", vomText: "@vom 1" });
const trace = buildTraceV3({
registry,
drafts: [
{
op: "click",
captureTarget: { tag: "button", role: "button", name: "Save" },
preStateId: state.id,
postStateId: state.id,
},
],
startedAt: "2026-08-12T00:00:00.000Z",
stoppedBy: "user_finish",
bskVersion: "test",
});
expect(trace.version).toBe(TRACE_VERSION_V3);
expect(trace.recorder.vom).toBe(VOM_FORMAT_VERSION);
expect(trace.steps[0]).toMatchObject({
op: "click",
target: { role: "button", name: "Save", unmatched: true },
});
});
});
@@ -0,0 +1,252 @@
import { describe, expect, it } from "vitest";
import { resetStateIdCounterForTests } from "@/lib/record-constants";
import {
reduceTraceSteps,
registerObservation,
resolveTraceStartUrl,
shouldRecordPress,
} from "../trace-reducer";
describe("shouldRecordPress", () => {
it("keeps Enter and Escape", () => {
expect(shouldRecordPress("Enter")).toBe(true);
expect(shouldRecordPress("Escape")).toBe(true);
});
it("drops modifiers, clipboard shortcuts, and bare typing", () => {
expect(shouldRecordPress("Meta")).toBe(false);
expect(shouldRecordPress("c", ["meta"])).toBe(false);
expect(shouldRecordPress("a", ["ctrl"])).toBe(false);
expect(shouldRecordPress("x")).toBe(false);
expect(shouldRecordPress("中")).toBe(false);
});
});
describe("reduceTraceSteps v3", () => {
it("builds steps with state/result references and deduped states", () => {
resetStateIdCounterForTests();
const registry = new Map();
const body = "@vom 1\nL1 page";
const s1 = registerObservation(registry, {
url: "https://example.com/search",
rawVomText: body,
});
const s2 = registerObservation(registry, {
url: "https://example.com/results/42",
rawVomText: "@vom 1\nL1 results",
});
const { states, steps } = reduceTraceSteps(
[
{
op: "navigate",
url: "https://example.com/search",
preStateId: s1,
postStateId: s1,
},
{
op: "fill",
target: { ref: "e1", role: "textbox", name: "搜索" },
value: "browser skill",
commit: "enter",
preStateId: s1,
postStateId: s1,
},
{
op: "press",
key: "Enter",
target: { ref: "e1", role: "textbox", name: "搜索" },
preStateId: s1,
postStateId: s2,
navigated_to: "https://example.com/results/42",
},
{
op: "click",
target: { ref: "e2", role: "button", name: "发布" },
preStateId: s2,
postStateId: s2,
},
],
registry,
);
expect(states).toHaveLength(2);
expect(steps.map((s) => s.op)).toEqual(["navigate", "fill", "press", "click"]);
expect(steps[1]).toMatchObject({
op: "fill",
state: s1,
result: { state: s1 },
commit: "enter",
});
expect(steps[2]).toMatchObject({
op: "press",
state: s1,
result: { state: s2 },
});
expect(JSON.stringify(steps)).not.toContain("page");
expect(JSON.stringify(steps)).not.toContain("effect");
});
it("collapses consecutive navigations", () => {
resetStateIdCounterForTests();
const registry = new Map();
const s1 = registerObservation(registry, {
url: "https://a.example/redirect1",
rawVomText: "a",
});
const { steps } = reduceTraceSteps(
[
{ op: "navigate", url: "https://a.example/redirect1", preStateId: s1, postStateId: s1 },
{ op: "navigate", url: "https://a.example/final", preStateId: s1, postStateId: s1 },
],
registry,
);
expect(steps).toHaveLength(1);
expect(steps[0]).toMatchObject({
op: "navigate",
to: "https://a.example/final",
});
});
it("keeps the origin state and cause of the hop that started a redirect chain", () => {
resetStateIdCounterForTests();
const registry = new Map();
const start = registerObservation(registry, {
url: "https://example.com/",
rawVomText: "start",
});
const hop = registerObservation(registry, {
url: "https://iwiki.woa.com/",
rawVomText: "hop",
});
const landing = registerObservation(registry, {
url: "https://iwiki.woa.com/dashboard",
rawVomText: "landing",
});
const { steps, stepIdByDraftId } = reduceTraceSteps(
[
{
op: "navigate",
url: "https://iwiki.woa.com/",
preStateId: start,
postStateId: hop,
transitionType: "typed",
},
{
op: "navigate",
url: "https://iwiki.woa.com/dashboard",
preStateId: hop,
postStateId: landing,
transitionType: "link",
},
],
registry,
);
expect(steps).toHaveLength(1);
expect(steps[0]).toMatchObject({
op: "navigate",
id: 1,
state: start,
result: { state: landing },
to: "https://iwiki.woa.com/dashboard",
cause: "user_typed",
});
// Both drafts fold into step 1, so anything tagged with either draft id
// must resolve to it.
expect(stepIdByDraftId.get(1)).toBe(1);
expect(stepIdByDraftId.get(2)).toBe(1);
});
it("maps draft ids to published step ids across dropped drafts", () => {
resetStateIdCounterForTests();
const registry = new Map();
const s1 = registerObservation(registry, { url: "https://a.example/", rawVomText: "a" });
const { steps, stepIdByDraftId } = reduceTraceSteps(
[
// Dropped: bare typing press.
{ op: "press", key: "a", preStateId: s1, postStateId: s1 },
{ op: "click", target: { ref: "e2" }, preStateId: s1, postStateId: s1 },
],
registry,
);
expect(steps.map((step) => step.op)).toEqual(["click"]);
expect(stepIdByDraftId.get(1)).toBeUndefined();
expect(stepIdByDraftId.get(2)).toBe(1);
});
it("keeps committed empty fills", () => {
resetStateIdCounterForTests();
const registry = new Map();
const s1 = registerObservation(registry, { url: "https://a.example/", rawVomText: "a" });
const { steps } = reduceTraceSteps(
[
{
op: "fill",
target: { ref: "e1", role: "textbox", name: "Search" },
value: "",
preStateId: s1,
postStateId: s1,
},
{
op: "click",
target: { ref: "e2", role: "button", name: "Apply" },
preStateId: s1,
postStateId: s1,
},
],
registry,
);
expect(steps.map((step) => step.op)).toEqual(["fill", "click"]);
expect(steps[0]).toMatchObject({ op: "fill", value: "" });
});
it("keeps hover steps before menu clicks", () => {
resetStateIdCounterForTests();
const registry = new Map();
const state = registerObservation(registry, {
url: "https://example.com/app",
rawVomText: '@e1 button "Account"\n@e2 link "Profile"',
});
const { steps } = reduceTraceSteps(
[
{
op: "hover",
target: { ref: "e1", role: "button", name: "Account" },
page_url: "https://example.com/app",
preStateId: state,
postStateId: state,
},
{
op: "click",
target: { ref: "e2", role: "link", name: "Profile" },
page_url: "https://example.com/app",
preStateId: state,
postStateId: state,
},
],
registry,
);
expect(steps.map((s) => s.op)).toEqual(["hover", "click"]);
expect(steps[0]).toMatchObject({
op: "hover",
state,
result: { state },
target: { name: "Account" },
});
});
it("resolveTraceStartUrl prefers explicit start URL", () => {
expect(
resolveTraceStartUrl(
[{ op: "navigate", url: "https://example.com/other", preStateId: "s1", postStateId: "s1" }],
"https://example.com/start",
),
).toBe("https://example.com/start");
});
});
@@ -313,6 +313,15 @@ export type HandshakeVerdict =
| { kind: "version_skew" } | { kind: "version_skew" }
| { kind: "rejected"; reason: string }; | { kind: "rejected"; reason: string };
const UPDATE_BOTH_VERSION_MESSAGE =
"The installed bsk CLI and BrowserSkill extension are incompatible. Update both, reload the extension, then run `bsk daemon restart`.";
const UPDATE_BSK_VERSION_MESSAGE =
"The installed bsk CLI is out of date for this BrowserSkill extension. Update bsk, then run `bsk daemon restart`.";
const UPDATE_EXTENSION_VERSION_MESSAGE =
"This BrowserSkill extension is out of date for the installed bsk CLI. Update and reload the extension, then try again.";
const INVALID_VERSION_MESSAGE =
"BrowserSkill could not verify version compatibility. Update bsk and the BrowserSkill extension, reload the extension, then run `bsk daemon restart`.";
/** /**
* Symmetric extension-side **protocol** compat check. * Symmetric extension-side **protocol** compat check.
* *
@@ -339,19 +348,19 @@ export function computeConnectedState(
if (daemonMajor === null || ourMajor === null || daemonMajor !== ourMajor) { if (daemonMajor === null || ourMajor === null || daemonMajor !== ourMajor) {
return { return {
kind: "rejected", kind: "rejected",
reason: `protocol-major mismatch (daemon protocol v${handshake.protocol_version}, extension protocol v${PROTOCOL_VERSION})`, reason: UPDATE_BOTH_VERSION_MESSAGE,
}; };
} }
if (compareProtocol(handshake.protocol_version, localMinCompatibleProtocol) === null) { if (compareProtocol(handshake.protocol_version, localMinCompatibleProtocol) === null) {
return { return {
kind: "rejected", kind: "rejected",
reason: `daemon protocol v${handshake.protocol_version} is unparseable`, reason: INVALID_VERSION_MESSAGE,
}; };
} }
if (compareProtocol(handshake.protocol_version, localMinCompatibleProtocol)! < 0) { if (compareProtocol(handshake.protocol_version, localMinCompatibleProtocol)! < 0) {
return { return {
kind: "rejected", kind: "rejected",
reason: `daemon protocol v${handshake.protocol_version} below extension min_compatible_protocol ${localMinCompatibleProtocol}`, reason: UPDATE_BSK_VERSION_MESSAGE,
}; };
} }
const peerFloor = handshake.min_compatible_protocol; const peerFloor = handshake.min_compatible_protocol;
@@ -359,13 +368,13 @@ export function computeConnectedState(
if (compareProtocol(PROTOCOL_VERSION, peerFloor) === null) { if (compareProtocol(PROTOCOL_VERSION, peerFloor) === null) {
return { return {
kind: "rejected", kind: "rejected",
reason: `daemon min_compatible_protocol ${peerFloor} is unparseable`, reason: INVALID_VERSION_MESSAGE,
}; };
} }
if (compareProtocol(PROTOCOL_VERSION, peerFloor)! < 0) { if (compareProtocol(PROTOCOL_VERSION, peerFloor)! < 0) {
return { return {
kind: "rejected", kind: "rejected",
reason: `extension protocol v${PROTOCOL_VERSION} below daemon min_compatible_protocol ${peerFloor}`, reason: UPDATE_EXTENSION_VERSION_MESSAGE,
}; };
} }
} }
+1 -1
View File
@@ -1,5 +1,5 @@
/** /**
* Build a semantic capture descriptor for an interacted element. * Build a semantic TargetDescriptor for an interacted element.
* *
* Trace steps are an LLM *textbook*: each click must say what to look for * Trace steps are an LLM *textbook*: each click must say what to look for
* on screen (usually a short visible name). Tag-only noise like * on screen (usually a short visible name). Tag-only noise like
@@ -0,0 +1,62 @@
import type { Step } from "@/transport/types";
import { OBSERVATION_FILE_VERSION } from "./record-constants";
export interface ObservationAnnotation {
stepId: number;
op: Step["op"];
line: number;
stateId: string;
detail?: string;
}
function formatAnnotation({ stepId, op, detail }: ObservationAnnotation): string {
const suffix = detail ? `: ${detail}` : "";
return ` ⟵ step ${stepId}: ${op}${suffix}`;
}
/** Serialize a page observation file (front matter + VOM body + step annotations). */
export function formatObservationFile(input: {
stateId: string;
url: string;
title?: string;
stepsHere: number[];
body: string;
annotations?: ObservationAnnotation[];
}): string {
const lines: string[] = [
`# bsk-observation ${OBSERVATION_FILE_VERSION}`,
`state: ${input.stateId}`,
`url: ${input.url}`,
];
if (input.title) lines.push(`title: ${input.title}`);
if (input.stepsHere.length > 0) {
lines.push(`steps_here: [${input.stepsHere.join(", ")}]`);
}
lines.push("---");
const bodyLines = input.body.split("\n");
const annotationMap = new Map<number, ObservationAnnotation[]>();
for (const ann of input.annotations ?? []) {
const bucket = annotationMap.get(ann.line) ?? [];
bucket.push(ann);
annotationMap.set(ann.line, bucket);
}
for (let i = 0; i < bodyLines.length; i += 1) {
let line = bodyLines[i] ?? "";
const anns = annotationMap.get(i);
if (anns) {
for (const ann of anns) {
line += formatAnnotation(ann);
}
}
lines.push(line);
}
return `${lines.join("\n")}\n`;
}
/** Hash input: VOM body **before** annotations are inserted. */
export function observationBodyForHash(vomText: string): string {
return vomText;
}
-19
View File
@@ -16,13 +16,6 @@ export const HELP_ACK = "bsk-help-ack";
export const HELP_QUERY = "bsk-help-query"; export const HELP_QUERY = "bsk-help-query";
export const HELP_FINISH = "bsk-help-finish"; export const HELP_FINISH = "bsk-help-finish";
export interface HelpHighlightRect {
top: number;
left: number;
width: number;
height: number;
}
export interface HelpRequestMessage { export interface HelpRequestMessage {
type: typeof HELP_REQUEST; type: typeof HELP_REQUEST;
requestId: string; requestId: string;
@@ -33,8 +26,6 @@ export interface HelpRequestMessage {
displayMode?: "full" | "compact"; displayMode?: "full" | "compact";
/** CSS selectors to scroll to + flash-highlight (may be empty). */ /** CSS selectors to scroll to + flash-highlight (may be empty). */
selectors: string[]; selectors: string[];
/** Top-viewport rectangles for targets that cannot be represented by a root selector. */
rects?: HelpHighlightRect[];
timeoutMs: number; timeoutMs: number;
} }
@@ -83,16 +74,6 @@ export function isHelpRequestMessage(msg: unknown): msg is HelpRequestMessage {
(m.displayMode === undefined || m.displayMode === "full" || m.displayMode === "compact") && (m.displayMode === undefined || m.displayMode === "full" || m.displayMode === "compact") &&
Array.isArray(m.selectors) && Array.isArray(m.selectors) &&
m.selectors.every((selector) => typeof selector === "string") && m.selectors.every((selector) => typeof selector === "string") &&
(m.rects === undefined ||
(Array.isArray(m.rects) &&
m.rects.every(
(rect) =>
typeof rect === "object" &&
rect !== null &&
["top", "left", "width", "height"].every((key) =>
Number.isFinite((rect as Record<string, unknown>)[key]),
),
))) &&
typeof m.timeoutMs === "number" typeof m.timeoutMs === "number"
); );
} }
+103
View File
@@ -0,0 +1,103 @@
import type { RenderedRef } from "@browser-skill/vom";
import type { CapturedNode } from "@/tools/vom/capture";
import type { CaptureGeometry, TargetDescriptor } from "@/transport/types";
import type { CaptureTargetDescriptor } from "./describe-target";
import { GEOM_MATCH_TOLERANCE_PX } from "./record-constants";
export interface MatchTargetInput {
geometry: CaptureGeometry;
captured: CapturedNode[];
refs: RenderedRef[];
fallback?: CaptureTargetDescriptor;
}
function withinTolerance(a: number, b: number): boolean {
return Math.abs(a - b) <= GEOM_MATCH_TOLERANCE_PX;
}
function rectsMatch(
a: { x: number; y: number; w: number; h: number },
b: { x: number; y: number; w: number; h: number },
): boolean {
return (
withinTolerance(a.x, b.x) &&
withinTolerance(a.y, b.y) &&
withinTolerance(a.w, b.w) &&
withinTolerance(a.h, b.h)
);
}
function nodeViewportRect(
node: CapturedNode,
): { x: number; y: number; w: number; h: number } | null {
return node.localRect ?? node.rect;
}
function matchingRects(
geometry: CaptureGeometry,
node: CapturedNode,
): {
target: { x: number; y: number; w: number; h: number };
node: { x: number; y: number; w: number; h: number } | null;
} {
const topFrame = (geometry.ownerFrameBackendNodeId ?? null) === null;
const viewportPosition = geometry.position === "fixed" || geometry.position === "sticky";
if (!topFrame || viewportPosition || !node.documentRect) {
return { target: geometry.rect, node: nodeViewportRect(node) };
}
return {
target: {
x: geometry.rect.x + geometry.scrollX,
y: geometry.rect.y + geometry.scrollY,
w: geometry.rect.w,
h: geometry.rect.h,
},
node: node.documentRect,
};
}
function tagMatches(geometryTag: string, nodeTag: string): boolean {
return geometryTag.toLowerCase() === nodeTag.toLowerCase();
}
/** Best description available when the element cannot be located in the VOM. */
export function fallbackDescriptor(fallback?: CaptureTargetDescriptor): TargetDescriptor {
if (!fallback) {
return { unmatched: true };
}
return {
...(fallback.role ? { role: fallback.role } : {}),
...(fallback.name ? { name: fallback.name } : {}),
unmatched: true,
};
}
/** Locate the interacted element in the last settled observation by geometry. */
export function matchTarget(input: MatchTargetInput): TargetDescriptor {
const ownerFrame = input.geometry.ownerFrameBackendNodeId ?? null;
const candidates = input.captured.filter((node) => {
if (ownerFrame !== (node.ownerFrameBackendNodeId ?? null)) return false;
if (!tagMatches(input.geometry.tag, node.tag)) return false;
const rects = matchingRects(input.geometry, node);
if (!rects.node) return false;
return rectsMatch(rects.target, rects.node);
});
if (candidates.length !== 1) {
return fallbackDescriptor(input.fallback);
}
const backendNodeId = candidates[0]!.backendNodeId;
const refEntry = input.refs.find((r) => r.backendNodeId === backendNodeId);
if (!refEntry) {
return fallbackDescriptor(input.fallback);
}
return {
ref: refEntry.ref,
...(refEntry.role ? { role: refEntry.role } : {}),
...(refEntry.name ? { name: refEntry.name } : {}),
...(refEntry.ctx ? { ctx: refEntry.ctx } : {}),
};
}
+92
View File
@@ -0,0 +1,92 @@
// Deciding *when* a page is worth observing.
//
// A fixed delay cannot serve both a modal that toggles in one frame and a
// route change that renders for a second. Instead of guessing, ask the page:
// a document that has stopped mutating and is no longer loading is done
// reacting to whatever the user just did.
import type { CdpRunner } from "@/tools/shared";
import { SETTLE_MAX_MS, SETTLE_MIN_MS, SETTLE_POLL_MS, SETTLE_QUIET_MS } from "./record-constants";
/**
* Installed once per document. Records only the timestamp of the last DOM
* change so each poll stays O(1) no matter how large the page is.
*/
const QUIET_PROBE = `(() => {
const scope = window;
let probe = scope.__bskRecordQuiet;
if (!probe) {
probe = { changedAt: Date.now() };
const observer = new MutationObserver(() => {
probe.changedAt = Date.now();
});
observer.observe(document.documentElement, {
subtree: true,
childList: true,
attributes: true,
characterData: true,
});
scope.__bskRecordQuiet = probe;
}
return { idleMs: Date.now() - probe.changedAt, readyState: document.readyState };
})()`;
export type SettleOutcome =
/** The page stopped changing on its own. */
| "quiet"
/** Still changing when the budget ran out; observe it as it is. */
| "timeout"
/** A newer action took over; this observation is no longer wanted. */
| "cancelled";
interface QuietProbe {
idleMs: number;
readyState: string;
}
function sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
async function readQuietProbe(cdp: CdpRunner, tabId: number): Promise<QuietProbe | null> {
try {
const reply = await cdp.send<{ result?: { value?: unknown } }>(tabId, "Runtime.evaluate", {
expression: QUIET_PROBE,
returnByValue: true,
});
const value = reply.result?.value;
if (!value || typeof value !== "object") return null;
const { idleMs, readyState } = value as { idleMs?: unknown; readyState?: unknown };
if (typeof idleMs !== "number" || typeof readyState !== "string") return null;
return { idleMs, readyState };
} catch {
// An unreadable page is a page mid-swap; that is a reason to keep waiting,
// not a reason to give up.
return null;
}
}
/** Wait until the page has finished reacting, or until the budget runs out. */
export async function waitForPageSettled(
cdp: CdpRunner,
tabId: number,
options: { cancelled?: () => boolean } = {},
): Promise<SettleOutcome> {
const startedAt = Date.now();
const floor = startedAt + SETTLE_MIN_MS;
const deadline = startedAt + SETTLE_MAX_MS;
for (;;) {
if (options.cancelled?.()) return "cancelled";
await sleep(SETTLE_POLL_MS);
if (options.cancelled?.()) return "cancelled";
const probe = await readQuietProbe(cdp, tabId);
const now = Date.now();
if (now < floor) continue;
if (now >= deadline) return "timeout";
if (!probe) continue;
if (probe.readyState === "loading") continue;
if (probe.idleMs >= SETTLE_QUIET_MS) return "quiet";
}
}
+7 -30
View File
@@ -16,10 +16,6 @@ export interface RecordStartAck {
ok: true; ok: true;
} }
export type RecordStepAck =
| { ok: true; sequence: number }
| { ok: false; expectedSequence: number; error: string };
export type RecordStopAck = { ok: true } | { ok: false; error: string }; export type RecordStopAck = { ok: true } | { ok: false; error: string };
export interface RecordQueryMessage { export interface RecordQueryMessage {
@@ -45,7 +41,7 @@ export interface RecordStartMessage {
} }
export interface RecordStepPayload { export interface RecordStepPayload {
op: "click" | "hover" | "fill" | "press" | "select" | "navigate"; op: "click" | "hover" | "fill" | "press" | "select" | "navigate" | "scroll";
target?: CaptureTargetDescriptor; target?: CaptureTargetDescriptor;
value?: string; value?: string;
key?: string; key?: string;
@@ -57,10 +53,14 @@ export interface RecordStepPayload {
commit?: "enter" | "suggestion" | "blur"; commit?: "enter" | "suggestion" | "blur";
/** Page URL when the step was captured. */ /** Page URL when the step was captured. */
page_url?: string; page_url?: string;
/** Event-target bounds in the source frame's viewport coordinate space. */ /** Document geometry at capture time for background geometric matching. */
geometry?: { geometry?: {
rect: { x: number; y: number; w: number; h: number }; rect: { x: number; y: number; w: number; h: number };
scrollX: number;
scrollY: number;
position: string;
tag: string; tag: string;
ownerFrameBackendNodeId?: number | null;
}; };
/** Capture-only hint; never persisted unless converted to navigated_to. */ /** Capture-only hint; never persisted unless converted to navigated_to. */
expects_navigation?: boolean; expects_navigation?: boolean;
@@ -74,10 +74,6 @@ export interface RecordStepPayload {
export interface RecordStepMessage { export interface RecordStepMessage {
type: typeof RECORD_STEP; type: typeof RECORD_STEP;
requestId: string; requestId: string;
/** Stable for the lifetime of one content-script document. */
producerId: string;
/** Monotonic within one content-script producer and recording request. */
sequence: number;
step: RecordStepPayload; step: RecordStepPayload;
} }
@@ -129,27 +125,8 @@ export function isRecordQueryMessage(msg: unknown): msg is RecordQueryMessage {
export function isRecordStepMessage(msg: unknown): msg is RecordStepMessage { export function isRecordStepMessage(msg: unknown): msg is RecordStepMessage {
if (typeof msg !== "object" || msg === null) return false; if (typeof msg !== "object" || msg === null) return false;
const m = msg as Record<string, unknown>; const m = msg as Record<string, unknown>;
if ( if (m.type !== RECORD_STEP || typeof m.requestId !== "string") return false;
m.type !== RECORD_STEP ||
typeof m.requestId !== "string" ||
typeof m.producerId !== "string" ||
m.producerId.length === 0 ||
typeof m.sequence !== "number" ||
!Number.isSafeInteger(m.sequence) ||
m.sequence < 1
) {
return false;
}
const step = m.step; const step = m.step;
if (typeof step !== "object" || step === null) return false; if (typeof step !== "object" || step === null) return false;
return typeof (step as RecordStepPayload).op === "string"; return typeof (step as RecordStepPayload).op === "string";
} }
export function isAcceptedRecordStepAck(
value: unknown,
sequence: number,
): value is Extract<RecordStepAck, { ok: true }> {
if (typeof value !== "object" || value === null) return false;
const ack = value as Partial<RecordStepAck>;
return ack.ok === true && ack.sequence === sequence;
}
@@ -0,0 +1,97 @@
import type { FillCommit, NavigationCause } from "@/transport/types";
/**
* Floor before a post-action observation may be taken. A page usually has not
* started reacting in the first frames after an action, and capturing then
* would record the page as it was, not as the action left it.
*/
export const SETTLE_MIN_MS = 150;
/** How long the DOM must stop changing before a page counts as settled. */
export const SETTLE_QUIET_MS = 250;
/** Upper bound on waiting for a page to settle; animations never stop. */
export const SETTLE_MAX_MS = 2_000;
/** How often to ask the page whether it has gone quiet. */
export const SETTLE_POLL_MS = 60;
/** Minimum interval between consecutive observations on the same tab. */
export const OBSERVATION_MIN_INTERVAL_MS = 200;
/** Delay before retrying a capture that lost its execution context. */
export const CAPTURE_RETRY_DELAY_MS = 250;
/** Document-coordinate matching tolerance in CSS pixels. */
export const GEOM_MATCH_TOLERANCE_PX = 2;
/** Default max tokens per page observation when CLI omits `--max-page-tokens`. */
export const DEFAULT_MAX_PAGE_TOKENS = 3000;
/** VOM observation file format version (front matter header). */
export const OBSERVATION_FILE_VERSION = 1;
/** FNV-1a 64-bit offset basis. */
const FNV_OFFSET = 0xcbf29ce484222325n;
/** FNV-1a 64-bit prime. */
const FNV_PRIME = 0x100000001b3n;
/** Content-hash for state deduplication (sync, non-cryptographic). */
export function fnv1a64(text: string): string {
let hash = FNV_OFFSET;
for (let i = 0; i < text.length; i += 1) {
hash ^= BigInt(text.charCodeAt(i));
hash = (hash * FNV_PRIME) & 0xffffffffffffffffn;
}
return hash.toString(16).padStart(16, "0");
}
let nextStateSerial = 0;
/** Monotonic state id generator (`s1`, `s2`, …). */
export function nextStateId(): string {
nextStateSerial += 1;
return `s${nextStateSerial}`;
}
/** Test seam: reset the state id counter. */
export function resetStateIdCounterForTests(): void {
nextStateSerial = 0;
}
const REDIRECT_QUALIFIERS = new Set(["client_redirect", "server_redirect"]);
const TRANSITION_TO_CAUSE: Record<string, NavigationCause> = {
typed: "user_typed",
generated: "user_typed",
keyword: "user_typed",
keyword_generated: "user_typed",
link: "link",
form_submit: "form_submit",
reload: "reload",
auto_bookmark: "browser",
start_page: "browser",
};
export interface NavigationTransitionMeta {
transitionType?: string;
transitionQualifiers?: string[];
navigationActionPending?: boolean;
}
/** Map webNavigation metadata to protocol `NavigationCause`. Returns null for redirects. */
export function mapNavigationCause(meta: NavigationTransitionMeta): NavigationCause | null {
const qualifiers = meta.transitionQualifiers ?? [];
if (qualifiers.includes("forward_back")) return "history";
if (qualifiers.some((q) => REDIRECT_QUALIFIERS.has(q))) return null;
if (qualifiers.includes("from_address_bar")) return "user_typed";
const type = meta.transitionType ?? "";
const mapped = TRANSITION_TO_CAUSE[type];
if (mapped) return mapped;
if (!type && meta.navigationActionPending === false) return "script";
return "browser";
}
export const DEFAULT_FILL_COMMIT: FillCommit = "blur";
@@ -0,0 +1,663 @@
import type { RenderedRef } from "@browser-skill/vom";
import { formatObservationFile, type ObservationAnnotation } from "@/lib/format-observation-file";
import { fallbackDescriptor, matchTarget } from "@/lib/match-target";
import { waitForPageSettled } from "@/lib/page-settled";
import {
CAPTURE_RETRY_DELAY_MS,
DEFAULT_MAX_PAGE_TOKENS,
OBSERVATION_MIN_INTERVAL_MS,
} from "@/lib/record-constants";
import { registerObservation, type StateRegistryEntry } from "@/lib/trace-reducer";
import { captureVomObservation } from "@/tools/capture-vom-observation";
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
import type { CapturedNode } from "@/tools/vom/capture";
import type { DraftTraceStep, Step, StopReason, Trace, TraceState } from "@/transport/types";
import { reduceTraceSteps, resolveTraceStartUrl } from "./trace-reducer";
export interface LastSettledObservation {
stateId: string;
captured: CapturedNode[];
refs: RenderedRef[];
url: string;
title?: string;
vomText: string;
}
/** A post-action observation that has been queued but not yet written down. */
interface PendingSettle {
draftIndex: number;
/** Set when a newer action has already decided where this step landed. */
cancelled: boolean;
}
/** Latest URL in an in-flight redirect chain awaiting coalesce + settle. */
export interface PendingRedirectLanding {
url: string;
/** Bumped on every hop so an in-flight `waitForPageSettled` can cancel. */
generation: number;
}
export interface RecordingObservationState {
stateRegistry: Map<string, StateRegistryEntry>;
lastSettled: LastSettledObservation | null;
maxPageTokens: number;
redactValues: boolean;
lastCaptureAtMs: number;
/**
* Tail of the settle chain. Observations run one at a time so that a slow
* capture can never land after a faster one and report the pages out of the
* order the user visited them.
*/
settleQueue: Promise<void>;
/** Queued and in-flight settles, by draft index, so newer actions can supersede them. */
settles: Map<number, PendingSettle>;
stepAnnotations: Map<number, ObservationAnnotation[]>;
/**
* OAuth / server redirects are coalesced here: intermediate hops only update
* `url`+`generation`, and one `navigate` is emitted after the page settles.
*/
pendingRedirect: PendingRedirectLanding | null;
redirectFlushQueue: Promise<void>;
}
export function createObservationState(options?: {
maxPageTokens?: number;
redactValues?: boolean;
}): RecordingObservationState {
return {
stateRegistry: new Map(),
lastSettled: null,
maxPageTokens: options?.maxPageTokens ?? DEFAULT_MAX_PAGE_TOKENS,
redactValues: options?.redactValues ?? false,
lastCaptureAtMs: 0,
settleQueue: Promise.resolve(),
settles: new Map(),
stepAnnotations: new Map(),
pendingRedirect: null,
redirectFlushQueue: Promise.resolve(),
};
}
function sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
async function readTabMeta(
tabsApi: ChromeTabsApi,
tabId: number,
): Promise<{ url: string; title?: string }> {
try {
const tab = await tabsApi.get(tabId);
return { url: tab.url ?? "about:blank", title: tab.title };
} catch {
return { url: "about:blank" };
}
}
export async function captureAndRegisterObservation(
obs: RecordingObservationState,
cdp: CdpRunner,
tabId: number,
tabsApi: ChromeTabsApi,
urlOverride?: string,
): Promise<LastSettledObservation> {
const now = Date.now();
const waitMs = Math.max(0, OBSERVATION_MIN_INTERVAL_MS - (now - obs.lastCaptureAtMs));
if (waitMs > 0) await sleep(waitMs);
const { url, title } = urlOverride
? { url: urlOverride, title: undefined }
: await readTabMeta(tabsApi, tabId);
const meta = urlOverride ? await readTabMeta(tabsApi, tabId) : { url, title };
const resolvedTitle = title ?? meta.title;
const rendered = await captureVomObservation(cdp, tabId, url, {
maxTokens: obs.maxPageTokens,
redactValues: obs.redactValues,
conditionalSurfaceProbe: false,
});
const stateId = registerObservation(obs.stateRegistry, {
url,
title: resolvedTitle,
rawVomText: rendered.text,
truncated: rendered.truncated,
});
obs.lastCaptureAtMs = Date.now();
const settled: LastSettledObservation = {
stateId,
captured: rendered.captured,
refs: rendered.refs,
url,
title: resolvedTitle,
vomText: rendered.text,
};
obs.lastSettled = settled;
return settled;
}
/**
* Bind a draft to the page it was performed on: origin state, geometric target
* match, and the inline annotation. All three read the *current* observation,
* which is only the page the user acted on while the draft is still fresh —
* once the action settles, `lastSettled` is the destination page instead.
*/
export function applyTargetMatching(
obs: RecordingObservationState,
draft: DraftTraceStep,
stepId: number,
): void {
if (!obs.lastSettled) {
// Initial observation can fail or race the first action. Keep whatever
// the content script captured so the exported step is still teachable.
if ("target" in draft && "captureTarget" in draft) {
draft.target = fallbackDescriptor(draft.captureTarget);
}
rememberStepAnnotation(obs, stepId, draft);
return;
}
draft.preStateId = obs.lastSettled.stateId;
if ("target" in draft) {
draft.target =
"geometry" in draft && draft.geometry
? matchTarget({
geometry: draft.geometry,
captured: obs.lastSettled.captured,
refs: obs.lastSettled.refs,
fallback: draft.captureTarget,
})
: // No geometry to match on, but the capture still knew what the user
// touched — keep that instead of an anonymous unmatched target.
fallbackDescriptor(draft.captureTarget);
}
rememberStepAnnotation(obs, stepId, draft);
}
function fillDetailForDraft(
obs: RecordingObservationState,
draft: DraftTraceStep,
): string | undefined {
if (obs.redactValues) return undefined;
if (draft.op === "fill") return JSON.stringify(draft.value);
return undefined;
}
function refLineForDraft(
obs: RecordingObservationState,
draft: DraftTraceStep,
): number | undefined {
if (!("target" in draft) || !obs.lastSettled) return undefined;
const targetRef = draft.target?.ref;
if (!targetRef) return undefined;
return obs.lastSettled.refs.find((r) => r.ref === targetRef)?.line;
}
function rememberStepAnnotation(
obs: RecordingObservationState,
stepId: number,
draft: DraftTraceStep,
): void {
const line = refLineForDraft(obs, draft);
if (line === undefined || draft.op === "navigate" || draft.op === "scroll") return;
if (!draft.preStateId) return;
const ann: ObservationAnnotation = {
stepId,
op: draft.op,
line,
stateId: draft.preStateId,
detail: fillDetailForDraft(obs, draft),
};
const bucket = obs.stepAnnotations.get(line) ?? [];
bucket.push(ann);
obs.stepAnnotations.set(line, bucket);
}
/** Record the step under the page it was performed on, not the one it led to. */
export function rememberStepOnPage(
obs: RecordingObservationState,
draft: DraftTraceStep,
stepId: number,
): void {
const stateId = draft.preStateId;
if (!stateId) return;
const entry = obs.stateRegistry.get(stateId);
if (entry && !entry.stepsHere.includes(stepId)) {
entry.stepsHere.push(stepId);
}
}
/** Drop an in-flight redirect coalesce (e.g. a real navigate superseded it). */
export function clearPendingRedirectLanding(obs: RecordingObservationState): void {
obs.pendingRedirect = null;
}
function enqueueRedirectFlush(obs: RecordingObservationState, task: () => Promise<void>): void {
obs.redirectFlushQueue = obs.redirectFlushQueue.then(task, task).catch(() => {});
}
/**
* Remember a redirect hop and schedule a settle-then-emit of a single navigate
* to the final URL. Later hops only bump `generation` / replace `url`.
*/
export function scheduleRedirectLandingFlush(
obs: RecordingObservationState,
steps: DraftTraceStep[],
cdp: CdpRunner,
tabId: number,
tabsApi: ChromeTabsApi,
url: string,
options: { cancelled?: () => boolean } = {},
): void {
const generation = (obs.pendingRedirect?.generation ?? 0) + 1;
obs.pendingRedirect = { url, generation };
enqueueRedirectFlush(obs, () =>
coalesceRedirectLanding(obs, steps, cdp, tabId, tabsApi, options),
);
}
/**
* Drain any coalesced redirect so the next action matches against the real
* landing page. Safe to call when nothing is pending.
*/
export async function flushPendingRedirectLanding(
obs: RecordingObservationState,
steps: DraftTraceStep[],
cdp: CdpRunner | undefined,
tabId: number,
tabsApi: ChromeTabsApi,
options: { cancelled?: () => boolean } = {},
): Promise<void> {
if (obs.pendingRedirect && cdp) {
enqueueRedirectFlush(obs, () =>
coalesceRedirectLanding(obs, steps, cdp, tabId, tabsApi, options),
);
}
await obs.redirectFlushQueue;
}
/**
* Wait until the redirect chain stops changing the document, then emit one
* `navigate` (cause `browser`) to the tab's final URL and settle its observation.
*/
async function coalesceRedirectLanding(
obs: RecordingObservationState,
steps: DraftTraceStep[],
cdp: CdpRunner,
tabId: number,
tabsApi: ChromeTabsApi,
options: { cancelled?: () => boolean },
): Promise<void> {
while (obs.pendingRedirect) {
if (options.cancelled?.()) {
obs.pendingRedirect = null;
return;
}
const snap = obs.pendingRedirect;
const outcome = await waitForPageSettled(cdp, tabId, {
cancelled: () =>
!!options.cancelled?.() ||
obs.pendingRedirect === null ||
obs.pendingRedirect.generation !== snap.generation,
});
if (options.cancelled?.()) {
obs.pendingRedirect = null;
return;
}
// A newer hop cancelled this wait — loop and settle against the latest URL.
if (outcome === "cancelled") continue;
if (!obs.pendingRedirect || obs.pendingRedirect.generation !== snap.generation) {
continue;
}
const finalUrl = (await readTabMeta(tabsApi, tabId)).url || snap.url;
obs.pendingRedirect = null;
if (!finalUrl || finalUrl === "about:blank") return;
if (obs.lastSettled?.url === finalUrl) return;
const last = steps[steps.length - 1];
if (last?.op === "navigate" && last.url === finalUrl) {
if (!last.postStateId) {
const draftIndex = steps.length - 1;
scheduleDraftSettle(obs, draftIndex, draftIndex + 1, cdp, tabId, tabsApi, steps);
await obs.settleQueue;
}
return;
}
if (outcome === "timeout") {
console.debug(
`[bsk record] redirect landing still changing after settle budget; ` +
`recording navigate to ${finalUrl}`,
);
}
const draft: DraftTraceStep = {
op: "navigate",
url: finalUrl,
page_url: finalUrl,
cause: "browser",
preStateId: obs.lastSettled?.stateId,
};
steps.push(draft);
const draftIndex = steps.length - 1;
const stepId = draftIndex + 1;
rememberStepOnPage(obs, draft, stepId);
scheduleDraftSettle(obs, draftIndex, stepId, cdp, tabId, tabsApi, steps);
// Callers that flush before the next action need `lastSettled` to already
// be the landing page so target matching does not use the pre-redirect view.
await obs.settleQueue;
return;
}
}
/**
* A capture that lands mid-navigation fails on a destroyed execution context.
* That is transient, so give it one more chance before the step has to fall
* back to a stale state.
*/
async function captureWithRetry(
obs: RecordingObservationState,
cdp: CdpRunner,
tabId: number,
tabsApi: ChromeTabsApi,
urlOverride?: string,
): Promise<LastSettledObservation> {
try {
return await captureAndRegisterObservation(obs, cdp, tabId, tabsApi, urlOverride);
} catch (err) {
console.debug("[bsk record] observation failed, retrying once", err);
await sleep(CAPTURE_RETRY_DELAY_MS);
return captureAndRegisterObservation(obs, cdp, tabId, tabsApi, urlOverride);
}
}
export async function settleDraftObservation(
obs: RecordingObservationState,
cdp: CdpRunner,
tabId: number,
tabsApi: ChromeTabsApi,
steps: DraftTraceStep[],
draftIndex: number,
pending?: PendingSettle,
): Promise<void> {
const cancelled = () => pending?.cancelled === true;
const outcome = await waitForPageSettled(cdp, tabId, { cancelled });
if (outcome === "cancelled") return;
if (outcome === "timeout") {
console.debug(
`[bsk record] step ${draftIndex + 1} was still changing the page after the settle ` +
"budget; observing it as it stands",
);
}
const started = steps[draftIndex];
if (!started) return;
// The URL attached to the draft may only be an intermediate redirect hop.
// Once the page is settled, register the capture against live tab metadata.
const settled = await captureWithRetry(obs, cdp, tabId, tabsApi);
// The observation still counts as the latest view of the page even when this
// step no longer wants it — the next action will start from it.
if (cancelled()) return;
// Re-read the slot: a navigation observed while the capture ran may have
// rewritten this draft, and the observation belongs to whatever occupies the
// slot now — writing to the object we started with would strand it.
const draft = steps[draftIndex] ?? started;
draft.postStateId = settled.stateId;
}
/**
* Recover landing states from the shape of the recording itself: wherever the
* next action was performed is, by definition, where the previous one landed.
* Only the immediately following draft counts — a later one would claim a page
* that several unobserved actions away, which is worse than admitting we saw
* no change.
*/
export function inferMissingPostStates(steps: DraftTraceStep[]): void {
for (let i = 0; i < steps.length - 1; i += 1) {
const draft = steps[i];
const next = steps[i + 1];
if (!draft || !next || draft.postStateId || !next.preStateId) continue;
draft.postStateId = next.preStateId;
console.debug(
`[bsk record] step ${i + 1} (${draft.op}) had no post-action observation; ` +
`using where step ${i + 2} started (${next.preStateId})`,
);
}
}
/**
* Last chance for the closing actions of a recording. Stopping right after the
* final action is normal, and so is that action navigating away, so take one
* final look at the page. Only trailing drafts may claim it: an earlier step
* did not land on whatever the user happens to be looking at when they stop.
*/
export async function settleUnsettledDrafts(
obs: RecordingObservationState,
cdp: CdpRunner,
tabId: number,
tabsApi: ChromeTabsApi,
steps: DraftTraceStep[],
): Promise<void> {
const trailing: DraftTraceStep[] = [];
for (let i = steps.length - 1; i >= 0; i -= 1) {
const draft = steps[i];
if (!draft || draft.postStateId) break;
trailing.push(draft);
}
if (trailing.length === 0) return;
let settled: LastSettledObservation;
try {
settled = await captureWithRetry(obs, cdp, tabId, tabsApi);
} catch (err) {
console.warn("[bsk record] final observation at stop failed", err);
return;
}
for (const draft of trailing) draft.postStateId = settled.stateId;
console.debug(
`[bsk record] settled ${trailing.length} trailing step(s) against the page at stop ` +
`(${settled.stateId} ${settled.url})`,
);
}
/**
* Performing a new action ends the previous one: the page the user reached for
* is, by definition, where the earlier step left them. Taking the landing from
* the newer step also keeps the trace monotonic — a capture that finished late
* would otherwise credit an earlier step with a page that only exists because
* of the newer action.
*/
function supersedeEarlierSettles(
obs: RecordingObservationState,
draftIndex: number,
steps: DraftTraceStep[],
): void {
const landing = steps[draftIndex]?.preStateId;
for (const [index, pending] of obs.settles) {
if (index >= draftIndex) continue;
pending.cancelled = true;
obs.settles.delete(index);
const draft = steps[index];
if (!draft || draft.postStateId) continue;
if (!landing) continue;
draft.postStateId = landing;
console.debug(
`[bsk record] step ${index + 1} (${draft.op}) was still settling when step ` +
`${draftIndex + 1} started; landing it on ${landing}`,
);
}
}
export function scheduleDraftSettle(
obs: RecordingObservationState,
draftIndex: number,
stepId: number,
cdp: CdpRunner,
tabId: number,
tabsApi: ChromeTabsApi,
steps: DraftTraceStep[],
): void {
supersedeEarlierSettles(obs, draftIndex, steps);
// Rescheduling the same step (a navigation showed up after the action)
// replaces the pending observation rather than racing it.
const superseded = obs.settles.get(draftIndex);
if (superseded) superseded.cancelled = true;
const pending: PendingSettle = { draftIndex, cancelled: false };
obs.settles.set(draftIndex, pending);
enqueueSettle(obs, async () => {
if (pending.cancelled) return;
try {
await settleDraftObservation(obs, cdp, tabId, tabsApi, steps, draftIndex, pending);
} catch (err) {
// Recoverable: stop-time repair still gives the step a landing page.
const op = steps[draftIndex]?.op ?? "?";
console.warn(`[bsk record] post-action observation failed for step ${stepId} (${op})`, err);
} finally {
if (obs.settles.get(draftIndex) === pending) obs.settles.delete(draftIndex);
}
});
}
function enqueueSettle(obs: RecordingObservationState, task: () => Promise<void>): void {
obs.settleQueue = obs.settleQueue.then(task, task).catch(() => {});
}
export function cancelPendingSettles(obs: RecordingObservationState): void {
for (const pending of obs.settles.values()) pending.cancelled = true;
obs.settles.clear();
}
/** Bound the drain loop so a self-rescheduling settle cannot block stop. */
const MAX_SETTLE_FLUSH_ROUNDS = 10;
/**
* Drain settle work before exporting, including work queued while draining —
* a navigation observed during the last capture schedules another one, and
* the trace would drop it if stop did not wait.
*/
export async function flushPendingSettles(obs: RecordingObservationState): Promise<void> {
for (let round = 0; round < MAX_SETTLE_FLUSH_ROUNDS; round += 1) {
const drained = obs.settleQueue;
await drained;
if (obs.settleQueue === drained) return;
}
console.warn("[bsk record] settle queue kept growing at stop; exporting what has been observed");
}
function finalizeStateBodies(
entries: StateRegistryEntry[],
annotationsByState: Map<string, ObservationAnnotation[]>,
stepIdByDraftId: Map<number, number>,
idByOldId: Map<string, string>,
): TraceState[] {
return entries.map((entry) => {
const id = idByOldId.get(entry.id) ?? entry.id;
const body = formatObservationFile({
stateId: id,
url: entry.url,
title: entry.title,
stepsHere: remapStepIds(entry.stepsHere, stepIdByDraftId),
body: entry.rawVomText,
annotations: annotationsByState.get(entry.id) ?? [],
});
return {
id,
url: entry.url,
...(entry.title ? { title: entry.title } : {}),
body,
...(entry.truncated ? { truncated: true } : {}),
};
});
}
/** Draft ids only become step ids after collapsing and filtering. */
function remapStepIds(draftIds: number[], stepIdByDraftId: Map<number, number>): number[] {
const mapped = new Set<number>();
for (const draftId of draftIds) {
const stepId = stepIdByDraftId.get(draftId);
if (stepId !== undefined) mapped.add(stepId);
}
return [...mapped].sort((a, b) => a - b);
}
function collectAnnotationsByState(
obs: RecordingObservationState,
stepIdByDraftId: Map<number, number>,
): Map<string, ObservationAnnotation[]> {
const byState = new Map<string, ObservationAnnotation[]>();
for (const anns of obs.stepAnnotations.values()) {
for (const ann of anns) {
const stepId = stepIdByDraftId.get(ann.stepId);
if (stepId === undefined) continue;
const bucket = byState.get(ann.stateId) ?? [];
bucket.push({ ...ann, stepId });
byState.set(ann.stateId, bucket);
}
}
return byState;
}
/**
* Keep only the pages the published steps point at. Redirect hops and
* mid-load captures land in the registry too, and shipping them would invite
* a reader to treat a page the flow merely passed through as a real stop.
* With no steps at all the first observation is the whole artifact, so it
* stays.
*/
function selectPublishedStates(
registry: Map<string, StateRegistryEntry>,
steps: Step[],
): StateRegistryEntry[] {
const entries = [...registry.values()];
if (steps.length === 0) return entries.slice(0, 1);
const referenced = new Set<string>();
for (const step of steps) {
referenced.add(step.state);
referenced.add(step.result.state);
}
return entries.filter((entry) => referenced.has(entry.id));
}
export function buildTraceV3(input: {
obs: RecordingObservationState;
steps: DraftTraceStep[];
startedAt: string;
purpose?: string;
startUrl?: string;
stoppedBy: StopReason;
bskVersion: string;
}): Trace {
const { steps, stepIdByDraftId } = reduceTraceSteps(input.steps, input.obs.stateRegistry);
const published = selectPublishedStates(input.obs.stateRegistry, steps);
// Renumber so the shipped dictionary reads s1..sN without holes where the
// dropped captures used to be.
const idByOldId = new Map(published.map((entry, index) => [entry.id, `s${index + 1}`]));
for (const step of steps) {
step.state = idByOldId.get(step.state) ?? step.state;
step.result.state = idByOldId.get(step.result.state) ?? step.result.state;
}
const annotationsByState = collectAnnotationsByState(input.obs, stepIdByDraftId);
const states = finalizeStateBodies(published, annotationsByState, stepIdByDraftId, idByOldId);
const startUrl = resolveTraceStartUrl(input.steps, input.startUrl, states);
return {
version: 3,
...(input.purpose ? { purpose: input.purpose } : {}),
recorded_at: new Date().toISOString(),
started_at: input.startedAt,
stopped_by: input.stoppedBy,
entry: { start_url: startUrl },
recorder: { bsk: input.bskVersion, vom: 1 },
states,
steps,
};
}
@@ -0,0 +1,212 @@
import type { DraftTraceStep, NavigationCause } from "@/transport/types";
import type { RecordStepPayload } from "./record-bridge";
import { mapNavigationCause } from "./record-constants";
export interface RecordingStepBuffer {
steps: DraftTraceStep[];
currentUrl?: string;
pendingNavigation: boolean;
pendingNavigationDeadline?: number;
lastTransitionType?: string;
lastTransitionQualifiers?: string[];
}
const NAVIGATION_TRIGGER_WINDOW_MS = 3_000;
function toDraftStep(payload: RecordStepPayload): DraftTraceStep | null {
const pageUrl = payload.page_url;
const geometry = payload.geometry;
switch (payload.op) {
case "click":
return payload.target
? {
op: "click",
target: { unmatched: true },
captureTarget: payload.target,
...(geometry ? { geometry } : {}),
...(pageUrl ? { page_url: pageUrl } : {}),
}
: null;
case "hover":
return payload.target
? {
op: "hover",
target: { unmatched: true },
captureTarget: payload.target,
...(geometry ? { geometry } : {}),
...(pageUrl ? { page_url: pageUrl } : {}),
}
: null;
case "fill":
return payload.target
? {
op: "fill",
target: { unmatched: true },
captureTarget: payload.target,
...(geometry ? { geometry } : {}),
value: payload.value ?? "",
...(payload.commit ? { commit: payload.commit } : {}),
...(payload.redacted ? { redacted: true } : {}),
...(pageUrl ? { page_url: pageUrl } : {}),
}
: null;
case "press":
return payload.key
? {
op: "press",
key: payload.key,
...(payload.target
? { target: { unmatched: true }, captureTarget: payload.target }
: {}),
...(geometry ? { geometry } : {}),
...(payload.modifiers?.length ? { modifiers: payload.modifiers } : {}),
...(pageUrl ? { page_url: pageUrl } : {}),
}
: null;
case "select":
return payload.target && payload.values
? {
op: "select",
target: { unmatched: true },
captureTarget: payload.target,
...(geometry ? { geometry } : {}),
values: payload.values,
...(payload.labels?.length ? { labels: payload.labels } : {}),
...(pageUrl ? { page_url: pageUrl } : {}),
}
: null;
case "scroll":
return { op: "scroll", ...(pageUrl ? { page_url: pageUrl } : {}) };
case "navigate":
return null;
}
}
/** Annotate the latest action with `navigated_to`. Returns its index, or -1. */
function annotateLastStepNavigation(buffer: RecordingStepBuffer, url: string): number {
for (let i = buffer.steps.length - 1; i >= 0; i -= 1) {
const step = buffer.steps[i];
if (!step) continue;
if (step.op === "click" || step.op === "press" || step.op === "select" || step.op === "fill") {
// Mutate in place: an in-flight settle holds this object, and replacing
// it would strand that settle's observation on the discarded copy.
step.navigated_to = url;
return i;
}
break;
}
return -1;
}
function causeFromBuffer(buffer: RecordingStepBuffer): NavigationCause | null {
return mapNavigationCause({
transitionType: buffer.lastTransitionType,
transitionQualifiers: buffer.lastTransitionQualifiers,
});
}
/** Outcome of folding a main-frame URL change into the draft buffer. */
export type NavigationObserveResult =
| { kind: "noop" }
| { kind: "annotated"; index: number }
| { kind: "appended"; index: number }
/**
* Redirect hop: do not emit a step yet. The recorder coalesces the chain and
* emits one `navigate` after `waitForPageSettled` on the final URL.
*/
| { kind: "coalesce_redirect"; url: string };
export function observeRecordedNavigation(
buffer: RecordingStepBuffer,
url: string,
causedByAction?: boolean,
transitionType?: string,
transitionQualifiers?: string[],
): NavigationObserveResult {
if (!url || url === buffer.currentUrl) return { kind: "noop" };
buffer.currentUrl = url;
if (transitionType !== undefined) buffer.lastTransitionType = transitionType;
if (transitionQualifiers !== undefined) {
buffer.lastTransitionQualifiers = transitionQualifiers;
}
const pendingIsCurrent =
buffer.pendingNavigation &&
(buffer.pendingNavigationDeadline === undefined ||
buffer.pendingNavigationDeadline >= Date.now());
if (causedByAction === true || (causedByAction === undefined && pendingIsCurrent)) {
buffer.pendingNavigation = false;
buffer.pendingNavigationDeadline = undefined;
const annotatedIndex = annotateLastStepNavigation(buffer, url);
if (annotatedIndex >= 0) {
return { kind: "annotated", index: annotatedIndex };
}
const cause = causeFromBuffer(buffer);
if (cause === null) {
return { kind: "coalesce_redirect", url };
}
buffer.steps.push({
op: "navigate",
url,
page_url: url,
cause,
transitionType: buffer.lastTransitionType,
transitionQualifiers: buffer.lastTransitionQualifiers,
});
return { kind: "appended", index: buffer.steps.length - 1 };
}
buffer.pendingNavigation = false;
buffer.pendingNavigationDeadline = undefined;
const cause = causeFromBuffer(buffer);
if (cause === null) {
return { kind: "coalesce_redirect", url };
}
buffer.steps.push({
op: "navigate",
url,
page_url: url,
cause,
transitionType: buffer.lastTransitionType,
transitionQualifiers: buffer.lastTransitionQualifiers,
});
return { kind: "appended", index: buffer.steps.length - 1 };
}
/**
* Buffer one captured payload. Returns the index of the draft it appended, or
* `null` when the payload produced none (an unnamed click target, a navigation
* that only annotated the previous draft, …). Callers must not fall back to
* "the last draft" on `null`: that would re-match an already recorded step
* against a newer observation and overwrite its origin state.
*/
export function appendRecordedPayload(
buffer: RecordingStepBuffer,
payload: RecordStepPayload,
): number | null {
if (payload.op === "navigate") {
if (!payload.url) return null;
const result = observeRecordedNavigation(
buffer,
payload.url,
payload.navigation_caused_by_action,
payload.transitionType,
payload.transitionQualifiers,
);
return result.kind === "appended" ? result.index : null;
}
const step = toDraftStep({
...payload,
page_url: payload.page_url ?? buffer.currentUrl,
});
if (!step) return null;
buffer.steps.push(step);
if (step.op === "click" || step.op === "press" || step.op === "select" || step.op === "fill") {
buffer.pendingNavigation = payload.expects_navigation === true;
buffer.pendingNavigationDeadline = buffer.pendingNavigation
? Date.now() + NAVIGATION_TRIGGER_WINDOW_MS
: undefined;
}
return buffer.steps.length - 1;
}
@@ -1,39 +0,0 @@
import {
RECORD_DOCUMENT_ATTRIBUTE,
recordingDocumentMarkerValue,
} from "@/shared/recording-document-identity";
export interface RecordingDocumentMarker {
restore(): void;
ensure(): void;
}
export function waitForDocumentElement(): Promise<HTMLElement> {
if (document.documentElement) return Promise.resolve(document.documentElement);
return new Promise((resolve) => {
const observer = new MutationObserver(() => {
if (!document.documentElement) return;
observer.disconnect();
resolve(document.documentElement);
});
observer.observe(document, { childList: true });
});
}
export function markRecordingDocument(
producerId: string,
root: HTMLElement = document.documentElement,
): RecordingDocumentMarker {
const previous = root.getAttribute(RECORD_DOCUMENT_ATTRIBUTE);
const markerValue = recordingDocumentMarkerValue(producerId);
const ensure = () => root.setAttribute(RECORD_DOCUMENT_ATTRIBUTE, markerValue);
ensure();
return {
ensure,
restore() {
if (root.getAttribute(RECORD_DOCUMENT_ATTRIBUTE) !== markerValue) return;
if (previous === null) root.removeAttribute(RECORD_DOCUMENT_ATTRIBUTE);
else root.setAttribute(RECORD_DOCUMENT_ATTRIBUTE, previous);
},
};
}
@@ -1,134 +0,0 @@
import type { CdpTarget } from "@/browser-driver/frame-graph";
import { type CdpRunner, sendToCdpTarget } from "@/tools/shared";
const SETTLE_MIN_MS = 150;
const SETTLE_QUIET_MS = 250;
const SETTLE_MAX_MS = 2_000;
const SETTLE_POLL_MS = 60;
const SETTLE_WORLD_NAME = "__bsk_record_settle__";
const QUIET_PROBE = `(() => {
const scope = window;
let probe = scope.__bskRecordQuiet;
if (!probe) {
probe = { changedAt: Date.now() };
const observer = new MutationObserver(() => {
probe.changedAt = Date.now();
});
observer.observe(document.documentElement, {
subtree: true,
childList: true,
attributes: true,
characterData: true,
});
scope.__bskRecordQuiet = probe;
}
return { idleMs: Date.now() - probe.changedAt, readyState: document.readyState };
})()`;
export interface DocumentSettleScope {
target: CdpTarget;
/** Omit only for the target's root document. */
frameId?: string;
}
type SettleOutcome = "quiet" | "timeout" | "cancelled";
interface QuietProbe {
idleMs: number;
readyState: string;
}
interface ProbeContext {
executionContextId?: number;
}
function sleep(ms: number, signal?: AbortSignal): Promise<void> {
return new Promise((resolve) => {
if (signal?.aborted) return resolve();
const timer = setTimeout(resolve, ms);
signal?.addEventListener(
"abort",
() => {
clearTimeout(timer);
resolve();
},
{ once: true },
);
});
}
async function executionContextForScope(
cdp: CdpRunner,
scope: DocumentSettleScope,
): Promise<number | undefined> {
if (!scope.frameId) return undefined;
const result = await sendToCdpTarget<{ executionContextId?: number }>(
cdp,
scope.target,
"Page.createIsolatedWorld",
{
frameId: scope.frameId,
worldName: SETTLE_WORLD_NAME,
grantUniveralAccess: false,
},
);
return result.executionContextId;
}
async function readQuietProbe(
cdp: CdpRunner,
scope: DocumentSettleScope,
context: ProbeContext,
): Promise<QuietProbe | null> {
try {
if (scope.frameId && context.executionContextId === undefined) {
context.executionContextId = await executionContextForScope(cdp, scope);
if (context.executionContextId === undefined) return null;
}
const reply = await sendToCdpTarget<{ result?: { value?: unknown } }>(
cdp,
scope.target,
"Runtime.evaluate",
{
expression: QUIET_PROBE,
returnByValue: true,
...(context.executionContextId !== undefined
? { contextId: context.executionContextId }
: {}),
},
);
const value = reply.result?.value;
if (!value || typeof value !== "object") return null;
const { idleMs, readyState } = value as { idleMs?: unknown; readyState?: unknown };
if (typeof idleMs !== "number" || typeof readyState !== "string") return null;
return { idleMs, readyState };
} catch {
context.executionContextId = undefined;
return null;
}
}
export async function waitForDocumentSettled(
cdp: CdpRunner,
scope: DocumentSettleScope,
options: { signal?: AbortSignal } = {},
): Promise<SettleOutcome> {
const startedAt = Date.now();
const floor = startedAt + SETTLE_MIN_MS;
const deadline = startedAt + SETTLE_MAX_MS;
const context: ProbeContext = {};
for (;;) {
if (options.signal?.aborted) return "cancelled";
await sleep(SETTLE_POLL_MS, options.signal);
if (options.signal?.aborted) return "cancelled";
const probe = await readQuietProbe(cdp, scope, context);
const now = Date.now();
if (now < floor) continue;
if (now >= deadline) return "timeout";
if (!probe || probe.readyState === "loading") continue;
if (probe.idleMs >= SETTLE_QUIET_MS) return "quiet";
}
}
@@ -1,34 +0,0 @@
import type { RecordingDraftStep } from "./types";
const CLIPBOARD_KEYS = new Set(["a", "c", "v", "x", "A", "C", "V", "X"]);
const MODIFIER_ONLY_KEYS = new Set(["Meta", "Control", "Alt", "Shift", "OS", "Hyper", "Super"]);
export function shouldRecordPress(
key: string,
modifiers?: Array<"alt" | "ctrl" | "meta" | "shift">,
): boolean {
if (MODIFIER_ONLY_KEYS.has(key)) return false;
const mods = modifiers ?? [];
const hasCtrlOrMeta = mods.includes("ctrl") || mods.includes("meta");
if (hasCtrlOrMeta && CLIPBOARD_KEYS.has(key)) return false;
if (key === "Enter" || key === "Escape") return true;
if (key.length === 1 && !hasCtrlOrMeta && !mods.includes("alt")) return false;
return false;
}
export function shouldIncludeDraft(step: RecordingDraftStep): boolean {
return step.op !== "press" || shouldRecordPress(step.key, step.modifiers);
}
export function resolveDraftStartUrl(
drafts: RecordingDraftStep[],
explicitStartUrl?: string,
fallbackUrl?: string,
): string {
if (explicitStartUrl) return explicitStartUrl;
const navigation = drafts.find(
(step): step is Extract<RecordingDraftStep, { op: "navigate" }> => step.op === "navigate",
);
if (navigation) return navigation.url;
return drafts.find((step) => step.pageUrl)?.pageUrl ?? fallbackUrl ?? "about:blank";
}
@@ -1,65 +0,0 @@
export const RECORD_FRAME_PORT = "bsk-record-frame";
export const RECORD_FRAME_QUERY = "bsk-record-frame-query";
export const RECORD_FRAME_START = "bsk-record-frame-start";
export interface RecordFrameQueryMessage {
type: typeof RECORD_FRAME_QUERY;
}
export interface RecordFrameQueryResponse {
active: boolean;
requestId?: string;
startedAtMs?: number;
}
export interface RecordFrameStartMessage {
type: typeof RECORD_FRAME_START;
requestId: string;
startedAtMs: number;
}
export type RecordFramePortMessage =
| {
type: "ready";
requestId: string;
producerId: string;
}
| {
type: "ready_ack";
requestId: string;
producerId: string;
}
| {
type: "stop";
requestId: string;
commandId: string;
}
| {
type: "cancel";
requestId: string;
}
| {
type: "stopped";
requestId: string;
commandId: string;
ok: boolean;
error?: string;
};
export function isRecordFrameQueryMessage(value: unknown): value is RecordFrameQueryMessage {
return (
typeof value === "object" &&
value !== null &&
(value as { type?: unknown }).type === RECORD_FRAME_QUERY
);
}
export function isRecordFrameStartMessage(value: unknown): value is RecordFrameStartMessage {
if (typeof value !== "object" || value === null) return false;
const message = value as Partial<RecordFrameStartMessage>;
return (
message.type === RECORD_FRAME_START &&
typeof message.requestId === "string" &&
typeof message.startedAtMs === "number"
);
}
@@ -1,399 +0,0 @@
import {
isRecordFrameQueryMessage,
RECORD_FRAME_PORT,
RECORD_FRAME_START,
type RecordFramePortMessage,
type RecordFrameQueryResponse,
type RecordFrameStartMessage,
} from "./frame-bridge";
export interface RecordingCaptureScope {
tabId: number;
documentId: string;
browserFrameId: number;
producerId: string;
}
interface ArmedRecording {
requestId: string;
startedAtMs: number;
tabIds: Set<number>;
agents: Map<string, FrameAgent>;
onDocumentReady?: (scope: RecordingCaptureScope) => void | Promise<void>;
finishing: boolean;
}
interface FrameAgent extends RecordingCaptureScope {
requestId: string;
port: chrome.runtime.Port;
stopWaiters: Map<string, (ok: boolean) => void>;
}
interface BrowserFrame {
frameId: number;
documentId?: string;
}
interface BrowserFrameNavigation extends BrowserFrame {
tabId: number;
lifecycle: "committed" | "completed";
}
const RECORD_FRAME_STOP_TIMEOUT_MS = 5_000;
export interface RecordFrameCoordinatorDeps {
getAllFrames(tabId: number): Promise<BrowserFrame[]>;
sendToDocument(
tabId: number,
message: RecordFrameStartMessage,
target: { documentId?: string; frameId?: number },
): Promise<unknown>;
subscribeFrameNavigation?(listener: (frame: BrowserFrameNavigation) => void): () => void;
}
function documentKey(tabId: number, documentId: string): string {
return `${tabId}:${documentId}`;
}
function senderAddress(
sender: chrome.runtime.MessageSender,
): { tabId: number; documentId: string; browserFrameId: number } | null {
const tabId = sender.tab?.id;
const documentId = sender.documentId;
const browserFrameId = sender.frameId;
if (
typeof tabId !== "number" ||
typeof documentId !== "string" ||
typeof browserFrameId !== "number"
) {
return null;
}
return { tabId, documentId, browserFrameId };
}
function defaultDeps(): RecordFrameCoordinatorDeps {
return {
async getAllFrames(tabId) {
return (await chrome.webNavigation.getAllFrames({ tabId })) ?? [];
},
sendToDocument(tabId, message, target) {
return chrome.tabs.sendMessage(tabId, message, target);
},
subscribeFrameNavigation(listener) {
const onCommitted = (details: chrome.webNavigation.WebNavigationTransitionCallbackDetails) =>
listener({ ...details, lifecycle: "committed" });
const onCompleted = (details: chrome.webNavigation.WebNavigationFramedCallbackDetails) =>
listener({ ...details, lifecycle: "completed" });
chrome.webNavigation.onCommitted.addListener(onCommitted);
chrome.webNavigation.onCompleted.addListener(onCompleted);
return () => {
chrome.webNavigation.onCommitted.removeListener(onCommitted);
chrome.webNavigation.onCompleted.removeListener(onCompleted);
};
},
};
}
export class RecordFrameCoordinator {
readonly #deps: RecordFrameCoordinatorDeps;
readonly #recordings = new Map<string, ArmedRecording>();
#attached = false;
#detachFrameNavigation: (() => void) | null = null;
constructor(deps: RecordFrameCoordinatorDeps = defaultDeps()) {
this.#deps = deps;
}
attach(): () => void {
if (this.#attached) return () => {};
this.#attached = true;
chrome.runtime.onConnect.addListener(this.#onConnect);
chrome.runtime.onMessage.addListener(this.#onMessage);
this.#detachFrameNavigation =
this.#deps.subscribeFrameNavigation?.(this.#onFrameNavigation) ?? null;
return () => {
if (!this.#attached) return;
this.#attached = false;
chrome.runtime.onConnect.removeListener(this.#onConnect);
chrome.runtime.onMessage.removeListener(this.#onMessage);
this.#detachFrameNavigation?.();
this.#detachFrameNavigation = null;
for (const recording of this.#recordings.values()) this.#cancelAgents(recording);
this.#recordings.clear();
};
}
begin(
requestId: string,
startedAtMs: number,
initialTabId: number,
onDocumentReady?: (scope: RecordingCaptureScope) => void | Promise<void>,
): void {
const previous = this.#recordings.get(requestId);
if (previous) this.#cancelAgents(previous);
this.#recordings.set(requestId, {
requestId,
startedAtMs,
// Register the initial tab before navigation begins. A newly-created
// child document can query at document_start before armTab's frame
// snapshot sees it, especially when Chromium promotes it to an OOPIF.
tabIds: new Set([initialTabId]),
agents: new Map(),
...(onDocumentReady ? { onDocumentReady } : {}),
finishing: false,
});
}
async armTab(requestId: string, tabId: number): Promise<boolean> {
const recording = this.#recordings.get(requestId);
if (!recording || recording.finishing) return false;
recording.tabIds.add(tabId);
let frames: BrowserFrame[];
try {
frames = await this.#deps.getAllFrames(tabId);
} catch {
frames = [{ frameId: 0 }];
}
if (!frames.some((frame) => frame.frameId === 0)) frames.unshift({ frameId: 0 });
const results = await Promise.all(
frames.map(async (frame) => {
const started = await this.#startFrame(recording, tabId, frame);
return { frameId: frame.frameId, started };
}),
);
return results.some((result) => result.frameId === 0 && result.started);
}
sourceFor(
requestId: string,
producerId: string,
sender: chrome.runtime.MessageSender,
): RecordingCaptureScope | null {
const address = senderAddress(sender);
const recording = this.#recordings.get(requestId);
if (!address || !recording) return null;
const agent = recording.agents.get(documentKey(address.tabId, address.documentId));
if (!agent || agent.producerId !== producerId) return null;
return {
tabId: agent.tabId,
documentId: agent.documentId,
browserFrameId: agent.browserFrameId,
producerId: agent.producerId,
};
}
async stop(requestId: string): Promise<boolean> {
const recording = this.#recordings.get(requestId);
if (!recording) return true;
recording.finishing = true;
const commandId = crypto.randomUUID();
const results = await Promise.all(
[...recording.agents.values()].map((agent) => this.#stopAgent(agent, requestId, commandId)),
);
const succeeded = results.every(Boolean);
if (succeeded) this.#recordings.delete(requestId);
else recording.finishing = false;
return succeeded;
}
cancel(requestId: string): void {
const recording = this.#recordings.get(requestId);
if (!recording) return;
this.#cancelAgents(recording);
this.#recordings.delete(requestId);
}
readonly #onMessage = (
message: unknown,
sender: chrome.runtime.MessageSender,
sendResponse: (response: RecordFrameQueryResponse) => void,
): boolean => {
if (!isRecordFrameQueryMessage(message)) return false;
const tabId = sender.tab?.id;
const recording =
typeof tabId === "number"
? [...this.#recordings.values()].find(
(candidate) => !candidate.finishing && candidate.tabIds.has(tabId),
)
: undefined;
sendResponse(
recording
? {
active: true,
requestId: recording.requestId,
startedAtMs: recording.startedAtMs,
}
: { active: false },
);
return false;
};
readonly #onFrameNavigation = (frame: BrowserFrameNavigation): void => {
if (frame.frameId === 0) return;
for (const recording of this.#recordings.values()) {
if (recording.finishing || !recording.tabIds.has(frame.tabId)) continue;
// onCommitted can race document_start, while onCompleted runs after the
// content script is available. Starting twice is safe and closes both
// the early-query and late-SPA-frame gaps.
void (async () => {
const started = await this.#startFrame(recording, frame.tabId, frame);
if (!started || frame.lifecycle !== "completed" || recording.finishing) return;
const agent = frame.documentId
? recording.agents.get(documentKey(frame.tabId, frame.documentId))
: [...recording.agents.values()].find(
(candidate) =>
candidate.tabId === frame.tabId && candidate.browserFrameId === frame.frameId,
);
if (!agent) return;
await recording.onDocumentReady?.({
tabId: agent.tabId,
documentId: agent.documentId,
browserFrameId: agent.browserFrameId,
producerId: agent.producerId,
});
})();
}
};
readonly #onConnect = (port: chrome.runtime.Port): void => {
if (port.name !== RECORD_FRAME_PORT) return;
const address = senderAddress(port.sender ?? {});
if (!address) {
port.disconnect();
return;
}
let registered: FrameAgent | null = null;
port.onMessage.addListener((raw: unknown) => {
const message = raw as Partial<RecordFramePortMessage>;
if (message.type === "ready") {
if (typeof message.requestId !== "string" || typeof message.producerId !== "string") {
port.disconnect();
return;
}
const requestId = message.requestId;
const producerId = message.producerId;
if (registered) {
if (registered.requestId === requestId && registered.producerId === producerId) {
port.postMessage({
type: "ready_ack",
requestId,
producerId,
} satisfies RecordFramePortMessage);
} else {
port.disconnect();
}
return;
}
const recording = this.#recordings.get(requestId);
if (
!recording ||
recording.finishing ||
!recording.tabIds.has(address.tabId) ||
producerId.length === 0
) {
port.disconnect();
return;
}
const agent: FrameAgent = {
...address,
requestId,
producerId,
port,
stopWaiters: new Map(),
};
registered = agent;
const key = documentKey(address.tabId, address.documentId);
const previous = recording.agents.get(key);
if (previous && previous !== agent) previous.port.disconnect();
recording.agents.set(key, agent);
port.postMessage({
type: "ready_ack",
requestId,
producerId,
} satisfies RecordFramePortMessage);
return;
}
if (
message.type === "stopped" &&
registered &&
message.requestId === registered.requestId &&
typeof message.commandId === "string"
) {
registered.stopWaiters.get(message.commandId)?.(message.ok === true);
registered.stopWaiters.delete(message.commandId);
}
});
port.onDisconnect.addListener(() => {
if (!registered) return;
for (const resolve of registered.stopWaiters.values()) resolve(true);
const recording = this.#recordings.get(registered.requestId);
const key = documentKey(registered.tabId, registered.documentId);
if (recording?.agents.get(key) === registered) recording.agents.delete(key);
});
};
#stopAgent(agent: FrameAgent, requestId: string, commandId: string): Promise<boolean> {
return new Promise((resolve) => {
const timer = setTimeout(() => {
agent.stopWaiters.delete(commandId);
resolve(false);
}, RECORD_FRAME_STOP_TIMEOUT_MS);
agent.stopWaiters.set(commandId, (ok) => {
clearTimeout(timer);
resolve(ok);
});
try {
agent.port.postMessage({
type: "stop",
requestId,
commandId,
} satisfies RecordFramePortMessage);
} catch {
clearTimeout(timer);
agent.stopWaiters.delete(commandId);
resolve(true);
}
});
}
async #startFrame(
recording: ArmedRecording,
tabId: number,
frame: BrowserFrame,
): Promise<boolean> {
const message: RecordFrameStartMessage = {
type: RECORD_FRAME_START,
requestId: recording.requestId,
startedAtMs: recording.startedAtMs,
};
try {
const response = await this.#deps.sendToDocument(tabId, message, {
...(frame.documentId ? { documentId: frame.documentId } : { frameId: frame.frameId }),
});
return isStarted(response);
} catch {
return false;
}
}
#cancelAgents(recording: ArmedRecording): void {
for (const agent of recording.agents.values()) {
try {
agent.port.postMessage({
type: "cancel",
requestId: recording.requestId,
} satisfies RecordFramePortMessage);
} catch {
// A destroyed Document has no remaining capture state to cancel.
}
}
}
}
function isStarted(value: unknown): boolean {
return typeof value === "object" && value !== null && (value as { ok?: unknown }).ok === true;
}
export const recordFrameCoordinator = new RecordFrameCoordinator();
@@ -1,5 +0,0 @@
const REDIRECT_QUALIFIERS = new Set(["client_redirect", "server_redirect"]);
export function hasRedirectQualifier(qualifiers?: readonly string[]): boolean {
return (qualifiers ?? []).some((qualifier) => REDIRECT_QUALIFIERS.has(qualifier));
}
@@ -1,140 +0,0 @@
import type { RenderedRef } from "@browser-skill/vom";
import type { CdpTarget } from "@/browser-driver/frame-graph";
import {
type CaptureVomMatchNode,
type CaptureVomObservationResult,
captureVomObservation,
} from "@/tools/capture-vom-observation";
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
export interface IndexedObservationNode {
frameId: string;
geometry: CaptureVomMatchNode;
ref?: RenderedRef;
}
export interface CapturedRecordingObservation {
rootFrameId: string;
index: ObservationNodeIndex;
url: string;
title?: string;
vomText: string;
truncated: boolean;
}
export interface RegisteredObservation {
stateId: string;
rootFrameId: string;
index: ObservationNodeIndex;
url: string;
}
export interface RecordingDocumentScope {
frameId: string;
target: CdpTarget;
}
function nodeKey(frameId: string, backendNodeId: number): string {
return `${frameId}:${backendNodeId}`;
}
function frameTagKey(frameId: string, tag: string): string {
return `${frameId}:${tag.toLowerCase()}`;
}
export class ObservationNodeIndex {
readonly #nodesByFrameTag = new Map<string, IndexedObservationNode[]>();
readonly #refById = new Map<string, RenderedRef>();
readonly #refsByFrame = new Map<string, RenderedRef[]>();
readonly #scopeByProducer = new Map<string, RecordingDocumentScope | null>();
constructor(
input: Pick<CaptureVomObservationResult, "rootFrameId" | "matchNodes" | "refs"> &
Partial<Pick<CaptureVomObservationResult, "frames">>,
) {
const refByNode = new Map<string, RenderedRef>();
for (const ref of input.refs) {
const frameId = ref.frameId ?? input.rootFrameId;
refByNode.set(nodeKey(frameId, ref.backendNodeId), ref);
this.#refById.set(ref.ref, ref);
const frameRefs = this.#refsByFrame.get(frameId) ?? [];
frameRefs.push(ref);
this.#refsByFrame.set(frameId, frameRefs);
}
for (const frame of input.frames ?? []) {
const documentId = frame.recordingDocumentId;
if (!documentId) continue;
this.#scopeByProducer.set(
documentId,
this.#scopeByProducer.has(documentId)
? null
: { frameId: frame.frameId, target: frame.target },
);
}
for (const geometry of input.matchNodes) {
const { frameId } = geometry;
const entry = {
frameId,
geometry,
ref: refByNode.get(nodeKey(frameId, geometry.backendNodeId)),
};
const key = frameTagKey(frameId, geometry.tag);
const bucket = this.#nodesByFrameTag.get(key) ?? [];
bucket.push(entry);
this.#nodesByFrameTag.set(key, bucket);
}
}
candidates(frameId: string, tag: string): readonly IndexedObservationNode[] {
return this.#nodesByFrameTag.get(frameTagKey(frameId, tag)) ?? [];
}
ref(refId: string): RenderedRef | undefined {
return this.#refById.get(refId);
}
refs(frameId: string): readonly RenderedRef[] {
return this.#refsByFrame.get(frameId) ?? [];
}
documentScope(producerId: string): RecordingDocumentScope | undefined {
return this.#scopeByProducer.get(producerId) ?? undefined;
}
}
async function readTabMeta(
tabsApi: ChromeTabsApi,
tabId: number,
): Promise<{ url: string; title?: string }> {
try {
const tab = await tabsApi.get(tabId);
return { url: tab.url ?? "about:blank", title: tab.title };
} catch {
return { url: "about:blank" };
}
}
export async function captureRecordingObservation(input: {
cdp: CdpRunner;
tabsApi: ChromeTabsApi;
tabId: number;
maxTokens: number;
redactValues: boolean;
signal?: AbortSignal;
}): Promise<CapturedRecordingObservation> {
const { url, title } = await readTabMeta(input.tabsApi, input.tabId);
const captured = await captureVomObservation(input.cdp, input.tabId, url, {
maxTokens: input.maxTokens,
redactValues: input.redactValues,
conditionalSurfaceProbe: false,
signal: input.signal,
});
return {
rootFrameId: captured.rootFrameId,
index: new ObservationNodeIndex(captured),
url,
title,
vomText: captured.text,
truncated: captured.truncated,
};
}
@@ -1,128 +0,0 @@
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
import { captureRecordingObservation, type RegisteredObservation } from "./observation-capture";
import { RecordingStateRegistry } from "./state-registry";
import { matchObservationTarget, unmatchedTarget } from "./target-matcher";
import type { RecordingDraftStep, StepAnnotation, TargetedRecordingDraft } from "./types";
const DEFAULT_MAX_PAGE_TOKENS = 3_000;
const MIN_CAPTURE_INTERVAL_MS = 200;
export interface TabObservationCursor {
lastSettled: RegisteredObservation | null;
lastCaptureAt: number;
}
function abortableDelay(ms: number, signal?: AbortSignal): Promise<void> {
return new Promise((resolve) => {
if (signal?.aborted) return resolve();
const timer = setTimeout(resolve, ms);
signal?.addEventListener(
"abort",
() => {
clearTimeout(timer);
resolve();
},
{ once: true },
);
});
}
function isTargeted(draft: RecordingDraftStep): draft is TargetedRecordingDraft {
return (
draft.op === "click" ||
draft.op === "hover" ||
draft.op === "fill" ||
draft.op === "press" ||
draft.op === "select"
);
}
export class RecordingObservationSession {
readonly registry: RecordingStateRegistry;
readonly cursor: TabObservationCursor;
readonly annotations: StepAnnotation[];
readonly #maxTokens: number;
readonly #redactValues: boolean;
constructor(
options: {
registry?: RecordingStateRegistry;
cursor?: TabObservationCursor;
annotations?: StepAnnotation[];
maxTokens?: number;
redactValues?: boolean;
} = {},
) {
this.registry = options.registry ?? new RecordingStateRegistry();
this.cursor = options.cursor ?? { lastSettled: null, lastCaptureAt: 0 };
this.annotations = options.annotations ?? [];
this.#maxTokens = options.maxTokens ?? DEFAULT_MAX_PAGE_TOKENS;
this.#redactValues = options.redactValues ?? false;
}
async capture(
cdp: CdpRunner,
tabsApi: ChromeTabsApi,
tabId: number,
signal?: AbortSignal,
): Promise<RegisteredObservation> {
const waitMs = Math.max(0, MIN_CAPTURE_INTERVAL_MS - (Date.now() - this.cursor.lastCaptureAt));
if (waitMs > 0) await abortableDelay(waitMs, signal);
if (signal?.aborted) throw new DOMException("observation aborted", "AbortError");
const captured = await captureRecordingObservation({
cdp,
tabsApi,
tabId,
maxTokens: this.#maxTokens,
redactValues: this.#redactValues,
signal,
});
const state = this.registry.register({
url: captured.url,
title: captured.title,
vomText: captured.vomText,
truncated: captured.truncated,
});
const observation: RegisteredObservation = {
stateId: state.id,
rootFrameId: captured.rootFrameId,
index: captured.index,
url: captured.url,
};
this.cursor.lastSettled = observation;
this.cursor.lastCaptureAt = Date.now();
return observation;
}
bindDraft(draft: RecordingDraftStep, draftId: number, previousActionPending = false): void {
const observation = this.cursor.lastSettled;
if (isTargeted(draft)) {
draft.matchedTarget = observation
? matchObservationTarget({
observation,
hint: draft.targetHint,
fallback: draft.captureTarget,
})
: unmatchedTarget(draft.captureTarget);
}
if (!observation) return;
const unmatched = isTargeted(draft) && draft.matchedTarget?.unmatched === true;
if (previousActionPending && unmatched) return;
draft.preStateId = observation.stateId;
this.registry.markStep(observation.stateId, draftId);
if (!isTargeted(draft) || !draft.matchedTarget?.ref) return;
const ref = observation.index.ref(draft.matchedTarget.ref);
if (!ref) return;
this.annotations.push({
draftId,
op: draft.op,
line: ref.line,
stateId: observation.stateId,
...(draft.op === "fill" && !this.#redactValues
? { detail: JSON.stringify(draft.value) }
: {}),
});
}
}
@@ -1,277 +0,0 @@
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
import type { StopReason, TraceV3 } from "@/transport/types";
import { type DocumentSettleScope, waitForDocumentSettled } from "./document-settle";
import type { RegisteredObservation } from "./observation-capture";
import { RecordingObservationSession } from "./observation-session";
import { inferMissingPostStates, SettleController } from "./settle-controller";
import { RecordingStateRegistry } from "./state-registry";
import { buildTraceV3 } from "./trace-builder-v3";
import type { RecordingDraftStep, StepAnnotation } from "./types";
interface TabRecordingContext {
session: RecordingObservationSession;
settle: SettleController;
pendingCapture: PendingCapture | null;
frameRefresh: Promise<void>;
frameRefreshAbort: AbortController | null;
}
interface PendingCapture {
promise: Promise<RegisteredObservation>;
abort: AbortController;
}
export class RecordingObservationRuntime {
readonly #cdp: CdpRunner;
readonly #tabsApi: ChromeTabsApi;
readonly #registry = new RecordingStateRegistry();
readonly #annotations: StepAnnotation[] = [];
readonly #contexts = new Map<number, TabRecordingContext>();
readonly #maxTokens?: number;
readonly #redactValues: boolean;
constructor(input: {
cdp: CdpRunner;
tabsApi: ChromeTabsApi;
maxTokens?: number;
redactValues?: boolean;
}) {
this.#cdp = input.cdp;
this.#tabsApi = input.tabsApi;
this.#maxTokens = input.maxTokens;
this.#redactValues = input.redactValues ?? false;
}
#context(tabId: number): TabRecordingContext {
const existing = this.#contexts.get(tabId);
if (existing) return existing;
const session = new RecordingObservationSession({
registry: this.#registry,
annotations: this.#annotations,
maxTokens: this.#maxTokens,
redactValues: this.#redactValues,
});
const context: TabRecordingContext = {
session,
settle: new SettleController({
session,
cdp: this.#cdp,
tabsApi: this.#tabsApi,
tabId,
}),
pendingCapture: null,
frameRefresh: Promise.resolve(),
frameRefreshAbort: null,
};
this.#contexts.set(tabId, context);
return context;
}
async #capture(tabId: number): Promise<RegisteredObservation> {
const context = this.#context(tabId);
if (context.pendingCapture) return context.pendingCapture.promise;
const abort = new AbortController();
let pending!: PendingCapture;
const promise = context.session.capture(this.#cdp, this.#tabsApi, tabId, abort.signal);
pending = {
abort,
promise: promise.finally(() => {
if (context.pendingCapture === pending) context.pendingCapture = null;
}),
};
context.pendingCapture = pending;
return pending.promise;
}
async captureInitial(tabId: number): Promise<void> {
const context = this.#context(tabId);
if (context.session.cursor.lastSettled) return;
await this.#capture(tabId);
}
/** Refresh the safe observation after a late child Document becomes usable. */
async refreshDocument(tabId: number, producerId: string): Promise<void> {
const context = this.#context(tabId);
context.frameRefreshAbort?.abort();
const abort = new AbortController();
context.frameRefreshAbort = abort;
const previous = context.frameRefresh;
const refresh = (async () => {
await previous;
if (abort.signal.aborted) return;
if (context.pendingCapture) await Promise.allSettled([context.pendingCapture.promise]);
if (abort.signal.aborted) return;
// The first capture discovers the safe producer -> CDP Document scope.
await this.#capture(tabId);
if (abort.signal.aborted) return;
const scope = context.session.cursor.lastSettled?.index.documentScope(producerId);
if (scope) {
await waitForDocumentSettled(this.#cdp, scope, { signal: abort.signal });
}
if (abort.signal.aborted) return;
// Capture again after the child SPA is quiet so the next action can be
// matched against refs that actually exist in its pre-action state.
await this.#capture(tabId);
})();
const tracked = refresh.catch(() => {});
context.frameRefresh = tracked;
try {
await refresh;
} finally {
if (context.frameRefresh === tracked) context.frameRefreshAbort = null;
}
}
async captureTabTransition(
fromTabId: number,
toTabId: number,
): Promise<{ preStateId?: string; postStateId?: string; targetUrl?: string }> {
const from = this.#context(fromTabId);
await this.#flushContext(from);
if (!from.session.cursor.lastSettled) {
try {
await this.captureInitial(fromTabId);
} catch {
// A target-tab observation can still preserve the forward transition.
}
}
const to = this.#context(toTabId);
let target = to.session.cursor.lastSettled;
try {
target = await this.#capture(toTabId);
} catch {
// Reuse the last settled target state when a refresh races navigation.
}
const source = from.session.cursor.lastSettled;
return {
...(source ? { preStateId: source.stateId } : {}),
...(target ? { postStateId: target.stateId, targetUrl: target.url } : {}),
};
}
bindTabTransition(
draft: Extract<RecordingDraftStep, { op: "switch_tab" }>,
draftId: number,
): void {
if (draft.preStateId) this.#registry.markStep(draft.preStateId, draftId);
}
async processDraft(
tabId: number,
drafts: RecordingDraftStep[],
draftIndex: number,
producerId?: string,
): Promise<void> {
const draft = drafts[draftIndex];
if (!draft) return;
const context = this.#context(tabId);
if (!context.session.cursor.lastSettled) {
try {
await this.captureInitial(tabId);
} catch {
// Post-action settle can still provide a usable state.
}
}
const scope = producerId
? context.session.cursor.lastSettled?.index.documentScope(producerId)
: undefined;
if (
producerId &&
(draft.op === "click" ||
draft.op === "hover" ||
draft.op === "fill" ||
draft.op === "press" ||
draft.op === "select")
) {
draft.targetHint = {
...(draft.targetHint ?? {}),
frameId: scope?.frameId ?? null,
geometrySpace: "local",
};
}
context.session.bindDraft(draft, draftIndex + 1, context.settle.hasPending);
context.settle.schedule(drafts, draftIndex, scope);
}
scheduleSettle(
tabId: number,
drafts: RecordingDraftStep[],
draftIndex: number,
scope?: DocumentSettleScope,
): void {
this.#context(tabId).settle.schedule(drafts, draftIndex, scope);
}
clearRedirect(tabId: number): void {
this.#contexts.get(tabId)?.settle.clearRedirect();
}
scheduleRedirect(tabId: number, drafts: RecordingDraftStep[], url: string): void {
this.#context(tabId).settle.scheduleRedirect(drafts, url);
}
async #flushContext(context: TabRecordingContext): Promise<void> {
await context.frameRefresh;
if (context.pendingCapture) await Promise.allSettled([context.pendingCapture.promise]);
await context.settle.flushRedirects();
await context.settle.flush();
}
async flushRedirects(tabId: number): Promise<void> {
await this.#contexts.get(tabId)?.settle.flushRedirects();
}
async flush(): Promise<void> {
for (const context of this.#contexts.values()) await this.#flushContext(context);
}
async settleTrailing(tabId: number, drafts: RecordingDraftStep[]): Promise<void> {
const context = this.#context(tabId);
if (context.pendingCapture) await Promise.allSettled([context.pendingCapture.promise]);
if (!context.session.cursor.lastSettled) {
try {
await this.captureInitial(tabId);
} catch {
// The trace may remain observation-free when CDP is unavailable.
}
}
await context.settle.settleTrailing(drafts);
inferMissingPostStates(drafts);
}
cancel(): void {
for (const context of this.#contexts.values()) {
context.frameRefreshAbort?.abort();
context.pendingCapture?.abort.abort();
context.settle.cancel();
}
}
buildTrace(input: {
drafts: RecordingDraftStep[];
startedAt: string;
purpose?: string;
startUrl?: string;
stoppedBy: StopReason;
bskVersion: string;
includeTabSwitches?: boolean;
}): TraceV3 {
return buildTraceV3({
registry: this.#registry,
drafts: input.drafts,
annotations: this.#annotations,
startedAt: input.startedAt,
purpose: input.purpose,
startUrl: input.startUrl,
stoppedBy: input.stoppedBy,
bskVersion: input.bskVersion,
redactValues: this.#redactValues,
includeTabSwitches: input.includeTabSwitches,
});
}
}
@@ -1,233 +0,0 @@
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
import { type DocumentSettleScope, waitForDocumentSettled } from "./document-settle";
import { RecordingObservationSession } from "./observation-session";
import type { RecordingDraftStep } from "./types";
interface PendingSettle {
abort: AbortController;
scope: DocumentSettleScope;
}
interface PendingRedirect {
url: string;
abort: AbortController;
}
const CAPTURE_RETRY_DELAY_MS = 250;
function delay(ms: number, signal?: AbortSignal): Promise<void> {
return new Promise((resolve) => {
if (signal?.aborted) return resolve();
const timer = setTimeout(resolve, ms);
signal?.addEventListener(
"abort",
() => {
clearTimeout(timer);
resolve();
},
{ once: true },
);
});
}
function isAbortError(error: unknown): boolean {
return (
typeof error === "object" &&
error !== null &&
(error as { name?: string }).name === "AbortError"
);
}
export function inferMissingPostStates(drafts: RecordingDraftStep[]): void {
for (let index = 0; index < drafts.length - 1; index += 1) {
const draft = drafts[index];
const next = drafts[index + 1];
if (draft && next && !draft.postStateId && next.preStateId) {
draft.postStateId = next.preStateId;
}
}
}
export class SettleController {
readonly #session: RecordingObservationSession;
readonly #cdp: CdpRunner;
readonly #tabsApi: ChromeTabsApi;
readonly #tabId: number;
readonly #rootScope: DocumentSettleScope;
readonly #pending = new Map<number, PendingSettle>();
#queue = Promise.resolve();
#pendingRedirect: PendingRedirect | null = null;
#redirectQueue = Promise.resolve();
constructor(input: {
session: RecordingObservationSession;
cdp: CdpRunner;
tabsApi: ChromeTabsApi;
tabId: number;
}) {
this.#session = input.session;
this.#cdp = input.cdp;
this.#tabsApi = input.tabsApi;
this.#tabId = input.tabId;
this.#rootScope = { target: { tabId: input.tabId } };
}
get hasPending(): boolean {
return this.#pending.size > 0 || this.#pendingRedirect !== null;
}
async #captureWithRetry(signal?: AbortSignal) {
try {
return await this.#session.capture(this.#cdp, this.#tabsApi, this.#tabId, signal);
} catch (error) {
if (signal?.aborted) throw error;
await delay(CAPTURE_RETRY_DELAY_MS, signal);
return this.#session.capture(this.#cdp, this.#tabsApi, this.#tabId, signal);
}
}
schedule(
drafts: RecordingDraftStep[],
draftIndex: number,
scope: DocumentSettleScope = this.#rootScope,
): void {
if (scope.target.tabId !== this.#tabId) {
throw new Error(
`settle scope tab ${scope.target.tabId} does not belong to tab ${this.#tabId}`,
);
}
const landing = drafts[draftIndex]?.preStateId;
for (const [index, pending] of this.#pending) {
if (index >= draftIndex) continue;
pending.abort.abort();
this.#pending.delete(index);
if (landing && drafts[index] && !drafts[index].postStateId) {
drafts[index].postStateId = landing;
}
}
this.#pending.get(draftIndex)?.abort.abort();
const pending = { abort: new AbortController(), scope };
this.#pending.set(draftIndex, pending);
const task = async () => {
if (pending.abort.signal.aborted) return;
try {
const outcome = await waitForDocumentSettled(this.#cdp, pending.scope, {
signal: pending.abort.signal,
});
if (outcome === "cancelled") return;
const observation = await this.#captureWithRetry(pending.abort.signal);
if (!pending.abort.signal.aborted && drafts[draftIndex]) {
drafts[draftIndex].postStateId = observation.stateId;
}
} catch (error) {
if (!isAbortError(error)) {
console.warn(
`[bsk record] post-action observation failed for step ${draftIndex + 1}`,
error,
);
}
} finally {
if (this.#pending.get(draftIndex) === pending) this.#pending.delete(draftIndex);
}
};
this.#queue = this.#queue.then(task, task).catch(() => {});
}
cancel(): void {
for (const pending of this.#pending.values()) pending.abort.abort();
this.#pending.clear();
this.clearRedirect();
}
async flush(): Promise<void> {
for (let round = 0; round < 10; round += 1) {
const current = this.#queue;
await current;
if (current === this.#queue) return;
}
console.warn("[bsk record] settle queue kept growing; using completed observations");
}
async settleTrailing(drafts: RecordingDraftStep[]): Promise<void> {
const trailing: RecordingDraftStep[] = [];
for (let index = drafts.length - 1; index >= 0; index -= 1) {
const draft = drafts[index];
if (!draft || draft.postStateId) break;
trailing.push(draft);
}
if (trailing.length === 0) return;
try {
const observation = await this.#captureWithRetry();
for (const draft of trailing) draft.postStateId = observation.stateId;
} catch (error) {
console.warn("[bsk record] final observation at stop failed", error);
}
}
clearRedirect(): void {
this.#pendingRedirect?.abort.abort();
this.#pendingRedirect = null;
}
scheduleRedirect(drafts: RecordingDraftStep[], url: string): void {
this.#pendingRedirect?.abort.abort();
const pending: PendingRedirect = { url, abort: new AbortController() };
this.#pendingRedirect = pending;
const task = () => this.#settleRedirect(drafts, pending);
this.#redirectQueue = this.#redirectQueue.then(task, task).catch(() => {});
}
async #settleRedirect(drafts: RecordingDraftStep[], pending: PendingRedirect): Promise<void> {
const outcome = await waitForDocumentSettled(this.#cdp, this.#rootScope, {
signal: pending.abort.signal,
});
if (outcome === "cancelled" || this.#pendingRedirect !== pending) return;
let finalUrl = pending.url;
try {
finalUrl = (await this.#tabsApi.get(this.#tabId)).url || finalUrl;
} catch {
// The navigation event URL remains the best available destination.
}
if (this.#pendingRedirect !== pending) return;
this.#pendingRedirect = null;
if (
!finalUrl ||
finalUrl === "about:blank" ||
this.#session.cursor.lastSettled?.url === finalUrl
) {
return;
}
const last = drafts[drafts.length - 1];
if (last?.op === "navigate" && last.url === finalUrl) {
if (!last.postStateId) this.schedule(drafts, drafts.length - 1);
await this.flush();
return;
}
const draft: RecordingDraftStep = {
op: "navigate",
url: finalUrl,
pageUrl: finalUrl,
cause: "browser",
transitionQualifiers: ["server_redirect"],
preStateId: this.#session.cursor.lastSettled?.stateId,
};
drafts.push(draft);
const draftIndex = drafts.length - 1;
if (draft.preStateId) this.#session.registry.markStep(draft.preStateId, draftIndex + 1);
this.schedule(drafts, draftIndex);
await this.flush();
}
async flushRedirects(): Promise<void> {
for (let round = 0; round < 10; round += 1) {
const current = this.#redirectQueue;
await current;
if (current === this.#redirectQueue) return;
}
console.warn("[bsk record] redirect queue kept growing; using the final completed landing");
}
}
@@ -1,57 +0,0 @@
export interface RecordedStateEntry {
id: string;
url: string;
title?: string;
/** Rendered VOM text; capture indexes and source DOM/AX data never enter the registry. */
vomText: string;
truncated: boolean;
stepsHere: number[];
}
function stateIdentity(url: string, body: string): string {
return `${url}\0${body}`;
}
export class RecordingStateRegistry {
readonly #entriesById = new Map<string, RecordedStateEntry>();
readonly #idByIdentity = new Map<string, string>();
#nextId = 1;
register(input: {
url: string;
title?: string;
vomText: string;
truncated?: boolean;
}): RecordedStateEntry {
const identity = stateIdentity(input.url, input.vomText);
const existingId = this.#idByIdentity.get(identity);
if (existingId) {
const existing = this.#entriesById.get(existingId)!;
if (!existing.title && input.title) existing.title = input.title;
if (input.truncated) existing.truncated = true;
return existing;
}
const entry: RecordedStateEntry = {
id: `s${this.#nextId}`,
url: input.url,
...(input.title ? { title: input.title } : {}),
vomText: input.vomText,
truncated: input.truncated ?? false,
stepsHere: [],
};
this.#nextId += 1;
this.#entriesById.set(entry.id, entry);
this.#idByIdentity.set(identity, entry.id);
return entry;
}
values(): RecordedStateEntry[] {
return [...this.#entriesById.values()];
}
markStep(stateId: string, draftId: number): void {
const entry = this.#entriesById.get(stateId);
if (entry && !entry.stepsHere.includes(draftId)) entry.stepsHere.push(draftId);
}
}
@@ -1,155 +0,0 @@
import type { RecordStepPayload } from "../record-bridge";
import { hasRedirectQualifier } from "./navigation-policy";
import type { RecordingDraftStep, TargetMatchHint } from "./types";
export interface RecordingStepBuffer {
steps: RecordingDraftStep[];
navigation: RecordingNavigationCursor;
}
export interface RecordingNavigationCursor {
currentUrl?: string;
pendingNavigation: boolean;
pendingNavigationDeadline?: number;
}
const NAVIGATION_TRIGGER_WINDOW_MS = 3_000;
function toDraftStep(
payload: RecordStepPayload,
targetHint?: TargetMatchHint,
): RecordingDraftStep | null {
const pageUrl = payload.page_url;
const common = {
...(pageUrl ? { pageUrl } : {}),
...(targetHint ? { targetHint } : {}),
};
switch (payload.op) {
case "click":
return payload.target ? { op: "click", captureTarget: payload.target, ...common } : null;
case "hover":
return payload.target ? { op: "hover", captureTarget: payload.target, ...common } : null;
case "fill":
return payload.target
? {
op: "fill",
captureTarget: payload.target,
value: payload.value ?? "",
...(payload.commit ? { commit: payload.commit } : {}),
...(payload.redacted ? { redacted: true } : {}),
...common,
}
: null;
case "press":
return payload.key
? {
op: "press",
key: payload.key,
...(payload.target ? { captureTarget: payload.target } : {}),
...(payload.modifiers?.length ? { modifiers: payload.modifiers } : {}),
...common,
}
: null;
case "select":
return payload.target && payload.values
? {
op: "select",
captureTarget: payload.target,
values: payload.values,
...(payload.labels?.length ? { labels: payload.labels } : {}),
...common,
}
: null;
case "navigate":
return null;
}
}
function annotateLastStepNavigation(buffer: RecordingStepBuffer, url: string): number {
for (let index = buffer.steps.length - 1; index >= 0; index -= 1) {
const step = buffer.steps[index];
if (!step) continue;
if (step.op === "click" || step.op === "press" || step.op === "select" || step.op === "fill") {
step.navigatedTo = url;
return index;
}
break;
}
return -1;
}
export type NavigationObserveResult =
| { kind: "noop" }
| { kind: "annotated"; index: number }
| { kind: "appended"; index: number }
| { kind: "coalesce_redirect"; url: string };
export function observeRecordedNavigation(
buffer: RecordingStepBuffer,
url: string,
causedByAction?: boolean,
transitionType?: string,
transitionQualifiers?: string[],
): NavigationObserveResult {
const navigation = buffer.navigation;
if (!url || url === navigation.currentUrl) return { kind: "noop" };
navigation.currentUrl = url;
const pendingIsCurrent =
navigation.pendingNavigation &&
(navigation.pendingNavigationDeadline === undefined ||
navigation.pendingNavigationDeadline >= Date.now());
if (causedByAction === true || (causedByAction === undefined && pendingIsCurrent)) {
navigation.pendingNavigation = false;
navigation.pendingNavigationDeadline = undefined;
const annotatedIndex = annotateLastStepNavigation(buffer, url);
if (annotatedIndex >= 0) return { kind: "annotated", index: annotatedIndex };
} else {
navigation.pendingNavigation = false;
navigation.pendingNavigationDeadline = undefined;
}
if (hasRedirectQualifier(transitionQualifiers)) {
return { kind: "coalesce_redirect", url };
}
buffer.steps.push({
op: "navigate",
url,
pageUrl: url,
transitionType,
transitionQualifiers,
});
return { kind: "appended", index: buffer.steps.length - 1 };
}
export function appendRecordedPayload(
buffer: RecordingStepBuffer,
payload: RecordStepPayload,
targetHint?: TargetMatchHint,
): number | null {
if (payload.op === "navigate") {
if (!payload.url) return null;
const result = observeRecordedNavigation(
buffer,
payload.url,
payload.navigation_caused_by_action,
payload.transitionType,
payload.transitionQualifiers,
);
return result.kind === "appended" ? result.index : null;
}
const step = toDraftStep(
{ ...payload, page_url: payload.page_url ?? buffer.navigation.currentUrl },
targetHint,
);
if (!step) return null;
buffer.steps.push(step);
if (step.op === "click" || step.op === "press" || step.op === "select" || step.op === "fill") {
buffer.navigation.pendingNavigation = payload.expects_navigation === true;
buffer.navigation.pendingNavigationDeadline = buffer.navigation.pendingNavigation
? Date.now() + NAVIGATION_TRIGGER_WINDOW_MS
: undefined;
}
return buffer.steps.length - 1;
}
@@ -1,61 +0,0 @@
import type { RecordingNavigationCursor } from "./step-buffer";
export interface TabActivation {
tabId: number;
revision: number;
}
export class RecordingTabCoordinator {
readonly #navigationByTab = new Map<number, RecordingNavigationCursor>();
#activeTabId: number;
#currentTabId: number;
#activationRevision = 0;
constructor(initialTabId: number, initialUrl?: string) {
this.#activeTabId = initialTabId;
this.#currentTabId = initialTabId;
this.#navigationByTab.set(initialTabId, {
currentUrl: initialUrl,
pendingNavigation: false,
});
}
get activeTabId(): number {
return this.#activeTabId;
}
get currentTabId(): number {
return this.#currentTabId;
}
noteActivation(tabId: number): TabActivation {
this.#activeTabId = tabId;
this.#activationRevision += 1;
return { tabId, revision: this.#activationRevision };
}
isLatest(activation: TabActivation): boolean {
return (
activation.tabId === this.#activeTabId && activation.revision === this.#activationRevision
);
}
commit(tabId: number, currentUrl?: string): void {
if (currentUrl !== undefined) this.navigation(tabId).currentUrl = currentUrl;
this.#currentTabId = tabId;
}
navigation(tabId: number, fallbackUrl?: string): RecordingNavigationCursor {
const existing = this.#navigationByTab.get(tabId);
if (existing) {
if (!existing.currentUrl && fallbackUrl) existing.currentUrl = fallbackUrl;
return existing;
}
const created: RecordingNavigationCursor = {
currentUrl: fallbackUrl,
pendingNavigation: false,
};
this.#navigationByTab.set(tabId, created);
return created;
}
}
@@ -1,86 +0,0 @@
import type { RenderedRef } from "@browser-skill/vom";
import type { TargetDescriptorV3 } from "@/transport/types";
import type { CaptureTargetDescriptor } from "../describe-target";
import type { IndexedObservationNode, RegisteredObservation } from "./observation-capture";
import type { TargetGeometry, TargetMatchHint } from "./types";
const MATCH_TOLERANCE_PX = 2;
function close(a: number, b: number): boolean {
return Math.abs(a - b) <= MATCH_TOLERANCE_PX;
}
function rectMatches(a: TargetGeometry["rect"], b: TargetGeometry["rect"]): boolean {
return close(a.x, b.x) && close(a.y, b.y) && close(a.w, b.w) && close(a.h, b.h);
}
function candidateMatches(
candidate: IndexedObservationNode,
geometry: TargetGeometry,
geometrySpace: "top" | "local",
): boolean {
const rect = geometrySpace === "local" ? candidate.geometry.localRect : candidate.geometry.rect;
return rect != null && rectMatches(geometry.rect, rect);
}
export function unmatchedTarget(fallback?: CaptureTargetDescriptor): TargetDescriptorV3 {
return {
...(fallback?.role ? { role: fallback.role } : {}),
...(fallback?.name ? { name: fallback.name } : {}),
unmatched: true,
};
}
function normalized(value: string | undefined): string {
return value?.replace(/\s+/g, " ").trim().toLowerCase() ?? "";
}
function matchesSemantics(ref: RenderedRef, fallback?: CaptureTargetDescriptor): boolean {
const role = normalized(fallback?.role);
const name = normalized(fallback?.name);
if (!role && !name) return false;
if (role && normalized(ref.role) !== role) return false;
if (name && normalized(ref.name) !== name) return false;
return true;
}
function descriptor(ref: RenderedRef): TargetDescriptorV3 {
return {
ref: ref.ref,
...(ref.role ? { role: ref.role } : {}),
...(ref.name ? { name: ref.name } : {}),
...(ref.ctx ? { ctx: ref.ctx } : {}),
};
}
export function matchObservationTarget(input: {
observation: RegisteredObservation;
hint?: TargetMatchHint;
fallback?: CaptureTargetDescriptor;
}): TargetDescriptorV3 {
if (input.hint?.frameId === null) return unmatchedTarget(input.fallback);
const frameId = input.hint?.frameId ?? input.observation.rootFrameId;
const geometry = input.hint?.geometry;
if (geometry) {
const matches = input.observation.index
.candidates(frameId, geometry.tag)
.filter(
(candidate) =>
candidate.ref &&
candidateMatches(candidate, geometry, input.hint?.geometrySpace ?? "top"),
);
if (matches.length === 1) return descriptor(matches[0]!.ref!);
const semanticMatches = matches.filter(
(candidate) => candidate.ref && matchesSemantics(candidate.ref, input.fallback),
);
if (semanticMatches.length === 1) return descriptor(semanticMatches[0]!.ref!);
return unmatchedTarget(input.fallback);
}
const semanticMatches = input.observation.index
.refs(frameId)
.filter((ref) => matchesSemantics(ref, input.fallback));
return semanticMatches.length === 1
? descriptor(semanticMatches[0]!)
: unmatchedTarget(input.fallback);
}
@@ -1,92 +0,0 @@
import {
type StepV3,
type StopReason,
TRACE_VERSION_V3,
type TraceStateV3,
type TraceV3,
VOM_FORMAT_VERSION,
} from "@/transport/types";
import { resolveDraftStartUrl } from "./draft-policy";
import type { RecordedStateEntry, RecordingStateRegistry } from "./state-registry";
import { reduceTraceStepsV3 } from "./trace-reducer-v3";
import { formatTraceStateBody } from "./trace-state-body";
import type { RecordingDraftStep, StepAnnotation } from "./types";
function publishedEntries(registry: RecordingStateRegistry, steps: StepV3[]): RecordedStateEntry[] {
const entries = registry.values();
if (steps.length === 0) return entries.slice(0, 1);
const referenced = new Set(steps.flatMap((step) => [step.state, step.result.state]));
return entries.filter((entry) => referenced.has(entry.id));
}
function remapDraftIds(draftIds: number[], stepIdByDraftId: Map<number, number>): number[] {
return [
...new Set(
draftIds.flatMap((id) => {
const stepId = stepIdByDraftId.get(id);
return stepId === undefined ? [] : [stepId];
}),
),
].sort((a, b) => a - b);
}
export function buildTraceV3(input: {
registry: RecordingStateRegistry;
drafts: RecordingDraftStep[];
annotations?: StepAnnotation[];
startedAt: string;
purpose?: string;
startUrl?: string;
stoppedBy: StopReason;
bskVersion: string;
redactValues?: boolean;
includeTabSwitches?: boolean;
}): TraceV3 {
const reduced = reduceTraceStepsV3(input.drafts, {
includeTabSwitches: input.includeTabSwitches,
redactValues: input.redactValues,
});
const entries = publishedEntries(input.registry, reduced.steps);
const publishedId = new Map(entries.map((entry, index) => [entry.id, `s${index + 1}`]));
const annotationsByState = new Map<string, StepAnnotation[]>();
for (const annotation of input.annotations ?? []) {
const bucket = annotationsByState.get(annotation.stateId) ?? [];
bucket.push(annotation);
annotationsByState.set(annotation.stateId, bucket);
}
const steps = reduced.steps.map((step) => ({
...step,
state: publishedId.get(step.state) ?? step.state,
result: { state: publishedId.get(step.result.state) ?? step.result.state },
}));
const states: TraceStateV3[] = entries.map((entry) => {
const id = publishedId.get(entry.id) ?? entry.id;
return {
id,
url: entry.url,
...(entry.title ? { title: entry.title } : {}),
body: formatTraceStateBody({
stateId: id,
url: entry.url,
title: entry.title,
stepIds: remapDraftIds(entry.stepsHere, reduced.stepIdByDraftId),
vomText: entry.vomText,
annotations: annotationsByState.get(entry.id) ?? [],
stepIdByDraftId: reduced.stepIdByDraftId,
}),
...(entry.truncated ? { truncated: true } : {}),
};
});
return {
version: TRACE_VERSION_V3,
...(input.purpose ? { purpose: input.purpose } : {}),
recorded_at: new Date().toISOString(),
started_at: input.startedAt,
stopped_by: input.stoppedBy,
entry: { start_url: resolveDraftStartUrl(input.drafts, input.startUrl, states[0]?.url) },
recorder: { bsk: input.bskVersion, vom: VOM_FORMAT_VERSION },
states,
steps,
};
}
@@ -1,215 +0,0 @@
import type { PageRefV2, SelectedOptionV2, StepV2, TraceV2 } from "@/transport/types";
import { resolveDraftStartUrl, shouldIncludeDraft } from "./draft-policy";
import type { RecordingDraftStep } from "./types";
/** Collapse consecutive navigations to the last hop. */
function collapseNavigations(steps: RecordingDraftStep[]): RecordingDraftStep[] {
const out: RecordingDraftStep[] = [];
for (const step of steps) {
const prev = out[out.length - 1];
if (step.op === "navigate" && prev?.op === "navigate") {
out[out.length - 1] = step;
continue;
}
out.push(step);
}
return out;
}
function collectUrls(steps: RecordingDraftStep[], startUrl?: string): string[] {
const urls: string[] = [];
if (startUrl) urls.push(startUrl);
for (const step of steps) {
if (step.op === "navigate") {
urls.push(step.url);
continue;
}
if (step.pageUrl) urls.push(step.pageUrl);
if ("navigatedTo" in step && step.navigatedTo) urls.push(step.navigatedTo);
}
const seen = new Set<string>();
const unique: string[] = [];
for (const url of urls) {
if (!url || seen.has(url)) continue;
seen.add(url);
unique.push(url);
}
return unique;
}
function buildPageRegistry(
steps: RecordingDraftStep[],
startUrl?: string,
): { pages: PageRefV2[]; urlToId: Map<string, string> } {
const urls = collectUrls(steps, startUrl);
const urlToId = new Map<string, string>();
const pages = urls.map((url, index) => {
const id = `p${index + 1}`;
urlToId.set(url, id);
return { id, url };
});
return { pages, urlToId };
}
function pageIdFor(
url: string | undefined,
urlToId: Map<string, string>,
fallbackUrl?: string,
): string {
if (url && urlToId.has(url)) return urlToId.get(url)!;
if (fallbackUrl && urlToId.has(fallbackUrl)) return urlToId.get(fallbackUrl)!;
return urlToId.values().next().value ?? "p1";
}
function pageUrlForDraft(step: RecordingDraftStep, fallbackUrl?: string): string | undefined {
if (step.op === "navigate") return step.pageUrl ?? step.url;
if (step.pageUrl) return step.pageUrl;
return fallbackUrl;
}
function effectForNavigation(
navigatedTo: string | undefined,
urlToId: Map<string, string>,
): StepV2["effect"] {
if (!navigatedTo) return undefined;
const pageId = urlToId.get(navigatedTo);
if (!pageId) return undefined;
return { navigated_to: pageId };
}
function withEffect(step: StepV2, effect: StepV2["effect"]): StepV2 {
if (!effect) return step;
return { ...step, effect };
}
function toSelection(values: string[], labels?: string[]): SelectedOptionV2[] {
return values.map((value, index) => ({
value,
...(labels?.[index] ? { label: labels[index] } : {}),
}));
}
function toV2Step(
step: RecordingDraftStep,
id: number,
urlToId: Map<string, string>,
fallbackUrl?: string,
): StepV2 | null {
if (!shouldIncludeDraft(step)) return null;
const pageUrl = pageUrlForDraft(step, fallbackUrl);
const page = pageIdFor(pageUrl, urlToId, fallbackUrl);
switch (step.op) {
case "navigate":
return {
op: "navigate",
id,
page: pageIdFor(step.url, urlToId, fallbackUrl),
to: step.url,
};
case "click":
if (!step.captureTarget) return null;
return withEffect(
{
op: "click",
id,
page,
target: step.captureTarget,
},
effectForNavigation(step.navigatedTo, urlToId),
);
case "fill":
if (!step.captureTarget) return null;
return {
op: "fill",
id,
page,
target: step.captureTarget,
value: step.value,
...(step.redacted ? { redacted: true } : {}),
};
case "press":
return withEffect(
{
op: "press",
id,
page,
key: step.key,
...(step.captureTarget ? { target: step.captureTarget } : {}),
...(step.modifiers?.length ? { modifiers: step.modifiers } : {}),
},
effectForNavigation(step.navigatedTo, urlToId),
);
case "select":
if (!step.captureTarget) return null;
return withEffect(
{
op: "select",
id,
page,
target: step.captureTarget,
selection: toSelection(step.values, step.labels),
},
effectForNavigation(step.navigatedTo, urlToId),
);
// `hover` only exists in Trace v3. Peers that negotiate v2 predate the
// step variant and fail to decode the whole result if we emit it.
case "hover":
case "scroll":
case "switch_tab":
return null;
}
}
interface ReducedTrace {
pages: PageRefV2[];
steps: StepV2[];
}
/**
* Compile capture drafts into record-only trace v2 steps.
* Variable inputs are NOT classified here — executing agents infer that at run time.
*/
function reduceTraceSteps(steps: RecordingDraftStep[], startUrl?: string): ReducedTrace {
const collapsed = collapseNavigations(steps);
const { pages, urlToId } = buildPageRegistry(collapsed, startUrl);
const out: StepV2[] = [];
let id = 1;
let lastUrl = startUrl;
for (const draft of collapsed) {
if (draft.op === "navigate") lastUrl = draft.url;
else if ("navigatedTo" in draft && draft.navigatedTo) lastUrl = draft.navigatedTo;
else if (draft.pageUrl) lastUrl = draft.pageUrl;
const step = toV2Step(draft, id, urlToId, lastUrl);
if (!step) continue;
out.push(step);
id += 1;
}
return { pages, steps: out };
}
function resolveTraceStartUrl(
drafts: RecordingDraftStep[],
startUrl?: string,
pages?: PageRefV2[],
): string {
return resolveDraftStartUrl(drafts, startUrl, pages?.[0]?.url);
}
export function buildTraceV2(input: {
steps: RecordingDraftStep[];
startedAt: string;
startUrl?: string;
purpose?: string;
}): TraceV2 {
const { pages, steps } = reduceTraceSteps(input.steps, input.startUrl);
return {
recorded_at: new Date().toISOString(),
started_at: input.startedAt,
...(input.purpose ? { purpose: input.purpose } : {}),
entry: { start_url: resolveTraceStartUrl(input.steps, input.startUrl, pages) },
pages,
steps,
};
}
@@ -1,154 +0,0 @@
import type { NavigationCause, StepV3 } from "@/transport/types";
import { shouldIncludeDraft } from "./draft-policy";
import { hasRedirectQualifier } from "./navigation-policy";
import { unmatchedTarget } from "./target-matcher";
import type { RecordingDraftStep } from "./types";
interface CollapsedDraft {
draft: RecordingDraftStep;
draftIds: number[];
}
const TRANSITION_CAUSES: Record<string, NavigationCause> = {
typed: "user_typed",
generated: "user_typed",
keyword: "user_typed",
keyword_generated: "user_typed",
link: "link",
form_submit: "form_submit",
reload: "reload",
auto_bookmark: "browser",
start_page: "browser",
};
function isRedirect(step: Extract<RecordingDraftStep, { op: "navigate" }>): boolean {
return hasRedirectQualifier(step.transitionQualifiers);
}
function collapseRedirects(steps: RecordingDraftStep[]): CollapsedDraft[] {
const output: CollapsedDraft[] = [];
steps.forEach((step, index) => {
const previous = output[output.length - 1];
if (step.op === "navigate" && previous?.draft.op === "navigate" && isRedirect(step)) {
previous.draft = {
...previous.draft,
url: step.url,
postStateId: step.postStateId ?? previous.draft.postStateId,
};
previous.draftIds.push(index + 1);
return;
}
output.push({ draft: { ...step }, draftIds: [index + 1] });
});
return output;
}
function navigationCause(step: Extract<RecordingDraftStep, { op: "navigate" }>): NavigationCause {
if (step.cause) return step.cause;
const qualifiers = step.transitionQualifiers ?? [];
if (qualifiers.includes("forward_back")) return "history";
if (qualifiers.includes("from_address_bar")) return "user_typed";
return TRANSITION_CAUSES[step.transitionType ?? ""] ?? "browser";
}
function selection(values: string[], labels?: string[]): Array<{ value: string; label?: string }> {
return values.map((value, index) => ({
value,
...(labels?.[index] ? { label: labels[index] } : {}),
}));
}
interface ReduceDraftOptions {
includeTabSwitches: boolean;
redactValues: boolean;
}
function reduceDraft(
draft: RecordingDraftStep,
id: number,
options: ReduceDraftOptions,
): StepV3 | null {
if (!shouldIncludeDraft(draft)) return null;
if (draft.op === "switch_tab") {
if (!options.includeTabSwitches || !draft.preStateId || !draft.postStateId) return null;
return {
op: "switch_tab",
id,
state: draft.preStateId,
result: { state: draft.postStateId },
};
}
const state = draft.preStateId ?? draft.postStateId;
const resultState = draft.postStateId ?? draft.preStateId;
if (!state || !resultState) return null;
const common = { id, state, result: { state: resultState } };
switch (draft.op) {
case "navigate":
return { op: "navigate", ...common, to: draft.url, cause: navigationCause(draft) };
case "click":
return {
op: "click",
...common,
target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget),
};
case "hover":
return {
op: "hover",
...common,
target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget),
};
case "fill":
const fillIsRedacted = options.redactValues || draft.redacted === true;
return {
op: "fill",
...common,
target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget),
value: fillIsRedacted ? "***" : draft.value,
commit: draft.commit ?? "blur",
...(fillIsRedacted ? { redacted: true } : {}),
};
case "press":
return {
op: "press",
...common,
key: draft.key,
...(draft.captureTarget || draft.matchedTarget
? { target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget) }
: {}),
...(draft.modifiers?.length ? { modifiers: draft.modifiers } : {}),
};
case "select":
return {
op: "select",
...common,
target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget),
...(!options.redactValues ? { selection: selection(draft.values, draft.labels) } : {}),
};
case "scroll":
return { op: "scroll", ...common };
}
}
export interface ReducedTraceV3 {
steps: StepV3[];
stepIdByDraftId: Map<number, number>;
}
export function reduceTraceStepsV3(
steps: RecordingDraftStep[],
options: { includeTabSwitches?: boolean; redactValues?: boolean } = {},
): ReducedTraceV3 {
const output: StepV3[] = [];
const stepIdByDraftId = new Map<number, number>();
for (const { draft, draftIds } of collapseRedirects(steps)) {
const step = reduceDraft(draft, output.length + 1, {
includeTabSwitches: options.includeTabSwitches === true,
redactValues: options.redactValues === true,
});
if (!step) continue;
output.push(step);
for (const draftId of draftIds) stepIdByDraftId.set(draftId, step.id);
}
return { steps: output, stepIdByDraftId };
}
@@ -1,39 +0,0 @@
import type { StepAnnotation } from "./types";
function annotationText(annotation: StepAnnotation, stepId: number): string {
return ` ⟵ step ${stepId}: ${annotation.op}${annotation.detail ? `: ${annotation.detail}` : ""}`;
}
export function formatTraceStateBody(input: {
stateId: string;
url: string;
title?: string;
stepIds: number[];
vomText: string;
annotations: StepAnnotation[];
stepIdByDraftId: Map<number, number>;
}): string {
const lines = ["# bsk-observation 1", `state: ${JSON.stringify(input.stateId)}`];
lines.push(`url: ${JSON.stringify(input.url)}`);
if (input.title) lines.push(`title: ${JSON.stringify(input.title)}`);
if (input.stepIds.length > 0) lines.push(`steps_here: [${input.stepIds.join(", ")}]`);
lines.push("---");
const byLine = new Map<number, Array<{ annotation: StepAnnotation; stepId: number }>>();
for (const annotation of input.annotations) {
const stepId = input.stepIdByDraftId.get(annotation.draftId);
if (stepId === undefined) continue;
const bucket = byLine.get(annotation.line) ?? [];
bucket.push({ annotation, stepId });
byLine.set(annotation.line, bucket);
}
input.vomText.split("\n").forEach((bodyLine, lineIndex) => {
let line = bodyLine;
for (const item of byLine.get(lineIndex) ?? []) {
line += annotationText(item.annotation, item.stepId);
}
lines.push(line);
});
return `${lines.join("\n")}\n`;
}
-85
View File
@@ -1,85 +0,0 @@
import type { CaptureTargetDescriptor } from "@/lib/describe-target";
import type {
FillCommit,
KeyModifier,
NavigationCause,
StepV3,
TargetDescriptorV3,
} from "@/transport/types";
export interface TargetGeometry {
/** Top-level viewport-relative CSS pixels, as defined by the geometry module. */
rect: { x: number; y: number; w: number; h: number };
tag: string;
}
export interface TargetMatchHint {
geometry?: TargetGeometry;
/** Missing means top frame; null means the source Document could not be resolved. */
frameId?: string | null;
geometrySpace?: "top" | "local";
}
export interface StepAnnotation {
draftId: number;
op: StepV3["op"];
line: number;
stateId: string;
detail?: string;
}
interface DraftStateLink {
pageUrl?: string;
preStateId?: string;
postStateId?: string;
}
interface DraftTarget {
captureTarget?: CaptureTargetDescriptor;
targetHint?: TargetMatchHint;
matchedTarget?: TargetDescriptorV3;
}
interface DraftNavigationEffect {
navigatedTo?: string;
}
export type RecordingDraftStep =
| ({ op: "click" } & DraftStateLink & DraftTarget & DraftNavigationEffect)
| ({ op: "hover" } & DraftStateLink & DraftTarget)
| ({
op: "fill";
value: string;
commit?: FillCommit;
redacted?: boolean;
} & DraftStateLink &
DraftTarget &
DraftNavigationEffect)
| ({
op: "press";
key: string;
modifiers?: KeyModifier[];
} & DraftStateLink &
DraftTarget &
DraftNavigationEffect)
| ({
op: "select";
values: string[];
labels?: string[];
} & DraftStateLink &
DraftTarget &
DraftNavigationEffect)
| ({ op: "scroll" } & DraftStateLink)
| ({ op: "switch_tab" } & DraftStateLink)
| ({
op: "navigate";
url: string;
cause?: NavigationCause;
transitionType?: string;
transitionQualifiers?: string[];
} & DraftStateLink);
export type TargetedRecordingDraft = Extract<
RecordingDraftStep,
{ op: "click" | "hover" | "fill" | "press" | "select" }
>;
+297
View File
@@ -0,0 +1,297 @@
import type { CaptureTargetDescriptor } from "@/lib/describe-target";
import type { DraftTraceStep, KeyModifier } from "@/transport/types";
const CLIPBOARD_KEYS = new Set(["a", "c", "v", "x", "A", "C", "V", "X"]);
const MODIFIER_ONLY_KEYS = new Set(["Meta", "Control", "Alt", "Shift", "OS", "Hyper", "Super"]);
export interface TargetDescriptorV2 {
role?: string;
name?: string;
tag: string;
name_attr?: string;
placeholder?: string;
nearby_label?: string;
}
export interface PageRef {
id: string;
url: string;
title?: string;
}
export interface SelectedOptionV2 {
value: string;
label?: string;
}
export interface StepEffectV2 {
navigated_to: string;
}
export type StepV2 =
| { op: "navigate"; id: number; page: string; to: string; effect?: StepEffectV2 }
| { op: "click"; id: number; page: string; target: TargetDescriptorV2; effect?: StepEffectV2 }
| {
op: "fill";
id: number;
page: string;
target: TargetDescriptorV2;
value: string;
redacted?: boolean;
effect?: StepEffectV2;
}
| {
op: "select";
id: number;
page: string;
target: TargetDescriptorV2;
selection: SelectedOptionV2[];
effect?: StepEffectV2;
}
| {
op: "press";
id: number;
page: string;
key: string;
modifiers?: KeyModifier[];
target?: TargetDescriptorV2;
effect?: StepEffectV2;
};
export interface TraceV2 {
recorded_at: string;
started_at?: string;
purpose?: string;
entry: { start_url: string };
pages: PageRef[];
steps: StepV2[];
}
export function shouldRecordPress(
key: string,
modifiers?: Array<"alt" | "ctrl" | "meta" | "shift">,
): boolean {
if (MODIFIER_ONLY_KEYS.has(key)) return false;
const mods = modifiers ?? [];
const hasCtrlOrMeta = mods.includes("ctrl") || mods.includes("meta");
if (hasCtrlOrMeta && CLIPBOARD_KEYS.has(key)) return false;
if (key === "Enter" || key === "Escape") return true;
if (key.length === 1 && !hasCtrlOrMeta && !mods.includes("alt")) return false;
return false;
}
function shouldIncludeDraft(step: DraftTraceStep): boolean {
if (step.op === "scroll" || step.op === "hover") return false;
if (step.op === "fill" && !(step.value ?? "").trim() && !step.redacted) return false;
if (step.op === "press" && !shouldRecordPress(step.key, step.modifiers)) return false;
return true;
}
function collapseNavigations(steps: DraftTraceStep[]): DraftTraceStep[] {
const out: DraftTraceStep[] = [];
for (const step of steps) {
const prev = out[out.length - 1];
if (step.op === "navigate" && prev?.op === "navigate") {
out[out.length - 1] = step;
continue;
}
out.push(step);
}
return out;
}
function collectUrls(steps: DraftTraceStep[], startUrl?: string): string[] {
const urls: string[] = [];
if (startUrl) urls.push(startUrl);
for (const step of steps) {
if (step.op === "navigate") {
urls.push(step.url);
continue;
}
if ("page_url" in step && step.page_url) urls.push(step.page_url);
if ("navigated_to" in step && step.navigated_to) urls.push(step.navigated_to);
}
const seen = new Set<string>();
const unique: string[] = [];
for (const url of urls) {
if (!url || seen.has(url)) continue;
seen.add(url);
unique.push(url);
}
return unique;
}
function buildPageRegistry(
steps: DraftTraceStep[],
startUrl?: string,
): { pages: PageRef[]; urlToId: Map<string, string> } {
const urls = collectUrls(steps, startUrl);
const urlToId = new Map<string, string>();
const pages = urls.map((url, index) => {
const id = `p${index + 1}`;
urlToId.set(url, id);
return { id, url };
});
return { pages, urlToId };
}
function pageIdFor(
url: string | undefined,
urlToId: Map<string, string>,
fallbackUrl?: string,
): string {
if (url && urlToId.has(url)) return urlToId.get(url)!;
if (fallbackUrl && urlToId.has(fallbackUrl)) return urlToId.get(fallbackUrl)!;
return urlToId.values().next().value ?? "p1";
}
function pageUrlForDraft(step: DraftTraceStep, fallbackUrl?: string): string | undefined {
if (step.op === "navigate") return step.page_url ?? step.url;
if ("page_url" in step && step.page_url) return step.page_url;
return fallbackUrl;
}
function effectForNavigation(
navigatedTo: string | undefined,
urlToId: Map<string, string>,
): StepEffectV2 | undefined {
if (!navigatedTo) return undefined;
const pageId = urlToId.get(navigatedTo);
if (!pageId) return undefined;
return { navigated_to: pageId };
}
function captureTargetToV2Target(capture?: CaptureTargetDescriptor): TargetDescriptorV2 {
return {
tag: capture?.tag ?? "unknown",
...(capture?.role ? { role: capture.role } : {}),
...(capture?.name ? { name: capture.name } : {}),
...(capture?.name_attr ? { name_attr: capture.name_attr } : {}),
...(capture?.placeholder ? { placeholder: capture.placeholder } : {}),
...(capture?.nearby_label ? { nearby_label: capture.nearby_label } : {}),
};
}
function targetForDraft(step: DraftTraceStep): TargetDescriptorV2 | undefined {
if (!("target" in step) && !("captureTarget" in step)) return undefined;
const capture = "captureTarget" in step ? step.captureTarget : undefined;
if (capture) return captureTargetToV2Target(capture);
if ("target" in step && step.target && "tag" in (step.target as object)) {
const legacy = step.target as TargetDescriptorV2 & { tag?: string };
if (legacy.tag) return legacy;
}
return undefined;
}
function toSelection(values: string[], labels?: string[]): SelectedOptionV2[] {
return values.map((value, index) => ({
value,
...(labels?.[index] ? { label: labels[index] } : {}),
}));
}
function toV2Step(
step: DraftTraceStep,
id: number,
urlToId: Map<string, string>,
fallbackUrl?: string,
): StepV2 | null {
if (!shouldIncludeDraft(step)) return null;
const pageUrl = pageUrlForDraft(step, fallbackUrl);
const page = pageIdFor(pageUrl, urlToId, fallbackUrl);
const effect =
"navigated_to" in step ? effectForNavigation(step.navigated_to, urlToId) : undefined;
switch (step.op) {
case "navigate":
return {
op: "navigate",
id,
page: pageIdFor(step.url, urlToId, fallbackUrl),
to: step.url,
};
case "click": {
const target = targetForDraft(step);
if (!target) return null;
return { op: "click", id, page, target, ...(effect ? { effect } : {}) };
}
case "hover":
return null;
case "fill": {
const target = targetForDraft(step);
if (!target) return null;
return {
op: "fill",
id,
page,
target,
value: step.value,
...(step.redacted ? { redacted: true } : {}),
};
}
case "press":
return {
op: "press",
id,
page,
key: step.key,
...(step.target ? { target: targetForDraft(step) } : {}),
...(step.modifiers?.length ? { modifiers: step.modifiers } : {}),
...(effect ? { effect } : {}),
};
case "select": {
const target = targetForDraft(step);
if (!target) return null;
return {
op: "select",
id,
page,
target,
selection: toSelection(step.values, step.labels),
...(effect ? { effect } : {}),
};
}
case "scroll":
return null;
}
}
export interface BuildTraceV2Input {
steps: DraftTraceStep[];
startedAt: string;
startUrl?: string;
purpose?: string;
}
export function buildTraceV2(input: BuildTraceV2Input): TraceV2 {
const collapsed = collapseNavigations(input.steps);
const startUrl =
input.startUrl ??
collapsed.find(
(step): step is Extract<DraftTraceStep, { op: "navigate" }> => step.op === "navigate",
)?.url ??
collapsed.find((step) => "page_url" in step && step.page_url)?.page_url ??
"about:blank";
const { pages, urlToId } = buildPageRegistry(collapsed, startUrl);
const out: StepV2[] = [];
let id = 1;
let lastUrl = startUrl;
for (const draft of collapsed) {
if (draft.op === "navigate") lastUrl = draft.url;
else if ("navigated_to" in draft && draft.navigated_to) lastUrl = draft.navigated_to;
else if ("page_url" in draft && draft.page_url) lastUrl = draft.page_url;
const step = toV2Step(draft, id, urlToId, lastUrl);
if (!step) continue;
out.push(step);
id += 1;
}
return {
recorded_at: new Date().toISOString(),
started_at: input.startedAt,
...(input.purpose ? { purpose: input.purpose } : {}),
entry: { start_url: startUrl },
pages,
steps: out,
};
}
+236
View File
@@ -0,0 +1,236 @@
import type {
DraftTraceStep,
FillCommit,
NavigationCause,
Step,
TraceState,
} from "@/transport/types";
import {
DEFAULT_FILL_COMMIT,
fnv1a64,
mapNavigationCause,
nextStateId,
resetStateIdCounterForTests,
} from "./record-constants";
const CLIPBOARD_KEYS = new Set(["a", "c", "v", "x", "A", "C", "V", "X"]);
const MODIFIER_ONLY_KEYS = new Set(["Meta", "Control", "Alt", "Shift", "OS", "Hyper", "Super"]);
export function shouldRecordPress(
key: string,
modifiers?: Array<"alt" | "ctrl" | "meta" | "shift">,
): boolean {
if (MODIFIER_ONLY_KEYS.has(key)) return false;
const mods = modifiers ?? [];
const hasCtrlOrMeta = mods.includes("ctrl") || mods.includes("meta");
if (hasCtrlOrMeta && CLIPBOARD_KEYS.has(key)) return false;
if (key === "Enter" || key === "Escape") return true;
if (key.length === 1 && !hasCtrlOrMeta && !mods.includes("alt")) return false;
return false;
}
function shouldIncludeDraft(step: DraftTraceStep): boolean {
if (step.op === "press" && !shouldRecordPress(step.key, step.modifiers)) return false;
return true;
}
/** A reduced draft plus the capture-order ids of every draft folded into it. */
interface CollapsedDraft {
draft: DraftTraceStep;
draftIds: number[];
}
/**
* Collapse a redirect chain into a single hop.
*
* The surviving step keeps the *first* hop's origin state and cause — that is
* where the user was and why they left — and the *last* hop's destination, so
* the intermediate URL the browser only passed through never becomes the
* step's `state` or `to`.
*/
function collapseNavigations(steps: DraftTraceStep[]): CollapsedDraft[] {
const out: CollapsedDraft[] = [];
steps.forEach((step, index) => {
const draftId = index + 1;
const prev = out[out.length - 1];
if (step.op === "navigate" && prev?.draft.op === "navigate") {
prev.draft = {
...prev.draft,
url: step.url,
postStateId: step.postStateId ?? prev.draft.postStateId,
};
prev.draftIds.push(draftId);
return;
}
out.push({ draft: step, draftIds: [draftId] });
});
return out;
}
export interface StateRegistryEntry extends TraceState {
contentHash: string;
rawVomText: string;
stepsHere: number[];
}
export function stateDedupKey(body: string, url: string): string {
return `${fnv1a64(body)}:${url}`;
}
/** Register or reuse a page observation in the states dictionary. */
export function registerObservation(
registry: Map<string, StateRegistryEntry>,
input: {
url: string;
title?: string;
rawVomText: string;
truncated?: boolean;
},
): string {
const key = stateDedupKey(input.rawVomText, input.url);
for (const entry of registry.values()) {
if (entry.contentHash === key) return entry.id;
}
const id = nextStateId();
registry.set(id, {
id,
url: input.url,
...(input.title ? { title: input.title } : {}),
truncated: input.truncated ?? false,
contentHash: key,
rawVomText: input.rawVomText,
stepsHere: [],
});
return id;
}
function toSelection(
values: string[],
labels?: string[],
): Array<{ value: string; label?: string }> {
return values.map((value, index) => ({
value,
...(labels?.[index] ? { label: labels[index] } : {}),
}));
}
function navigateCauseForDraft(
draft: Extract<DraftTraceStep, { op: "navigate" }>,
): NavigationCause {
if (draft.cause) return draft.cause;
const mapped = mapNavigationCause({
transitionType: draft.transitionType,
transitionQualifiers: draft.transitionQualifiers,
});
return mapped ?? "browser";
}
function toV3Step(draft: DraftTraceStep, id: number): Step | null {
if (!shouldIncludeDraft(draft)) return null;
// An action the user really performed must not disappear because one of its
// observations is missing — a capture racing a document swap is routine, and
// the last step of a recording is the most exposed to it. Fall back to the
// endpoint we do know; only a step with no observation at all has nothing to
// point at.
const preState = draft.preStateId ?? draft.postStateId;
const postState = draft.postStateId ?? draft.preStateId;
if (!preState || !postState) return null;
const common = { id, state: preState, result: { state: postState } };
switch (draft.op) {
case "navigate":
return {
op: "navigate",
...common,
to: draft.url,
cause: navigateCauseForDraft(draft),
};
case "click":
return { op: "click", ...common, target: draft.target };
case "hover":
return { op: "hover", ...common, target: draft.target };
case "fill":
return {
op: "fill",
...common,
target: draft.target,
value: draft.value,
commit: draft.commit ?? DEFAULT_FILL_COMMIT,
...(draft.redacted ? { redacted: true } : {}),
};
case "press":
return {
op: "press",
...common,
key: draft.key,
...(draft.target ? { target: draft.target } : {}),
...(draft.modifiers?.length ? { modifiers: draft.modifiers } : {}),
};
case "select":
return {
op: "select",
...common,
target: draft.target,
selection: toSelection(draft.values, draft.labels),
};
case "scroll":
return { op: "scroll", ...common };
}
}
export interface ReducedTrace {
states: TraceState[];
steps: Step[];
/**
* Capture-order draft id (`draftIndex + 1`) → published step id. Collapsed
* and dropped drafts make the two diverge, so anything the recorder tagged
* with a draft id (`steps_here`, inline annotations) must be remapped.
*/
stepIdByDraftId: Map<number, number>;
}
/**
* Compile capture drafts into record-only trace v3 steps.
* Drafts must already carry `preStateId` / `postStateId` from the recorder.
*/
export function reduceTraceSteps(
steps: DraftTraceStep[],
registry: Map<string, StateRegistryEntry>,
): ReducedTrace {
const collapsed = collapseNavigations(steps);
const out: Step[] = [];
const stepIdByDraftId = new Map<number, number>();
let id = 1;
for (const { draft, draftIds } of collapsed) {
const step = toV3Step(draft, id);
if (!step) continue;
out.push(step);
for (const draftId of draftIds) stepIdByDraftId.set(draftId, id);
id += 1;
}
const states = [...registry.values()].map(
({ contentHash: _hash, rawVomText: _body, stepsHere: _steps, ...state }) => state,
);
return { states, steps: out, stepIdByDraftId };
}
export function resolveTraceStartUrl(
drafts: DraftTraceStep[],
startUrl?: string,
states?: TraceState[],
): string {
if (startUrl) return startUrl;
const navigate = drafts.find((step): step is Extract<DraftTraceStep, { op: "navigate" }> => {
return step.op === "navigate";
});
if (navigate) return navigate.url;
for (const draft of drafts) {
if ("page_url" in draft && draft.page_url) return draft.page_url;
}
return states?.[0]?.url ?? "about:blank";
}
export type { FillCommit, NavigationCause };
/** Test seam */
export { resetStateIdCounterForTests };
@@ -33,19 +33,4 @@ describe("RefStore", () => {
expect(s.resolve("@e11")).toBe(11); expect(s.resolve("@e11")).toBe(11);
expect(s.size()).toBe(2); expect(s.size()).toBe(2);
}); });
it("preserves the child CDP session as part of ref identity", () => {
const s = new RefStore();
s.set("e1", 42, {
tabId: 7,
frameId: "child-frame",
cdpSessionId: "child-session",
});
expect(s.resolveEntry("e1")).toMatchObject({
backendNodeId: 42,
tabId: 7,
frameId: "child-frame",
cdpSessionId: "child-session",
});
});
}); });
@@ -1,10 +1,9 @@
/** /**
* Per-session map from `@e<N>` snapshot refs to a CDP node address. * Per-session map from `@e<N>` snapshot refs to CDP `backendNodeId`.
* *
* Each fresh `tool.snapshot` resets the store: M6 will call * Each fresh `tool.snapshot` resets the store: M6 will call
* `replace(...)` with the new ref → node address pairs. A node address * `replace(...)` with the new ref → backendNodeId pairs. Tools like
* includes the owning flat CDP session/frame when the element lives in * `tool.click` consume the store via `resolve("@e1")`.
* an OOPIF; tools resolve live geometry from that identity at call time.
* *
* Refs are session-scoped (§7): looking up a ref in the wrong session * Refs are session-scoped (§7): looking up a ref in the wrong session
* returns `null`, never silently leaks. Storing values in different * returns `null`, never silently leaks. Storing values in different
@@ -16,19 +15,10 @@ export type BackendNodeId = number;
export interface RefEntry { export interface RefEntry {
backendNodeId: BackendNodeId; backendNodeId: BackendNodeId;
tabId: number | null; tabId: number | null;
frameId?: string;
cdpSessionId?: string;
generation: number; generation: number;
} }
export type RefInput = type RefInput = BackendNodeId | { backendNodeId: BackendNodeId; tabId: number };
| BackendNodeId
| {
backendNodeId: BackendNodeId;
tabId: number;
frameId?: string;
cdpSessionId?: string;
};
export class RefStore { export class RefStore {
private readonly map = new Map<string, RefEntry>(); private readonly map = new Map<string, RefEntry>();
@@ -54,7 +44,7 @@ export class RefStore {
} }
/** /**
* Replace the entire store with a new ref → CDP node identity mapping. * Replace the entire store with a new ref → backendNodeId mapping.
* Used after every fresh `tool.snapshot`. * Used after every fresh `tool.snapshot`.
*/ */
replace(entries: Iterable<readonly [string, RefInput]>): void { replace(entries: Iterable<readonly [string, RefInput]>): void {
@@ -63,20 +53,10 @@ export class RefStore {
for (const [ref, input] of entries) this.map.set(normaliseRef(ref), this.entry(input)); for (const [ref, input] of entries) this.map.set(normaliseRef(ref), this.entry(input));
} }
set( set(ref: string, id: BackendNodeId, opts: { tabId?: number } = {}): void {
ref: string,
id: BackendNodeId,
opts: {
tabId?: number;
frameId?: string;
cdpSessionId?: string;
} = {},
): void {
this.map.set(normaliseRef(ref), { this.map.set(normaliseRef(ref), {
backendNodeId: id, backendNodeId: id,
tabId: opts.tabId ?? null, tabId: opts.tabId ?? null,
...(opts.frameId ? { frameId: opts.frameId } : {}),
...(opts.cdpSessionId ? { cdpSessionId: opts.cdpSessionId } : {}),
generation: this.generation, generation: this.generation,
}); });
} }
@@ -100,8 +80,6 @@ export class RefStore {
return { return {
backendNodeId: input.backendNodeId, backendNodeId: input.backendNodeId,
tabId: input.tabId, tabId: input.tabId,
...(input.frameId ? { frameId: input.frameId } : {}),
...(input.cdpSessionId ? { cdpSessionId: input.cdpSessionId } : {}),
generation: this.generation, generation: this.generation,
}; };
} }
@@ -1,22 +0,0 @@
export const RECORD_DOCUMENT_ATTRIBUTE = "data-bsk-record-document";
const RECORDING_DOCUMENT_ID_PATTERN =
/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
const RECORDING_DOCUMENT_MARKER_PREFIX = "bsk:";
export function recordingDocumentMarkerValue(identity: string): string {
if (!RECORDING_DOCUMENT_ID_PATTERN.test(identity)) {
throw new TypeError("recording Document identity must be a random UUID");
}
return `${RECORDING_DOCUMENT_MARKER_PREFIX}${identity}`;
}
/** Read BrowserSkill's opaque per-Document recording identity from captured attributes. */
export function readRecordingDocumentIdentity(
attrs: Readonly<Record<string, string>>,
): string | undefined {
const marker = attrs[RECORD_DOCUMENT_ATTRIBUTE];
if (!marker?.startsWith(RECORDING_DOCUMENT_MARKER_PREFIX)) return undefined;
const identity = marker.slice(RECORDING_DOCUMENT_MARKER_PREFIX.length);
return identity && RECORDING_DOCUMENT_ID_PATTERN.test(identity) ? identity : undefined;
}
@@ -612,14 +612,4 @@ describe("isInjectableContentScriptUrl", () => {
).toBe(false); ).toBe(false);
expect(isInjectableContentScriptUrl("https://chromewebstore.google.com/")).toBe(false); expect(isInjectableContentScriptUrl("https://chromewebstore.google.com/")).toBe(false);
}); });
it("rejects the Edge Add-ons store", () => {
expect(
isInjectableContentScriptUrl(
"https://microsoftedge.microsoft.com/addons/detail/browserskill/emacgiaaaiojkkpkddmmdfhmokgmnikg",
),
).toBe(false);
// Non-store paths on the same host stay injectable.
expect(isInjectableContentScriptUrl("https://microsoftedge.microsoft.com/")).toBe(true);
});
}); });
@@ -8,7 +8,6 @@ import type {
RequestFrame, RequestFrame,
} from "@/transport/types"; } from "@/transport/types";
import { ToolDispatcher } from "../dispatcher"; import { ToolDispatcher } from "../dispatcher";
import { resetBrowserObservationForTests, setBrowserObservationAttachForTests } from "../record";
type TestDispatcherCdp = NonNullable<ConstructorParameters<typeof ToolDispatcher>[0]["cdp"]>; type TestDispatcherCdp = NonNullable<ConstructorParameters<typeof ToolDispatcher>[0]["cdp"]>;
@@ -45,72 +44,9 @@ function makeRequest(method: string, params: unknown): RequestFrame {
describe("ToolDispatcher", () => { describe("ToolDispatcher", () => {
afterEach(() => { afterEach(() => {
resetBrowserObservationForTests();
vi.unstubAllGlobals(); vi.unstubAllGlobals();
}); });
it("uses the configured recording runtime for start and stop", async () => {
const { transport, sent, deliver } = fakeTransport();
const sessions = new SessionManager({
agentWindow: {
create: vi.fn(async () => 4242),
remove: vi.fn(),
ensureActiveTab: vi.fn(async () => {}),
},
});
await sessions.start("aa11");
const tab = {
id: 7,
windowId: 4242,
active: true,
status: "complete",
url: "https://example.com/start",
} as chrome.tabs.Tab;
const frameCoordinator = {
begin: vi.fn(),
armTab: vi.fn(async () => true),
sourceFor: vi.fn(() => null),
stop: vi.fn(async () => true),
cancel: vi.fn(),
};
const sendToTab = vi.fn(async () => ({ ok: true }));
setBrowserObservationAttachForTests(
() => () => {},
() => () => {},
);
const dispatcher = new ToolDispatcher({
transport,
sessions,
recording: {
tabsApi: {
get: vi.fn(async () => tab),
query: vi.fn(async () => [tab]),
},
frameCoordinator,
sendToTab,
},
});
dispatcher.start();
deliver(
makeRequest("tool.record_start", {
session_id: "aa11",
url: "https://example.com/start",
}),
);
await vi.waitFor(() => expect(sent).toHaveLength(1));
expect(sent[0]).toMatchObject({ result: { tab_id: 7, recording: true } });
expect(frameCoordinator.begin).toHaveBeenCalledOnce();
expect(frameCoordinator.armTab).toHaveBeenCalledWith(expect.any(String), 7);
deliver({ id: "r-2", method: "tool.record_stop", params: { session_id: "aa11" } });
await vi.waitFor(() => expect(sent).toHaveLength(2));
expect(frameCoordinator.stop).toHaveBeenCalledOnce();
expect(sent[1]).toMatchObject({ id: "r-2", result: { trace: { steps: [] } } });
});
it("routes tool.session_start to the SessionManager and replies with the window id", async () => { it("routes tool.session_start to the SessionManager and replies with the window id", async () => {
const { transport, sent, deliver } = fakeTransport(); const { transport, sent, deliver } = fakeTransport();
const sessions = new SessionManager({ const sessions = new SessionManager({
@@ -267,55 +203,6 @@ describe("ToolDispatcher", () => {
expect(sent[0]).toEqual({ id: "r-1", result: {} }); expect(sent[0]).toEqual({ id: "r-1", result: {} });
}); });
it("forwards daemon cancellation into an in-flight session stop", async () => {
const { transport, sent, deliver } = fakeTransport();
const remove = vi.fn(async () => {});
const sessions = new SessionManager({
agentWindow: {
create: vi.fn(async () => 4242),
remove,
ensureActiveTab: vi.fn(async () => {}),
},
});
await sessions.start("aa11");
let finishDetach: () => void = () => {};
const detach = new Promise<void>((resolve) => {
finishDetach = resolve;
});
const cdp = {
send: vi.fn(),
detachSession: vi.fn(() => detach),
ensureNetworkCapture: vi.fn(async () => {}),
networkEntriesSince: vi.fn(() => ({
tab_id: 7,
entries: [],
next_since: 0,
truncated: false,
})),
setDeviceMetricsOverride: vi.fn(async () => {}),
clearDeviceMetricsOverride: vi.fn(async () => {}),
setUserAgentOverride: vi.fn(async () => {}),
setTouchEmulationEnabled: vi.fn(async () => {}),
};
const dispatcher = new ToolDispatcher({ transport, sessions, cdp: cdp as TestDispatcherCdp });
dispatcher.start();
deliver(makeRequest("tool.session_stop", { session_id: "aa11" }));
await vi.waitFor(() => expect(cdp.detachSession).toHaveBeenCalledWith("aa11"));
deliver({ id: "cancel-stop", method: "cancel", params: { rpc_id: "r-1" } });
await flushMicrotasks();
finishDetach();
await flushMicrotasks();
expect(sent).toContainEqual({ id: "cancel-stop", result: { cancelled: true } });
expect(sent).toContainEqual({
id: "r-1",
error: expect.objectContaining({ code: "cancelled" }),
});
expect(sessions.has("aa11")).toBe(true);
expect(remove).not.toHaveBeenCalled();
});
it("returns not_found when stopping an unknown session", async () => { it("returns not_found when stopping an unknown session", async () => {
const { transport, sent, deliver } = fakeTransport(); const { transport, sent, deliver } = fakeTransport();
const sessions = new SessionManager({ const sessions = new SessionManager({
@@ -1,54 +1,24 @@
import { describe, expect, it } from "vitest"; import { describe, expect, it } from "vitest";
import { nodeContentRegion } from "../element-geometry"; import { nodeBoundingRect, quadBoundingRect } from "../element-geometry";
import type { CdpRunner } from "../shared"; import type { CdpRunner } from "../shared";
describe("nodeContentRegion", () => { describe("quadBoundingRect", () => {
it("rejects invalid content quads before using the box model fallback", async () => { it("computes axis-aligned bounds from an 8-double quad", () => {
const send = async (_tabId: number, method: string) => { expect(quadBoundingRect([10, 20, 110, 20, 110, 60, 10, 60])).toEqual({
if (method === "DOM.getContentQuads") return { quads: [[0, 0, 0, 0, 0, 0, 0, 0]] }; x: 10,
if (method === "DOM.getBoxModel") { y: 20,
return { model: { content: [5, 10, 25, 10, 25, 30, 5, 30] } }; width: 100,
} height: 40,
throw new Error(`unexpected CDP call ${method}`);
};
await expect(nodeContentRegion({ send: send as CdpRunner["send"] }, 7, 555)).resolves.toEqual([
[
{ x: 5, y: 10 },
{ x: 25, y: 10 },
{ x: 25, y: 30 },
{ x: 5, y: 30 },
],
]);
});
it("preserves every visible content fragment", async () => {
const send = async () => ({
quads: [
[10, 20, 60, 20, 60, 40, 10, 40],
[15, 50, 90, 50, 90, 80, 15, 80],
],
}); });
const cdp: CdpRunner = {
send: send as CdpRunner["send"],
};
await expect(nodeContentRegion(cdp, 7, 555)).resolves.toEqual([
[
{ x: 10, y: 20 },
{ x: 60, y: 20 },
{ x: 60, y: 40 },
{ x: 10, y: 40 },
],
[
{ x: 15, y: 50 },
{ x: 90, y: 50 },
{ x: 90, y: 80 },
{ x: 15, y: 80 },
],
]);
}); });
it("returns null for degenerate quads", () => {
expect(quadBoundingRect([0, 0, 0, 0, 0, 0, 0, 0])).toBeNull();
expect(quadBoundingRect([1, 2, 3])).toBeNull();
});
});
describe("nodeBoundingRect", () => {
it("falls back to visible descendant bounds for zero-size containers", async () => { it("falls back to visible descendant bounds for zero-size containers", async () => {
const calls: Array<{ method: string; params?: object }> = []; const calls: Array<{ method: string; params?: object }> = [];
const send = async (_tabId: number, method: string, params?: object) => { const send = async (_tabId: number, method: string, params?: object) => {
@@ -68,14 +38,12 @@ describe("nodeContentRegion", () => {
}; };
const cdp = { send: send as CdpRunner["send"] }; const cdp = { send: send as CdpRunner["send"] };
await expect(nodeContentRegion(cdp, 7, 555)).resolves.toEqual([ await expect(nodeBoundingRect(cdp, 7, 555)).resolves.toEqual({
[ x: 242,
{ x: 242, y: 468 }, y: 468,
{ x: 1011, y: 468 }, width: 769,
{ x: 1011, y: 648 }, height: 180,
{ x: 242, y: 648 }, });
],
]);
expect(calls.map((c) => c.method)).toEqual([ expect(calls.map((c) => c.method)).toEqual([
"DOM.getContentQuads", "DOM.getContentQuads",
"DOM.getBoxModel", "DOM.getBoxModel",
@@ -1,286 +0,0 @@
import { describe, expect, it, vi } from "vitest";
import { resolveFrameProjection, resolveNodeGeometry } from "../frame-geometry";
import {
clipPolygon,
polygonArea,
polygonBounds,
projectAndClipRegion,
projectUnitPoint,
type Quad,
rectPolygon,
regionBounds,
} from "../geometry";
import type { CdpRunner } from "../shared";
describe("frame geometry projection", () => {
it("keeps region bounds separate from polygon area", () => {
const region = [
rectPolygon({ x: 0, y: 0, w: 10, h: 10 }),
rectPolygon({ x: 20, y: 5, w: 5, h: 5 }),
];
expect(regionBounds(region)).toEqual({ x: 0, y: 0, width: 25, height: 10 });
expect(polygonArea(region[0])).toBe(100);
});
it("maps through the iframe content box instead of its border box", () => {
const contentQuad: Quad = [
{ x: 204, y: 306 },
{ x: 604, y: 306 },
{ x: 604, y: 506 },
{ x: 204, y: 506 },
];
const projected = projectAndClipRegion(
[rectPolygon({ x: 10, y: 20, w: 100, h: 40 })],
[{ sourceViewport: { width: 200, height: 100 }, destinationQuad: contentQuad }],
{ width: 1000, height: 800 },
);
expect(polygonBounds(projected.flat())).toEqual({ x: 224, y: 346, width: 200, height: 80 });
});
it("uses a projective mapping for perspective-transformed iframe quads", () => {
const quad: Quad = [
{ x: 0, y: 0 },
{ x: 200, y: 20 },
{ x: 180, y: 140 },
{ x: 20, y: 100 },
];
expect(projectUnitPoint({ x: 0, y: 0 }, quad)).toEqual(quad[0]);
expect(projectUnitPoint({ x: 1, y: 0 }, quad)).toEqual(quad[1]);
expect(projectUnitPoint({ x: 1, y: 1 }, quad)).toEqual(quad[2]);
expect(projectUnitPoint({ x: 0, y: 1 }, quad)).toEqual(quad[3]);
});
it("clips against the iframe polygon rather than its bounding box", () => {
const frameQuad: Quad = [
{ x: 50, y: 0 },
{ x: 100, y: 50 },
{ x: 50, y: 100 },
{ x: 0, y: 50 },
];
expect(clipPolygon(rectPolygon({ x: 0, y: 0, w: 10, h: 10 }), frameQuad)).toEqual([]);
});
it("does not reapply same-target frame transforms around an OOPIF", async () => {
const send = vi.fn(async (_tabId, method, params) => {
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 800, clientHeight: 600 } };
}
if (method === "DOM.getBoxModel") {
const backendNodeId = (params as { backendNodeId?: number }).backendNodeId;
return backendNodeId === 20
? { model: { content: [110, 70, 310, 70, 310, 170, 110, 170] } }
: { model: { content: [100, 50, 400, 50, 400, 250, 100, 250] } };
}
throw new Error(`unexpected root command ${method}`);
});
const cdp: CdpRunner = {
send: send as CdpRunner["send"],
sendToTarget: vi.fn(async (_target, method) => {
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
}
throw new Error(`unexpected child command ${method}`);
}) as CdpRunner["sendToTarget"],
};
const projection = await resolveFrameProjection(
cdp,
{
rootFrameId: "main",
frames: [
{ frameId: "main", target: { tabId: 4 } },
{
frameId: "same-process-parent",
parentFrameId: "main",
ownerBackendNodeId: 10,
target: { tabId: 4 },
},
{
frameId: "oopif",
parentFrameId: "same-process-parent",
ownerBackendNodeId: 20,
target: { tabId: 4, sessionId: "oopif-session" },
},
],
},
"oopif",
);
expect(projection?.edges).toEqual([
{
sourceViewport: { width: 200, height: 100 },
destinationQuad: [
{ x: 110, y: 70 },
{ x: 310, y: 70 },
{ x: 310, y: 170 },
{ x: 110, y: 170 },
],
destinationClips: [
[
{ x: 100, y: 50 },
{ x: 400, y: 50 },
{ x: 400, y: 250 },
{ x: 100, y: 250 },
],
],
},
]);
});
it("clips live OOPIF geometry at every frame boundary", async () => {
const cdp: CdpRunner = {
send: vi.fn(async (_tabId, method) => {
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 800, clientHeight: 600 } };
}
if (method === "DOM.getBoxModel") {
return { model: { content: [100, 100, 300, 100, 300, 200, 100, 200] } };
}
throw new Error(`unexpected root command ${method}`);
}) as CdpRunner["send"],
sendToTarget: vi.fn(async (_target, method) => {
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
}
if (method === "DOM.getContentQuads") {
return { quads: [[-50, -20, 250, -20, 250, 120, -50, 120]] };
}
throw new Error(`unexpected child command ${method}`);
}) as CdpRunner["sendToTarget"],
getFrameGraph: vi.fn(async () => ({
rootFrameId: "main",
frames: [
{ frameId: "main", target: { tabId: 4 } },
{
frameId: "child",
parentFrameId: "main",
ownerBackendNodeId: 10,
target: { tabId: 4, sessionId: "child-session" },
},
],
})),
};
const geometry = await resolveNodeGeometry(cdp, 4, {
target: { tabId: 4, sessionId: "child-session" },
frameId: "child",
backendNodeId: 101,
});
expect(geometry).toMatchObject({
topBounds: { x: 100, y: 100, width: 200, height: 100 },
actionPoint: { x: 200, y: 150 },
});
});
it("reuses one frame graph while scrolling and projecting live geometry", async () => {
const getFrameGraph = vi.fn(async () => ({
rootFrameId: "main",
frames: [
{ frameId: "main", target: { tabId: 4 } },
{
frameId: "child",
parentFrameId: "main",
ownerBackendNodeId: 10,
target: { tabId: 4, sessionId: "child-session" },
},
],
}));
const cdp: CdpRunner = {
send: vi.fn(async (_tabId, method) => {
if (method === "DOM.scrollIntoViewIfNeeded") return {};
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 800, clientHeight: 600 } };
}
if (method === "DOM.getBoxModel") {
return { model: { content: [100, 100, 300, 100, 300, 200, 100, 200] } };
}
throw new Error(`unexpected root command ${method}`);
}) as CdpRunner["send"],
sendToTarget: vi.fn(async (_target, method) => {
if (method === "DOM.scrollIntoViewIfNeeded") return {};
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
}
if (method === "DOM.getContentQuads") {
return { quads: [[0, 0, 20, 0, 20, 20, 0, 20]] };
}
throw new Error(`unexpected child command ${method}`);
}) as CdpRunner["sendToTarget"],
getFrameGraph,
};
await expect(
resolveNodeGeometry(
cdp,
4,
{
target: { tabId: 4, sessionId: "child-session" },
frameId: "child",
backendNodeId: 101,
},
{ scrollIntoView: true },
),
).resolves.not.toMatchObject({ code: expect.any(String) });
expect(getFrameGraph).toHaveBeenCalledOnce();
});
it("clips live geometry to same-target iframe ancestors", async () => {
const cdp: CdpRunner = {
send: vi.fn(async (_tabId, method) => {
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 800, clientHeight: 600 } };
}
if (method === "DOM.getBoxModel") {
return { model: { content: [100, 100, 300, 100, 300, 200, 100, 200] } };
}
if (method === "DOM.getContentQuads") {
return { quads: [[50, 50, 350, 50, 350, 250, 50, 250]] };
}
throw new Error(`unexpected ${method}`);
}) as CdpRunner["send"],
getFrameGraph: vi.fn(async () => ({
rootFrameId: "main",
frames: [
{ frameId: "main", target: { tabId: 4 } },
{
frameId: "child",
parentFrameId: "main",
ownerBackendNodeId: 10,
target: { tabId: 4 },
},
],
})),
};
const geometry = await resolveNodeGeometry(cdp, 4, {
target: { tabId: 4 },
frameId: "child",
backendNodeId: 101,
});
expect(geometry).toMatchObject({
topBounds: { x: 100, y: 100, width: 200, height: 100 },
actionPoint: { x: 200, y: 150 },
});
});
it("fails closed when an OOPIF frame graph is unavailable", async () => {
const cdp: CdpRunner = {
send: vi.fn(),
sendToTarget: vi.fn(async (_target, method) => {
if (method === "DOM.getContentQuads") {
return { quads: [[0, 0, 20, 0, 20, 20, 0, 20]] };
}
throw new Error(`unexpected ${method}`);
}) as CdpRunner["sendToTarget"],
};
await expect(
resolveNodeGeometry(cdp, 4, {
target: { tabId: 4, sessionId: "child-session" },
frameId: "child",
backendNodeId: 101,
}),
).resolves.toMatchObject({ code: "cdp_failed" });
});
});
@@ -1,6 +1,5 @@
import { afterEach, describe, expect, it, vi } from "vitest"; import { afterEach, describe, expect, it, vi } from "vitest";
import { SessionManager } from "@/session-manager/manager"; import { SessionManager } from "@/session-manager/manager";
import { RefStore } from "@/session-manager/ref-store";
import type { RequestHelpParams } from "@/transport/types"; import type { RequestHelpParams } from "@/transport/types";
import { handleRequestHelp, type RequestHelpDeps, resetHelpLifecycleForTests } from "../human-loop"; import { handleRequestHelp, type RequestHelpDeps, resetHelpLifecycleForTests } from "../human-loop";
@@ -51,10 +50,11 @@ function installHelpLifecycleChrome() {
} }
function fakeManager(sessionId: string, agentWindowId: number, tabId: number) { function fakeManager(sessionId: string, agentWindowId: number, tabId: number) {
const refStore = new RefStore();
const mgr = { const mgr = {
get: (id: string) => get: (id: string) =>
id === sessionId ? { sessionId, agentWindowId, refStore, borrowedTabs: new Map() } : null, id === sessionId
? { sessionId, agentWindowId, refStore: { resolve: () => null }, borrowedTabs: new Map() }
: null,
findByWindowId: (wid: number) => (wid === agentWindowId ? { sessionId } : null), findByWindowId: (wid: number) => (wid === agentWindowId ? { sessionId } : null),
} as unknown as SessionManager; } as unknown as SessionManager;
return mgr; return mgr;
@@ -339,23 +339,26 @@ describe("handleRequestHelp", () => {
ac.abort(); ac.abort();
await expect(pending).resolves.toMatchObject({ code: "cancelled" }); await expect(pending).resolves.toMatchObject({ code: "cancelled" });
expect(cdpSend).not.toHaveBeenCalledWith( expect(cdpSend).toHaveBeenCalledWith(
expect.anything(), 5,
"DOM.removeAttribute", "DOM.querySelectorAll",
expect.anything(), expect.objectContaining({ selector: "[data-bsk-help]" }),
);
expect(cdpSend).toHaveBeenCalledWith(
6,
"DOM.querySelectorAll",
expect.objectContaining({ selector: "[data-bsk-help]" }),
); );
}); });
it("resolves ref targets to explicit viewport rectangles without mutating the page", async () => { it("tags ref targets via CDP and reports them matched", async () => {
const refStore = new RefStore();
refStore.set("e1", 42, { tabId: 5 });
const mgr = { const mgr = {
get: (id: string) => get: (id: string) =>
id === "abcd" id === "abcd"
? { ? {
sessionId: "abcd", sessionId: "abcd",
agentWindowId: 99, agentWindowId: 99,
refStore, refStore: { resolve: () => 42 },
borrowedTabs: new Map(), borrowedTabs: new Map(),
} }
: null, : null,
@@ -363,17 +366,7 @@ describe("handleRequestHelp", () => {
} as unknown as SessionManager; } as unknown as SessionManager;
const deps = baseDeps({ const deps = baseDeps({
cdp: { cdp: {
send: vi.fn(async (_tabId, method) => { send: vi.fn(async () => ({ object: { objectId: "obj-1" } })),
if (method === "DOM.getDocument") return { root: { nodeId: 1 } };
if (method === "DOM.scrollIntoViewIfNeeded") return {};
if (method === "DOM.getContentQuads") {
return { quads: [[10, 20, 110, 20, 110, 60, 10, 60]] };
}
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
}
throw new Error(`unexpected ${method}`);
}),
} as unknown as RequestHelpDeps["cdp"], } as unknown as RequestHelpDeps["cdp"],
}); });
const res = await handleRequestHelp( const res = await handleRequestHelp(
@@ -382,97 +375,14 @@ describe("handleRequestHelp", () => {
deps, deps,
); );
const sentMsg = (deps.sendToTab as ReturnType<typeof vi.fn>).mock.calls[0][1]; const sentMsg = (deps.sendToTab as ReturnType<typeof vi.fn>).mock.calls[0][1];
expect(sentMsg.selectors).toEqual([]); expect(sentMsg.selectors).toContain('[data-bsk-help="0"]');
expect(sentMsg.rects).toEqual([{ top: 20, left: 10, width: 100, height: 40 }]);
expect(res).toMatchObject({ expect(res).toMatchObject({
outcome: "continued", outcome: "continued",
tab_id: 5, tab_id: 5,
resolved_targets: [ resolved_targets: [{ matched: true, ref: "@e1" }],
{
matched: true,
ref: "@e1",
},
],
}); });
}); });
it("projects an OOPIF ref through its live content quad for the help overlay", async () => {
const mgr = {
get: (id: string) =>
id === "abcd"
? {
sessionId: "abcd",
agentWindowId: 99,
refStore: {
resolveEntry: () => ({
backendNodeId: 42,
tabId: 5,
frameId: "child",
cdpSessionId: "child-session",
generation: 1,
}),
},
borrowedTabs: new Map(),
}
: null,
findByWindowId: (wid: number) => (wid === 99 ? { sessionId: "abcd" } : null),
} as unknown as SessionManager;
const send = vi.fn(async (_tabId, method) => {
if (method === "DOM.getDocument") return { root: { nodeId: 1 } };
if (method === "DOM.scrollIntoViewIfNeeded") return {};
if (method === "DOM.getBoxModel") {
return { model: { content: [204, 306, 604, 306, 604, 506, 204, 506] } };
}
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
}
throw new Error(`unexpected root ${method}`);
});
const sendToTarget = vi.fn(async (_target, method) => {
if (method === "DOM.scrollIntoViewIfNeeded") return {};
if (method === "DOM.getContentQuads") {
return { quads: [[10, 20, 110, 20, 110, 60, 10, 60]] };
}
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
}
throw new Error(`unexpected child ${method}`);
});
const deps = baseDeps({
cdp: {
send,
sendToTarget,
getFrameGraph: vi.fn(async () => ({
rootFrameId: "main",
frames: [
{ frameId: "main", target: { tabId: 5 } },
{
frameId: "child",
parentFrameId: "main",
ownerBackendNodeId: 99,
target: { tabId: 5, sessionId: "child-session" },
},
],
})),
} as unknown as NonNullable<RequestHelpDeps["cdp"]>,
});
const res = await handleRequestHelp(
mgr,
baseParams({ tab_id: 5, targets: [{ ref: "@e1" }] }),
deps,
);
const sentMsg = (deps.sendToTab as ReturnType<typeof vi.fn>).mock.calls[0][1];
expect(sentMsg.rects).toEqual([{ top: 346, left: 224, width: 200, height: 80 }]);
expect(res).toMatchObject({ resolved_targets: [{ matched: true, ref: "@e1" }] });
expect(sendToTarget).toHaveBeenCalledWith(
{ tabId: 5, sessionId: "child-session" },
"DOM.getContentQuads",
{ backendNodeId: 42 },
);
});
it("reports ref target unmatched when ref does not resolve", async () => { it("reports ref target unmatched when ref does not resolve", async () => {
const deps = baseDeps(); const deps = baseDeps();
const res = await handleRequestHelp( const res = await handleRequestHelp(
@@ -488,15 +398,16 @@ describe("handleRequestHelp", () => {
}); });
it("reports ref target unmatched when ref is for another tab", async () => { it("reports ref target unmatched when ref is for another tab", async () => {
const refStore = new RefStore();
refStore.set("e1", 42, { tabId: 4 });
const mgr = { const mgr = {
get: (id: string) => get: (id: string) =>
id === "abcd" id === "abcd"
? { ? {
sessionId: "abcd", sessionId: "abcd",
agentWindowId: 99, agentWindowId: 99,
refStore, refStore: {
resolve: (ref: string, opts: { tabId?: number }) =>
ref === "e1" && opts.tabId === 4 ? 42 : null,
},
borrowedTabs: new Map(), borrowedTabs: new Map(),
} }
: null, : null,
@@ -30,9 +30,6 @@ function makeFakeCdp(handlers: Record<string, (params: unknown) => unknown>) {
const sendImpl = async (tabId: number, method: string, params?: object) => { const sendImpl = async (tabId: number, method: string, params?: object) => {
sent.push({ tabId, method, params }); sent.push({ tabId, method, params });
const h = handlers[method]; const h = handlers[method];
if (!h && method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
}
if (!h) throw new Error(`unexpected CDP call ${method}`); if (!h) throw new Error(`unexpected CDP call ${method}`);
return h(params); return h(params);
}; };
@@ -167,63 +164,6 @@ describe("handleClick", () => {
}); });
}); });
it("resolves frame refs in their CDP session and dispatches input in top coordinates", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
ctx.refStore.set("e3", 1234, {
tabId: 4,
frameId: "child-frame",
cdpSessionId: "child-session",
});
const fake = makeFakeCdp({
"Input.dispatchMouseEvent": () => ({}),
"DOM.scrollIntoViewIfNeeded": () => ({}),
"DOM.getBoxModel": () => ({
model: { content: [204, 306, 604, 306, 604, 506, 204, 506] },
}),
});
fake.cdp.getFrameGraph = vi.fn(async () => ({
rootFrameId: "main",
frames: [
{ frameId: "main", target: { tabId: 4 } },
{
frameId: "child-frame",
parentFrameId: "main",
ownerBackendNodeId: 99,
target: { tabId: 4, sessionId: "child-session" },
},
],
}));
const targetCalls: Array<{ sessionId?: string; method: string }> = [];
fake.cdp.sendToTarget = vi.fn(async (target, method) => {
targetCalls.push({ sessionId: target.sessionId, method });
if (method === "DOM.scrollIntoViewIfNeeded") return {};
if (method === "DOM.getContentQuads") {
return { quads: [[10, 20, 110, 20, 110, 60, 10, 60]] };
}
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
}
throw new Error(`unexpected child CDP call ${method}`);
}) as CdpRunner["sendToTarget"];
const res = await handleClick(
sm,
{ session_id: "aa11", ref: "@e3" },
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
);
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
// The iframe content box starts at (204, 306) after its border and is scaled 2x.
expect(res).toMatchObject({ x: 324, y: 386 });
expect(targetCalls).toEqual([
{ sessionId: "child-session", method: "DOM.scrollIntoViewIfNeeded" },
{ sessionId: "child-session", method: "DOM.getContentQuads" },
{ sessionId: "child-session", method: "Page.getLayoutMetrics" },
]);
expect(fake.sent.filter((call) => call.method === "Input.dispatchMouseEvent")).toHaveLength(3);
});
it("enables overlay bypass before mouse events when overlay blocks the click point", async () => { it("enables overlay bypass before mouse events when overlay blocks the click point", async () => {
const order: string[] = []; const order: string[] = [];
const bypassOverlay = vi.fn(async (_tabId: number, enabled: boolean) => { const bypassOverlay = vi.fn(async (_tabId: number, enabled: boolean) => {
File diff suppressed because it is too large Load Diff
@@ -1,11 +1,19 @@
import { afterEach, describe, expect, it, vi } from "vitest"; import { afterEach, describe, expect, it, vi } from "vitest";
import { RECORD_FINISH, RECORD_START, RECORD_STEP, RECORD_STOP } from "@/lib/record-bridge"; import {
RECORD_FINISH,
RECORD_QUERY,
RECORD_START,
RECORD_STEP,
RECORD_STOP,
} from "@/lib/record-bridge";
import { resetStateIdCounterForTests } from "@/lib/trace-reducer";
import type { SessionManager } from "@/session-manager/manager"; import type { SessionManager } from "@/session-manager/manager";
import type { CdpRunner } from "@/tools/shared"; import type { CdpRunner } from "@/tools/shared";
import { EXTENSION_VERSION } from "@/transport/handshake"; import { EXTENSION_VERSION } from "@/transport/handshake";
import type { RecordedTrace, RecordStopResult, TraceV3 } from "@/transport/types"; import type { RecordedTrace, RecordStopResult, TraceV3 } from "@/transport/types";
import { import {
attachRecordFinishListener, attachRecordFinishListener,
attachRecordQueryListener,
attachRecordStepListener, attachRecordStepListener,
handleRecordStart, handleRecordStart,
handleRecordStop, handleRecordStop,
@@ -15,13 +23,7 @@ import {
const AGENT_WINDOW_ID = 100; const AGENT_WINDOW_ID = 100;
const TAB_ID = 4; const TAB_ID = 4;
const START_URL = "https://example.com/"; const START_URL = "https://example.com/";
const RECORD_START_V3 = { const RECORD_START_V3 = { session_id: "abcd", url: START_URL, trace_version: 3 as const };
session_id: "abcd",
url: START_URL,
trace_version: 3 as const,
supports_tab_switch_steps: true,
};
const stepSequenceByProducer = new Map<string, number>();
function asTraceV3(trace: RecordedTrace): TraceV3 { function asTraceV3(trace: RecordedTrace): TraceV3 {
if (!("version" in trace)) { if (!("version" in trace)) {
@@ -53,7 +55,6 @@ function chromeEvent<T extends (...args: never[]) => unknown>() {
function installChrome() { function installChrome() {
const runtimeOnMessage = chromeEvent<RuntimeListener>(); const runtimeOnMessage = chromeEvent<RuntimeListener>();
const tabsOnActivated = chromeEvent<(activeInfo: chrome.tabs.TabActiveInfo) => unknown>();
const webNavigationOnCompleted = const webNavigationOnCompleted =
chromeEvent<(details: chrome.webNavigation.WebNavigationFramedCallbackDetails) => unknown>(); chromeEvent<(details: chrome.webNavigation.WebNavigationFramedCallbackDetails) => unknown>();
const webNavigationOnCommitted = const webNavigationOnCommitted =
@@ -63,7 +64,7 @@ function installChrome() {
vi.stubGlobal("chrome", { vi.stubGlobal("chrome", {
runtime: { onMessage: runtimeOnMessage }, runtime: { onMessage: runtimeOnMessage },
tabs: { tabs: {
onActivated: tabsOnActivated, onActivated: chromeEvent(),
onCreated: chromeEvent(), onCreated: chromeEvent(),
onUpdated: chromeEvent(), onUpdated: chromeEvent(),
}, },
@@ -72,7 +73,7 @@ function installChrome() {
onCommitted: webNavigationOnCommitted, onCommitted: webNavigationOnCommitted,
}, },
}); });
return { runtimeOnMessage, tabsOnActivated, webNavigationOnCompleted, webNavigationOnCommitted }; return { runtimeOnMessage, webNavigationOnCompleted, webNavigationOnCommitted };
} }
function fakeManager() { function fakeManager() {
@@ -160,9 +161,9 @@ function makeFakeCdp(
return { result: { value: { idleMs, readyState: "complete" } } }; return { result: { value: { idleMs, readyState: "complete" } } };
}, },
"Accessibility.enable": () => ({}), "Accessibility.enable": () => ({}),
"Accessibility.getFullAXTree": (_params, _tabId) => { "Accessibility.getFullAXTree": (_params, tabId) => {
if (failing) throw new Error("Execution context was destroyed"); if (failing) throw new Error("Execution context was destroyed");
const tabTree = options?.treesByTab?.[_tabId]; const tabTree = options?.treesByTab?.[tabId];
if (tabTree) return tabTree; if (tabTree) return tabTree;
if (script.length === 0) return axTree("Example Domain", ["Submit"]); if (script.length === 0) return axTree("Example Domain", ["Submit"]);
return script.length === 1 ? script[0] : script.shift(); return script.length === 1 ? script[0] : script.shift();
@@ -227,13 +228,8 @@ function makeMultiTabsApi(tabs: chrome.tabs.Tab[]) {
(query.windowId === undefined || tab.windowId === query.windowId) && (query.windowId === undefined || tab.windowId === query.windowId) &&
(query.active === undefined || tab.active === query.active), (query.active === undefined || tab.active === query.active),
), ),
activate(tabId: number) { close(tabId: number) {
for (const tab of byId.values()) tab.active = tab.id === tabId; byId.delete(tabId);
},
goTo(tabId: number, url: string, title: string) {
const tab = byId.get(tabId);
if (!tab) throw new Error(`tab ${tabId} closed`);
Object.assign(tab, { url, title });
}, },
}; };
} }
@@ -247,26 +243,18 @@ function runtimeOnMessageEmit(
chromeApi: ReturnType<typeof installChrome>, chromeApi: ReturnType<typeof installChrome>,
requestId: string, requestId: string,
step: unknown, step: unknown,
tabId = TAB_ID, ): void {
active = true,
): ReturnType<typeof vi.fn> {
const producerId = `test-producer-${tabId}`;
const producerKey = `${requestId}:${producerId}`;
const sequence = (stepSequenceByProducer.get(producerKey) ?? 0) + 1;
stepSequenceByProducer.set(producerKey, sequence);
const sendResponse = vi.fn();
chromeApi.runtimeOnMessage.emit( chromeApi.runtimeOnMessage.emit(
{ type: RECORD_STEP, requestId, producerId, sequence, step }, { type: RECORD_STEP, requestId, step },
{ tab: { id: tabId, active } } as chrome.runtime.MessageSender, { tab: { id: TAB_ID } } as chrome.runtime.MessageSender,
sendResponse, () => {},
); );
return sendResponse;
} }
describe("recorded user steps reach the exported trace", () => { describe("recorded user steps reach the exported trace", () => {
afterEach(() => { afterEach(() => {
stepSequenceByProducer.clear();
resetBrowserObservationForTests(); resetBrowserObservationForTests();
resetStateIdCounterForTests();
vi.unstubAllGlobals(); vi.unstubAllGlobals();
}); });
@@ -296,14 +284,15 @@ describe("recorded user steps reach the exported trace", () => {
{ {
type: RECORD_STEP, type: RECORD_STEP,
requestId, requestId,
producerId: "test-producer-4",
sequence: 1,
step: { step: {
op: "click", op: "click",
page_url: START_URL, page_url: START_URL,
target: { role: "button", name: "Submit", tag: "button" }, target: { role: "button", name: "Submit", tag: "button" },
geometry: { geometry: {
rect: { x: 0, y: 0, w: 10, h: 10 }, rect: { x: 0, y: 0, w: 10, h: 10 },
scrollX: 0,
scrollY: 0,
position: "static",
tag: "button", tag: "button",
}, },
}, },
@@ -339,37 +328,21 @@ describe("recorded user steps reach the exported trace", () => {
await handleRecordStart(manager, RECORD_START_V3, { tabsApi, sendToTab, cdp: makeFakeCdp() }); await handleRecordStart(manager, RECORD_START_V3, { tabsApi, sendToTab, cdp: makeFakeCdp() });
attachRecordStepListener({ tabsApi, sendToTab }); attachRecordStepListener({ tabsApi, sendToTab });
let sequence = 0;
const emit = (step: unknown) => { const emit = (step: unknown) => {
sequence += 1;
runtimeOnMessage.emit( runtimeOnMessage.emit(
{ type: RECORD_STEP, requestId, producerId: "test-producer-4", sequence, step }, { type: RECORD_STEP, requestId, step },
{ tab: { id: TAB_ID } } as chrome.runtime.MessageSender, { tab: { id: TAB_ID } } as chrome.runtime.MessageSender,
() => {}, () => {},
); );
}; };
const fill = { emit({
op: "fill", op: "fill",
page_url: START_URL, page_url: START_URL,
target: { role: "textbox", name: "Search", tag: "input" }, target: { role: "textbox", name: "Search", tag: "input" },
value: "hello", value: "hello",
commit: "enter", commit: "enter",
}; });
emit(fill);
const duplicateAck = vi.fn();
runtimeOnMessage.emit(
{
type: RECORD_STEP,
requestId,
producerId: "test-producer-4",
sequence: 1,
step: fill,
},
{ tab: { id: TAB_ID } } as chrome.runtime.MessageSender,
duplicateAck,
);
expect(duplicateAck).toHaveBeenCalledWith({ ok: true, sequence: 1 });
emit({ emit({
op: "click", op: "click",
page_url: START_URL, page_url: START_URL,
@@ -438,13 +411,13 @@ describe("recorded user steps reach the exported trace", () => {
}); });
attachRecordStepListener({ tabsApi, sendToTab }); attachRecordStepListener({ tabsApi, sendToTab });
const commit = (url: string, transitionType: string, transitionQualifiers: string[] = []) => { const commit = (url: string, transitionType: string) => {
chromeApi.webNavigationOnCommitted.emit({ chromeApi.webNavigationOnCommitted.emit({
tabId: TAB_ID, tabId: TAB_ID,
frameId: 0, frameId: 0,
url, url,
transitionType, transitionType,
transitionQualifiers, transitionQualifiers: [],
} as unknown as chrome.webNavigation.WebNavigationTransitionCallbackDetails); } as unknown as chrome.webNavigation.WebNavigationTransitionCallbackDetails);
}; };
@@ -455,7 +428,7 @@ describe("recorded user steps reach the exported trace", () => {
await settleWait(); await settleWait();
tabsApi.goTo("https://iwiki.woa.com/dashboard", "工作台 - 腾讯iWiki"); tabsApi.goTo("https://iwiki.woa.com/dashboard", "工作台 - 腾讯iWiki");
commit("https://iwiki.woa.com/dashboard", "link", ["server_redirect"]); commit("https://iwiki.woa.com/dashboard", "link");
await settleWait(); await settleWait();
runtimeOnMessageEmit(chromeApi, requestId, { runtimeOnMessageEmit(chromeApi, requestId, {
@@ -1106,52 +1079,61 @@ describe("recorded user steps reach the exported trace", () => {
expect(asTraceV3((stopped as RecordStopResult).trace).stopped_by).toBe("user_finish"); expect(asTraceV3((stopped as RecordStopResult).trace).stopped_by).toBe("user_finish");
}); });
it("returns a shared finish failure to a concurrent CLI stop", async () => { it("retries STOP only on armed tabs whose final delivery did not ack", async () => {
const chromeApi = installChrome(); const chromeApi = installChrome();
const manager = fakeManager(); const manager = fakeManager();
const tabsApi = makeTabsApi(); const firstTab = {
let requestId = ""; id: TAB_ID,
let releaseStop!: () => void; windowId: AGENT_WINDOW_ID,
const stopReleased = new Promise<void>((resolve) => { active: true,
releaseStop = resolve; status: "complete",
}); url: START_URL,
let announceStop!: () => void; title: "First",
const stopStarted = new Promise<void>((resolve) => { } as chrome.tabs.Tab;
announceStop = resolve; const secondTab = {
}); id: 5,
const sendToTab = vi.fn(async (_tabId: number, msg: unknown) => { windowId: AGENT_WINDOW_ID,
const typed = msg as { type?: string; requestId?: string }; active: false,
if (typed.type === RECORD_START && typed.requestId) requestId = typed.requestId; status: "complete",
url: "https://example.com/second",
title: "Second",
} as chrome.tabs.Tab;
const tabsApi = makeMultiTabsApi([firstTab, secondTab]);
const stopTabs: number[] = [];
let secondStopAttempts = 0;
const sendToTab = vi.fn(async (tabId: number, msg: unknown) => {
const typed = msg as { type?: string };
if (typed.type === RECORD_STOP) { if (typed.type === RECORD_STOP) {
announceStop(); stopTabs.push(tabId);
await stopReleased; if (tabId === 5 && secondStopAttempts++ === 0) {
return { ok: false, error: "final capture failed" }; return { ok: false, error: "failed to deliver final fill" };
}
} }
return { ok: true }; return { ok: true };
}); });
const deps = { tabsApi, sendToTab, cdp: makeFakeCdp() }; const deps = { tabsApi, sendToTab, cdp: makeFakeCdp() };
await handleRecordStart(manager, RECORD_START_V3, deps); await handleRecordStart(manager, RECORD_START_V3, deps);
attachRecordFinishListener(deps); attachRecordQueryListener(deps);
chromeApi.runtimeOnMessage.emit( await new Promise<void>((resolve) => {
{ type: RECORD_FINISH, requestId }, chromeApi.runtimeOnMessage.emit(
{ tab: { id: TAB_ID } } as chrome.runtime.MessageSender, { type: RECORD_QUERY },
() => {}, { tab: secondTab } as chrome.runtime.MessageSender,
); () => resolve(),
await stopStarted; );
});
const cliStop = handleRecordStop(manager, { session_id: "abcd" }, deps); const firstStop = await handleRecordStop(manager, { session_id: "abcd" }, deps);
releaseStop(); expect(firstStop).toMatchObject({ code: "protocol_error" });
const result = await Promise.race([ expect(stopTabs).toEqual([TAB_ID, 5]);
cliStop,
new Promise<"timed_out">((resolve) => setTimeout(() => resolve("timed_out"), 500)),
]);
expect(result).not.toBe("timed_out"); const retry = await handleRecordStop(manager, { session_id: "abcd" }, deps);
expect(result).toMatchObject({ code: "protocol_error" }); const retryTrace = (retry as RecordStopResult).trace;
expect("version" in retryTrace && retryTrace.version).toBe(3);
expect(stopTabs).toEqual([TAB_ID, 5, 5]);
}); });
it("defaults to legacy trace v2 without starting observation capture", async () => { it("defaults to trace v2 when trace_version is omitted", async () => {
const { runtimeOnMessage } = installChrome(); const { runtimeOnMessage } = installChrome();
const manager = fakeManager(); const manager = fakeManager();
const tabsApi = makeTabsApi(); const tabsApi = makeTabsApi();
@@ -1169,23 +1151,6 @@ describe("recorded user steps reach the exported trace", () => {
{ {
type: RECORD_STEP, type: RECORD_STEP,
requestId, requestId,
producerId: "test-producer-4",
sequence: 1,
step: {
op: "hover",
page_url: START_URL,
target: { role: "button", name: "Submit", tag: "button" },
},
},
{ tab: { id: TAB_ID } } as chrome.runtime.MessageSender,
() => {},
);
runtimeOnMessage.emit(
{
type: RECORD_STEP,
requestId,
producerId: "test-producer-4",
sequence: 2,
step: { step: {
op: "click", op: "click",
page_url: START_URL, page_url: START_URL,
@@ -1200,8 +1165,6 @@ describe("recorded user steps reach the exported trace", () => {
const trace = (stopped as RecordStopResult).trace; const trace = (stopped as RecordStopResult).trace;
expect("version" in trace).toBe(false); expect("version" in trace).toBe(false);
expect("pages" in trace && trace.pages.length).toBeGreaterThan(0); expect("pages" in trace && trace.pages.length).toBeGreaterThan(0);
// `hover` has no v2 counterpart on peers that negotiate v2.
expect("steps" in trace && trace.steps.map((step) => step.op)).toEqual(["click"]);
}); });
it("rejects unsupported trace_version values", async () => { it("rejects unsupported trace_version values", async () => {
@@ -1214,37 +1177,142 @@ describe("recorded user steps reach the exported trace", () => {
expect(result).toMatchObject({ code: "invalid_params" }); expect(result).toMatchObject({ code: "invalid_params" });
}); });
it("records tab transitions and binds actions to each tab's observation", async () => { it("does not let a closed armed tab block finish", async () => {
const chromeApi = installChrome(); const chromeApi = installChrome();
const manager = fakeManager(); const manager = fakeManager();
const secondUrl = "https://example.com/second"; const firstTab = {
const tabsApi = makeMultiTabsApi([ id: TAB_ID,
{ windowId: AGENT_WINDOW_ID,
id: TAB_ID, active: true,
windowId: AGENT_WINDOW_ID, status: "complete",
active: true, url: START_URL,
status: "complete", } as chrome.tabs.Tab;
url: START_URL, const secondTab = {
title: "First tab", id: 5,
} as chrome.tabs.Tab, windowId: AGENT_WINDOW_ID,
{ active: false,
id: 5, status: "complete",
windowId: AGENT_WINDOW_ID, url: "https://example.com/second",
active: false, } as chrome.tabs.Tab;
status: "complete", const tabsApi = makeMultiTabsApi([firstTab, secondTab]);
url: secondUrl, const stopTabs: number[] = [];
title: "Second tab", const sendToTab = vi.fn(async (tabId: number, msg: unknown) => {
} as chrome.tabs.Tab, if ((msg as { type?: string }).type === RECORD_STOP) stopTabs.push(tabId);
]); return { ok: true };
const cdp = makeFakeCdp(undefined, {
treesByTab: {
[TAB_ID]: axTree("First tab", ["First action"]),
5: axTree("Second tab", ["Second action"]),
},
}); });
const deps = { tabsApi, sendToTab, cdp: makeFakeCdp() };
await handleRecordStart(manager, RECORD_START_V3, deps);
attachRecordQueryListener(deps);
await new Promise<void>((resolve) => {
chromeApi.runtimeOnMessage.emit(
{ type: RECORD_QUERY },
{ tab: secondTab } as chrome.runtime.MessageSender,
() => resolve(),
);
});
tabsApi.close(5);
const stopped = await handleRecordStop(manager, { session_id: "abcd" }, deps);
const stoppedTrace = (stopped as RecordStopResult).trace;
expect("version" in stoppedTrace && stoppedTrace.version).toBe(3);
expect(stopTabs).toEqual([TAB_ID]);
});
it("waits for an acknowledged in-flight rearm before stopping armed tabs", async () => {
const chromeApi = installChrome();
const manager = fakeManager();
const firstTab = {
id: TAB_ID,
windowId: AGENT_WINDOW_ID,
active: true,
status: "complete",
url: START_URL,
} as chrome.tabs.Tab;
const secondTab = {
id: 5,
windowId: AGENT_WINDOW_ID,
active: false,
status: "complete",
url: "https://example.com/second",
} as chrome.tabs.Tab;
const tabsApi = makeMultiTabsApi([firstTab, secondTab]);
let releaseSecondStart!: () => void;
const secondStartGate = new Promise<void>((resolve) => {
releaseSecondStart = resolve;
});
let announceSecondStart!: () => void;
const secondStartBegan = new Promise<void>((resolve) => {
announceSecondStart = resolve;
});
const stopTabs: number[] = [];
const sendToTab = vi.fn(async (tabId: number, msg: unknown) => {
const type = (msg as { type?: string }).type;
if (type === RECORD_START && tabId === 5) {
announceSecondStart();
await secondStartGate;
}
if (type === RECORD_STOP) stopTabs.push(tabId);
return { ok: true };
});
const deps = { tabsApi, sendToTab, cdp: makeFakeCdp() };
await handleRecordStart(manager, RECORD_START_V3, deps);
attachRecordQueryListener(deps);
chromeApi.runtimeOnMessage.emit(
{ type: RECORD_QUERY },
{ tab: secondTab } as chrome.runtime.MessageSender,
() => {},
);
await secondStartBegan;
const stopping = handleRecordStop(manager, { session_id: "abcd" }, deps);
await new Promise((resolve) => setTimeout(resolve, 50));
releaseSecondStart();
const stopped = await stopping;
const stoppedTrace = (stopped as RecordStopResult).trace;
expect("version" in stoppedTrace && stoppedTrace.version).toBe(3);
expect(stopTabs).toEqual([TAB_ID, 5]);
});
it("drains a navigation callback received while finish is settling", async () => {
const chromeApi = installChrome();
const manager = fakeManager();
const firstTab = {
id: TAB_ID,
windowId: AGENT_WINDOW_ID,
active: true,
status: "complete",
url: START_URL,
title: "Editor",
} as chrome.tabs.Tab;
const secondUrl = "https://example.com/late-navigation";
const secondTab = {
id: 5,
windowId: AGENT_WINDOW_ID,
active: false,
status: "complete",
url: secondUrl,
title: "Late page",
} as chrome.tabs.Tab;
let releaseLookup!: () => void;
const lookupGate = new Promise<void>((resolve) => {
releaseLookup = resolve;
});
const baseTabsApi = makeMultiTabsApi([firstTab, secondTab]);
const tabsApi = {
...baseTabsApi,
get: async (tabId: number) => {
if (tabId === 5) await lookupGate;
return baseTabsApi.get(tabId);
},
};
const cdp = makeFakeCdp([axTree("Editor", ["Save"]), axTree("Saved")]);
let requestId = ""; let requestId = "";
const sendToTab = vi.fn(async (_tabId: number, message: unknown) => { const sendToTab = vi.fn(async (_tabId: number, msg: unknown) => {
const typed = message as { type?: string; requestId?: string }; const typed = msg as { type?: string; requestId?: string };
if (typed.type === RECORD_START && typed.requestId) requestId = typed.requestId; if (typed.type === RECORD_START && typed.requestId) requestId = typed.requestId;
return { ok: true }; return { ok: true };
}); });
@@ -1252,352 +1320,80 @@ describe("recorded user steps reach the exported trace", () => {
await handleRecordStart(manager, RECORD_START_V3, deps); await handleRecordStart(manager, RECORD_START_V3, deps);
attachRecordStepListener(deps); attachRecordStepListener(deps);
tabsApi.activate(5);
chromeApi.tabsOnActivated.emit({ tabId: 5, windowId: AGENT_WINDOW_ID });
await settleWait();
runtimeOnMessageEmit(
chromeApi,
requestId,
{
op: "click",
page_url: secondUrl,
target: { role: "button", name: "Second action", tag: "button" },
},
5,
);
await settleWait();
tabsApi.activate(TAB_ID);
chromeApi.tabsOnActivated.emit({ tabId: TAB_ID, windowId: AGENT_WINDOW_ID });
await settleWait();
runtimeOnMessageEmit(chromeApi, requestId, { runtimeOnMessageEmit(chromeApi, requestId, {
op: "click", op: "click",
page_url: START_URL, page_url: START_URL,
target: { role: "button", name: "First action", tag: "button" }, target: { role: "button", name: "Save", tag: "button" },
}); });
await settleWait();
const stopped = await handleRecordStop(manager, { session_id: "abcd" }, deps); const stopping = handleRecordStop(manager, { session_id: "abcd" }, deps);
const trace = asTraceV3((stopped as RecordStopResult).trace);
const stateUrl = (stateId: string) => trace.states.find((state) => state.id === stateId)?.url;
const switches = trace.steps.filter((step) => step.op === "switch_tab");
const clicks = trace.steps.filter((step) => step.op === "click");
expect(switches).toHaveLength(2);
expect(clicks).toHaveLength(2);
expect(stateUrl(switches[0]!.state)).toBe(START_URL);
expect(stateUrl(switches[0]!.result.state)).toBe(secondUrl);
expect(stateUrl(clicks[0]!.state)).toBe(secondUrl);
expect(stateUrl(switches[1]!.result.state)).toBe(START_URL);
expect(stateUrl(clicks[1]!.state)).toBe(START_URL);
});
it("keeps switch_tab internal when the v3 caller did not advertise support", async () => {
const chromeApi = installChrome();
const manager = fakeManager();
const tabsApi = makeMultiTabsApi([
{
id: TAB_ID,
windowId: AGENT_WINDOW_ID,
active: true,
status: "complete",
url: START_URL,
} as chrome.tabs.Tab,
{
id: 5,
windowId: AGENT_WINDOW_ID,
active: false,
status: "complete",
url: "https://example.com/second",
} as chrome.tabs.Tab,
]);
const deps = { tabsApi, sendToTab: vi.fn(async () => ({ ok: true })), cdp: makeFakeCdp() };
await handleRecordStart(
manager,
{ session_id: "abcd", url: START_URL, trace_version: 3 },
deps,
);
tabsApi.activate(5);
chromeApi.tabsOnActivated.emit({ tabId: 5, windowId: AGENT_WINDOW_ID });
await settleWait();
const stopped = await handleRecordStop(manager, { session_id: "abcd" }, deps);
const trace = asTraceV3((stopped as RecordStopResult).trace);
expect(trace.steps.some((step) => step.op === "switch_tab")).toBe(false);
});
it("keeps the latest rapid tab activation when an earlier rearm acknowledges late", async () => {
const chromeApi = installChrome();
const manager = fakeManager();
const tabsApi = makeMultiTabsApi(
[TAB_ID, 5, 6].map(
(id) =>
({
id,
windowId: AGENT_WINDOW_ID,
active: id === TAB_ID,
status: "complete",
url: id === TAB_ID ? START_URL : `https://example.com/tab-${id}`,
}) as chrome.tabs.Tab,
),
);
let releaseSecond!: () => void;
const secondGate = new Promise<void>((resolve) => {
releaseSecond = resolve;
});
let announceSecond!: () => void;
const secondStarted = new Promise<void>((resolve) => {
announceSecond = resolve;
});
const sendToTab = vi.fn(async (tabId: number, message: unknown) => {
if ((message as { type?: string }).type === RECORD_START && tabId === 5) {
announceSecond();
await secondGate;
}
return { ok: true };
});
const deps = { tabsApi, sendToTab, cdp: makeFakeCdp() };
await handleRecordStart(manager, RECORD_START_V3, deps);
tabsApi.activate(5);
chromeApi.tabsOnActivated.emit({ tabId: 5, windowId: AGENT_WINDOW_ID });
await secondStarted;
tabsApi.activate(6);
chromeApi.tabsOnActivated.emit({ tabId: 6, windowId: AGENT_WINDOW_ID });
await settleWait();
releaseSecond();
await new Promise((resolve) => setTimeout(resolve, 50));
const stopped = await handleRecordStop(manager, { session_id: "abcd" }, deps);
const trace = asTraceV3((stopped as RecordStopResult).trace);
const switches = trace.steps.filter((step) => step.op === "switch_tab");
const stateUrl = (stateId: string) => trace.states.find((state) => state.id === stateId)?.url;
expect(switches).toHaveLength(1);
expect(stateUrl(switches[0]!.result.state)).toBe("https://example.com/tab-6");
});
it("does not retry an in-flight tab rearm after recording stops", async () => {
const chromeApi = installChrome();
const manager = fakeManager();
const tabsApi = makeMultiTabsApi([
{
id: TAB_ID,
windowId: AGENT_WINDOW_ID,
active: true,
status: "complete",
url: START_URL,
} as chrome.tabs.Tab,
{
id: 5,
windowId: AGENT_WINDOW_ID,
active: false,
status: "complete",
url: "https://example.com/second",
} as chrome.tabs.Tab,
]);
let failRearm!: () => void;
const rearmGate = new Promise<never>((_resolve, reject) => {
failRearm = () => reject(new Error("rearm failed after stop"));
});
let announceRearm!: () => void;
const rearmStarted = new Promise<void>((resolve) => {
announceRearm = resolve;
});
const sendToTab = vi.fn(async (tabId: number, message: unknown) => {
if ((message as { type?: string }).type === RECORD_START && tabId === 5) {
announceRearm();
await rearmGate;
}
return { ok: true };
});
const deps = { tabsApi, sendToTab, cdp: makeFakeCdp() };
await handleRecordStart(manager, RECORD_START_V3, deps);
tabsApi.activate(5);
chromeApi.tabsOnActivated.emit({ tabId: 5, windowId: AGENT_WINDOW_ID });
await rearmStarted;
const stoppedPromise = handleRecordStop(manager, { session_id: "abcd" }, deps);
await vi.waitFor(() => {
expect(sendToTab).toHaveBeenCalledWith(5, expect.objectContaining({ type: RECORD_STOP }));
});
failRearm();
const stopped = await stoppedPromise;
await new Promise((resolve) => setTimeout(resolve, 600));
const startsForSecondTab = sendToTab.mock.calls.filter(
([tabId, message]) => tabId === 5 && (message as { type?: string }).type === RECORD_START,
);
expect(startsForSecondTab).toHaveLength(1);
expect(asTraceV3((stopped as RecordStopResult).trace).steps).toHaveLength(0);
});
it("keeps pending action settles scoped to their tab during an immediate switch", async () => {
const chromeApi = installChrome();
const manager = fakeManager();
const secondUrl = "https://example.com/second";
const tabsApi = makeMultiTabsApi([
{
id: TAB_ID,
windowId: AGENT_WINDOW_ID,
active: true,
status: "complete",
url: START_URL,
} as chrome.tabs.Tab,
{
id: 5,
windowId: AGENT_WINDOW_ID,
active: false,
status: "complete",
url: secondUrl,
} as chrome.tabs.Tab,
]);
let requestId = "";
const sendToTab = vi.fn(async (_tabId: number, message: unknown) => {
const typed = message as { type?: string; requestId?: string };
if (typed.type === RECORD_START && typed.requestId) requestId = typed.requestId;
return { ok: true };
});
const deps = { tabsApi, sendToTab, cdp: makeFakeCdp() };
await handleRecordStart(manager, RECORD_START_V3, deps);
attachRecordStepListener(deps);
runtimeOnMessageEmit(chromeApi, requestId, {
op: "click",
page_url: START_URL,
target: { role: "button", name: "First action", tag: "button" },
});
tabsApi.activate(5);
chromeApi.tabsOnActivated.emit({ tabId: 5, windowId: AGENT_WINDOW_ID });
runtimeOnMessageEmit(
chromeApi,
requestId,
{
op: "click",
page_url: secondUrl,
target: { role: "button", name: "Second action", tag: "button" },
},
5,
);
await settleWait();
const stopped = await handleRecordStop(manager, { session_id: "abcd" }, deps);
const trace = asTraceV3((stopped as RecordStopResult).trace);
const stateUrl = (stateId: string) => trace.states.find((state) => state.id === stateId)?.url;
expect(trace.steps.map((step) => step.op)).toEqual(["click", "switch_tab", "click"]);
expect(stateUrl(trace.steps[0]!.result.state)).toBe(START_URL);
expect(stateUrl(trace.steps[1]!.result.state)).toBe(secondUrl);
expect(stateUrl(trace.steps[2]!.state)).toBe(secondUrl);
});
it("tracks the same navigation URL independently for each tab", async () => {
const chromeApi = installChrome();
const manager = fakeManager();
const tabsApi = makeMultiTabsApi([
{
id: TAB_ID,
windowId: AGENT_WINDOW_ID,
active: true,
status: "complete",
url: START_URL,
} as chrome.tabs.Tab,
{
id: 5,
windowId: AGENT_WINDOW_ID,
active: false,
status: "complete",
url: "https://example.com/second",
} as chrome.tabs.Tab,
]);
const deps = { tabsApi, sendToTab: vi.fn(async () => ({ ok: true })), cdp: makeFakeCdp() };
await handleRecordStart(manager, RECORD_START_V3, deps);
tabsApi.activate(5);
chromeApi.tabsOnActivated.emit({ tabId: 5, windowId: AGENT_WINDOW_ID });
await settleWait();
tabsApi.goTo(5, START_URL, "Same URL in second tab");
chromeApi.webNavigationOnCommitted.emit({ chromeApi.webNavigationOnCommitted.emit({
tabId: 5, tabId: 5,
frameId: 0, frameId: 0,
url: secondUrl,
transitionType: "link",
transitionQualifiers: [],
} as unknown as chrome.webNavigation.WebNavigationTransitionCallbackDetails);
const earlyStop = await Promise.race([
stopping.then((result) => ({ result })),
new Promise<null>((resolve) => setTimeout(() => resolve(null), 500)),
]);
releaseLookup();
const stopped = earlyStop?.result ?? (await stopping);
const trace = asTraceV3((stopped as RecordStopResult).trace);
expect(trace.steps.some((step) => step.op === "navigate" && step.to === secondUrl)).toBe(true);
const navigation = trace.steps.find((step) => step.op === "navigate" && step.to === secondUrl);
expect(navigation?.result.state).toBeTruthy();
}, 15_000);
it("captures a new-tab navigation from the emitting tab", async () => {
const chromeApi = installChrome();
const manager = fakeManager();
const firstTab = {
id: TAB_ID,
windowId: AGENT_WINDOW_ID,
active: true,
status: "complete",
url: START_URL, url: START_URL,
transitionType: "typed", title: "Old tab",
transitionQualifiers: ["from_address_bar"], } as chrome.tabs.Tab;
} as unknown as chrome.webNavigation.WebNavigationTransitionCallbackDetails); const secondUrl = "https://example.com/new-tab";
await settleWait(); const secondTab = {
id: 5,
const stopped = await handleRecordStop(manager, { session_id: "abcd" }, deps); windowId: AGENT_WINDOW_ID,
const trace = asTraceV3((stopped as RecordStopResult).trace); active: false,
expect( status: "complete",
trace.steps.filter((step) => step.op === "navigate" && step.to === START_URL), url: secondUrl,
).toHaveLength(1); title: "New tab",
}); } as chrome.tabs.Tab;
const tabsApi = makeMultiTabsApi([firstTab, secondTab]);
it("drops late input and frame navigation from an inactive tab", async () => { const cdp = makeFakeCdp(undefined, {
const chromeApi = installChrome(); treesByTab: {
const manager = fakeManager(); [TAB_ID]: axTree("Old tab", ["Old action"]),
const secondUrl = "https://example.com/second"; 5: axTree("New tab", ["New action"]),
const tabsApi = makeMultiTabsApi([ },
{
id: TAB_ID,
windowId: AGENT_WINDOW_ID,
active: true,
status: "complete",
url: START_URL,
} as chrome.tabs.Tab,
{
id: 5,
windowId: AGENT_WINDOW_ID,
active: false,
status: "complete",
url: secondUrl,
} as chrome.tabs.Tab,
]);
let requestId = "";
const sendToTab = vi.fn(async (_tabId: number, message: unknown) => {
const typed = message as { type?: string; requestId?: string };
if (typed.type === RECORD_START && typed.requestId) requestId = typed.requestId;
return { ok: true };
}); });
const deps = { tabsApi, sendToTab, cdp: makeFakeCdp() }; const sendToTab = vi.fn(async () => ({ ok: true }));
const deps = { tabsApi, sendToTab, cdp };
await handleRecordStart(manager, RECORD_START_V3, deps); await handleRecordStart(manager, RECORD_START_V3, deps);
attachRecordStepListener(deps);
tabsApi.activate(5);
chromeApi.tabsOnActivated.emit({ tabId: 5, windowId: AGENT_WINDOW_ID });
await settleWait();
const lateAck = runtimeOnMessageEmit(
chromeApi,
requestId,
{
op: "fill",
page_url: START_URL,
target: { role: "textbox", name: "Draft", tag: "input" },
value: "late",
},
TAB_ID,
false,
);
expect(lateAck).toHaveBeenCalledWith({ ok: true, sequence: 1 });
chromeApi.webNavigationOnCommitted.emit({
tabId: TAB_ID,
frameId: 0,
url: "https://example.com/late",
transitionType: "link",
transitionQualifiers: [],
} as unknown as chrome.webNavigation.WebNavigationTransitionCallbackDetails);
chromeApi.webNavigationOnCommitted.emit({ chromeApi.webNavigationOnCommitted.emit({
tabId: 5, tabId: 5,
frameId: 7, frameId: 0,
url: "https://frame.example/late", url: secondUrl,
transitionType: "link", transitionType: "link",
transitionQualifiers: [], transitionQualifiers: [],
} as unknown as chrome.webNavigation.WebNavigationTransitionCallbackDetails); } as unknown as chrome.webNavigation.WebNavigationTransitionCallbackDetails);
await new Promise((resolve) => setTimeout(resolve, 50)); await settleWait();
const stopped = await handleRecordStop(manager, { session_id: "abcd" }, deps); const stopped = await handleRecordStop(manager, { session_id: "abcd" }, deps);
const trace = asTraceV3((stopped as RecordStopResult).trace); const trace = asTraceV3((stopped as RecordStopResult).trace);
expect(trace.steps.map((step) => step.op)).toEqual(["switch_tab"]); const navigation = trace.steps.find((step) => step.op === "navigate" && step.to === secondUrl);
}); const landed = trace.states.find((state) => state.id === navigation?.result.state);
expect(landed?.url).toBe(secondUrl);
expect(landed?.body).toContain("New tab");
expect(landed?.body).not.toContain("Old action");
}, 15_000);
}); });
@@ -66,52 +66,6 @@ function makeApis(
} }
describe("handleSessionStop with auto-return", () => { describe("handleSessionStop with auto-return", () => {
it("leaves the session untouched when cancellation arrived before teardown", async () => {
const aw = fakeAgentWindow([100]);
const sm = new SessionManager({ agentWindow: aw });
await sm.start("aa11");
const cdp = { detachSession: vi.fn(async () => {}) };
const controller = new AbortController();
controller.abort();
const res = await handleSessionStop(
sm,
{ session_id: "aa11" },
{ cdp, signal: controller.signal },
);
expect(res).toMatchObject({ code: "cancelled" });
expect(sm.has("aa11")).toBe(true);
expect(cdp.detachSession).not.toHaveBeenCalled();
expect(aw.remove).not.toHaveBeenCalled();
});
it("does not close the Agent Window when cancellation lands during CDP detach", async () => {
const aw = fakeAgentWindow([100]);
const sm = new SessionManager({ agentWindow: aw });
await sm.start("aa11");
const controller = new AbortController();
let finishDetach: () => void = () => {};
const detach = new Promise<void>((resolve) => {
finishDetach = resolve;
});
const cdp = { detachSession: vi.fn(() => detach) };
const stopping = handleSessionStop(
sm,
{ session_id: "aa11" },
{ cdp, signal: controller.signal },
);
await vi.waitFor(() => expect(cdp.detachSession).toHaveBeenCalledWith("aa11"));
controller.abort();
finishDetach();
const res = await stopping;
expect(res).toMatchObject({ code: "cancelled" });
expect(sm.has("aa11")).toBe(true);
expect(aw.remove).not.toHaveBeenCalled();
});
it("returns every borrowed tab and closes the Agent Window in the right order", async () => { it("returns every borrowed tab and closes the Agent Window in the right order", async () => {
const aw = fakeAgentWindow([100]); const aw = fakeAgentWindow([100]);
const sm = new SessionManager({ agentWindow: aw }); const sm = new SessionManager({ agentWindow: aw });
@@ -46,25 +46,6 @@ describe("lookupSnapshotRef", () => {
}); });
describe("resolveSnapshotRef", () => { describe("resolveSnapshotRef", () => {
it("preserves the frame and child session needed to route iframe refs", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
ctx.refStore.set("e3", 1234, {
tabId: 4,
frameId: "child-frame",
cdpSessionId: "child-session",
});
const expected = {
backendNodeId: 1234,
refKey: "e3",
frameId: "child-frame",
cdpSessionId: "child-session",
};
expect(lookupSnapshotRef(ctx, "@e3", 4)).toEqual(expected);
expect(resolveSnapshotRef(ctx, "@e3", 4)).toEqual(expected);
});
it("returns not_found for unknown ref", async () => { it("returns not_found for unknown ref", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11"); const ctx = await sm.start("aa11");
@@ -250,10 +250,9 @@ const DEFAULT_NOTIFICATION_COPY: BorrowNotificationCopy = {
// URL injection-eligibility test // URL injection-eligibility test
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
const EXTENSION_STORE_RES = [ const CHROME_WEB_STORE_RES = [
/^https:\/\/chrome\.google\.com\/webstore/i, /^https:\/\/chrome\.google\.com\/webstore/i,
/^https:\/\/chromewebstore\.google\.com/i, /^https:\/\/chromewebstore\.google\.com/i,
/^https:\/\/microsoftedge\.microsoft\.com\/addons/i,
]; ];
/** /**
@@ -265,15 +264,14 @@ const EXTENSION_STORE_RES = [
* - `ftp://` no longer hosts content scripts reliably in modern Chrome. * - `ftp://` no longer hosts content scripts reliably in modern Chrome.
* - `about:` / `chrome:` / `chrome-extension:` / `edge:` / `devtools:` / * - `about:` / `chrome:` / `chrome-extension:` / `edge:` / `devtools:` /
* `view-source:` / `data:` / `blob:` are blocked by the platform. * `view-source:` / `data:` / `blob:` are blocked by the platform.
* - Extension storefronts are blocked by their own browser: the Chrome Web * - The Chrome Web Store (both legacy and new domains) is also blocked.
* Store (legacy and new domains) under Chrome, and Edge Add-ons under Edge.
* *
* Mirrors the implicit scheme list documented in * Mirrors the implicit scheme list documented in
* https://developer.chrome.com/docs/extensions/develop/concepts/match-patterns. * https://developer.chrome.com/docs/extensions/develop/concepts/match-patterns.
*/ */
export function isInjectableContentScriptUrl(url: string | undefined): boolean { export function isInjectableContentScriptUrl(url: string | undefined): boolean {
if (!url) return false; if (!url) return false;
for (const re of EXTENSION_STORE_RES) { for (const re of CHROME_WEB_STORE_RES) {
if (re.test(url)) return false; if (re.test(url)) return false;
} }
return /^https?:\/\//i.test(url); return /^https?:\/\//i.test(url);
@@ -1,9 +1,101 @@
export { import { type RenderedRef, renderVom, type VomOptions } from "@browser-skill/vom";
type CaptureVomObservationOptions, import type { CdpAxNode } from "./observation";
captureVomObservation, import { buildVomScene } from "./observation";
} from "./observation"; import type { CdpRunner } from "./shared";
export { import {
type CaptureVomFrame, type CapturedNode,
type CaptureVomMatchNode, type CapturedSurfaceProbe,
type CaptureVomObservationResult, type CapturedViewModel,
} from "./vom/record-safe-observation"; captureViewModel,
collectOverlayExcludedBackendIds,
} from "./vom/capture";
export interface CaptureVomObservationOptions extends VomOptions {
conditionalSurfaceProbe?: boolean;
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
}
export interface CaptureVomObservationResult {
text: string;
refs: RenderedRef[];
truncated: boolean;
captured: CapturedNode[];
surfaceProbes?: CapturedSurfaceProbe[];
}
function emptyCapturedViewModel(viewport = { width: 0, height: 0 }): CapturedViewModel {
return { viewport, nodes: [], iframeNodes: new Map(), excludedBackendNodeIds: new Set() };
}
interface LayoutMetricsViewportReply {
cssLayoutViewport?: { clientWidth?: number; clientHeight?: number };
layoutViewport?: { clientWidth?: number; clientHeight?: number };
}
async function fallbackCapturedViewModel(
cdp: CdpRunner,
tabId: number,
): Promise<CapturedViewModel> {
let viewport = { width: 0, height: 0 };
try {
const metrics = await cdp.send<LayoutMetricsViewportReply>(tabId, "Page.getLayoutMetrics", {});
const vpSrc = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
viewport = {
width: vpSrc.clientWidth ?? 0,
height: vpSrc.clientHeight ?? 0,
};
} catch {
// viewport stays zero-sized
}
const excludedBackendNodeIds = await collectOverlayExcludedBackendIds(cdp, tabId);
return { ...emptyCapturedViewModel(viewport), excludedBackendNodeIds };
}
async function captureForVom(
cdp: CdpRunner,
tabId: number,
conditionalSurfaceProbe: boolean,
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>,
): Promise<CapturedViewModel> {
try {
return await captureViewModel(cdp, tabId, {
conditionalSurfaceProbe,
hoverProbeBypassOverlay,
});
} catch {
return fallbackCapturedViewModel(cdp, tabId);
}
}
/** Pure VOM capture pipeline shared by snapshot, observe, and recording. */
export async function captureVomObservation(
cdp: CdpRunner,
tabId: number,
url: string | undefined,
opts: CaptureVomObservationOptions = {},
): Promise<CaptureVomObservationResult> {
await cdp.ensureAttachedToUrl?.(tabId, url);
await cdp.send<unknown>(tabId, "Accessibility.enable", {});
const result = await cdp.send<{ nodes: CdpAxNode[] }>(tabId, "Accessibility.getFullAXTree", {});
const axNodes = result.nodes ?? [];
const captured = await captureForVom(
cdp,
tabId,
opts.conditionalSurfaceProbe ?? false,
opts.hoverProbeBypassOverlay,
);
const scene = buildVomScene(axNodes, captured, { pageUrl: url });
const rendered = renderVom(scene, {
maxDepth: opts.maxDepth,
maxTokens: opts.maxTokens,
redactValues: opts.redactValues,
activeRegionPolicy: opts.activeRegionPolicy ?? true,
});
return {
text: rendered.text,
refs: rendered.refs,
truncated: rendered.truncated,
captured: captured.nodes,
surfaceProbes: captured.surfaceProbes,
};
}
+38 -40
View File
@@ -51,12 +51,7 @@ import {
handleScreenshot, handleScreenshot,
handleSnapshot, handleSnapshot,
} from "./observation"; } from "./observation";
import { import { handleRecordAwait, handleRecordStart, handleRecordStop } from "./record";
handleRecordAwait,
handleRecordStart,
handleRecordStop,
type RecordRuntimeDeps,
} from "./record";
import { import {
handleSessionStart, handleSessionStart,
handleSessionStop, handleSessionStop,
@@ -104,7 +99,6 @@ export interface DispatcherDeps {
transport: Transport; transport: Transport;
sessions: SessionManager; sessions: SessionManager;
cdp?: DispatcherCdpRunner; cdp?: DispatcherCdpRunner;
recording?: RecordRuntimeDeps;
/** /**
* Invoked whenever a dispatched RPC may have changed the live * Invoked whenever a dispatched RPC may have changed the live
* session set (currently `tool.session_start` and * session set (currently `tool.session_start` and
@@ -140,7 +134,6 @@ export class ToolDispatcher {
private readonly transport: Transport; private readonly transport: Transport;
private readonly sessions: SessionManager; private readonly sessions: SessionManager;
private readonly cdp?: DispatcherCdpRunner; private readonly cdp?: DispatcherCdpRunner;
private readonly recording?: RecordRuntimeDeps;
private readonly onSessionsChanged?: () => void; private readonly onSessionsChanged?: () => void;
private readonly onBrowserControlResumed?: (sessionId: string) => void; private readonly onBrowserControlResumed?: (sessionId: string) => void;
private readonly approveBorrow?: BorrowConfirmationApprover; private readonly approveBorrow?: BorrowConfirmationApprover;
@@ -160,7 +153,6 @@ export class ToolDispatcher {
this.transport = deps.transport; this.transport = deps.transport;
this.sessions = deps.sessions; this.sessions = deps.sessions;
this.cdp = deps.cdp; this.cdp = deps.cdp;
this.recording = deps.recording;
this.onSessionsChanged = deps.onSessionsChanged; this.onSessionsChanged = deps.onSessionsChanged;
this.onBrowserControlResumed = deps.onBrowserControlResumed; this.onBrowserControlResumed = deps.onBrowserControlResumed;
this.approveBorrow = deps.approveBorrow; this.approveBorrow = deps.approveBorrow;
@@ -298,7 +290,6 @@ export class ToolDispatcher {
await this.releaseHoverLatch((req.params as SessionStopParams).session_id); await this.releaseHoverLatch((req.params as SessionStopParams).session_id);
return handleSessionStop(this.sessions, req.params as SessionStopParams, { return handleSessionStop(this.sessions, req.params as SessionStopParams, {
cdp: this.cdp, cdp: this.cdp,
signal,
}); });
} }
case "tool.tab_list": case "tool.tab_list":
@@ -383,11 +374,7 @@ export class ToolDispatcher {
? { ? {
cdp: this.cdp, cdp: this.cdp,
tabsApi: chromeTabsCaptureApi, tabsApi: chromeTabsCaptureApi,
// Active hover probing is opt-in. A held hover latch still conditionalSurfaceProbe: !this.hasHoverLatchForScope(hoverScope),
// suppresses it, because probing would move the cursor off
// the element the caller is deliberately holding.
conditionalSurfaceProbe:
params.probe_hover === true && !this.hasHoverLatchForScope(hoverScope),
hoverProbeBypassOverlay: bypassOverlay, hoverProbeBypassOverlay: bypassOverlay,
} }
: undefined, : undefined,
@@ -543,26 +530,44 @@ export class ToolDispatcher {
signal, signal,
}); });
case "tool.record_start": case "tool.record_start":
return this.recording return handleRecordStart(this.sessions, req.params as RecordStartParams, {
? handleRecordStart(this.sessions, req.params as RecordStartParams, { tabsApi: chromeTabsApi,
...this.recording, sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
signal, bypassOverlay: async (tabId, enabled) => {
}) try {
: recordingRuntimeUnavailable(); await chrome.tabs.sendMessage(tabId, {
type: OVERLAY_AUTOMATION_BYPASS,
enabled,
});
} catch {
// Content script may be unavailable on restricted pages.
}
},
...(this.cdp ? { cdp: this.cdp } : {}),
signal,
});
case "tool.record_stop": case "tool.record_stop":
return this.recording return handleRecordStop(this.sessions, req.params as RecordStopParams, {
? handleRecordStop(this.sessions, req.params as RecordStopParams, { tabsApi: chromeTabsApi,
...this.recording, sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
signal, bypassOverlay: async (tabId, enabled) => {
}) try {
: recordingRuntimeUnavailable(); await chrome.tabs.sendMessage(tabId, {
type: OVERLAY_AUTOMATION_BYPASS,
enabled,
});
} catch {
// Content script may be unavailable on restricted pages.
}
},
signal,
});
case "tool.record_await": case "tool.record_await":
return this.recording return handleRecordAwait(this.sessions, req.params as RecordAwaitParams, {
? handleRecordAwait(this.sessions, req.params as RecordAwaitParams, { tabsApi: chromeTabsApi,
...this.recording, sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
signal, signal,
}) });
: recordingRuntimeUnavailable();
default: default:
return { return {
code: "unknown_method", code: "unknown_method",
@@ -698,13 +703,6 @@ function isRpcError(v: unknown): v is RpcError {
); );
} }
function recordingRuntimeUnavailable(): RpcError {
return {
code: "protocol_error",
message: "recording runtime is unavailable",
};
}
function sessionIdForBrowserControlMethod(req: RequestFrame): string | null { function sessionIdForBrowserControlMethod(req: RequestFrame): string | null {
switch (req.method) { switch (req.method) {
case "tool.tab_create": case "tool.tab_create":
+96 -41
View File
@@ -1,15 +1,9 @@
// Resolve and scroll nodes inside one CDP target. Cross-frame projection // Shared CDP helpers for scrolling nodes into view and computing
// belongs to frame-geometry.ts. // viewport-space bounding rectangles. Used by interaction tools (click
// centre) and observation tools (element screenshot clip).
import type { RpcError } from "@/transport/types"; import type { RpcError } from "@/transport/types";
import { rpcError } from "./errors"; import { rpcError } from "./errors";
import {
parseCdpQuad,
polygonBounds,
type Region,
rectPolygon,
type ViewportRect,
} from "./geometry";
import { type CdpRunner, isRpcError } from "./shared"; import { type CdpRunner, isRpcError } from "./shared";
const ELEMENT_NOT_VISIBLE_MESSAGE = const ELEMENT_NOT_VISIBLE_MESSAGE =
@@ -19,6 +13,45 @@ function elementNotVisibleError(): RpcError {
return rpcError("permission_denied", "element_not_visible", ELEMENT_NOT_VISIBLE_MESSAGE); return rpcError("permission_denied", "element_not_visible", ELEMENT_NOT_VISIBLE_MESSAGE);
} }
/** Viewport-space axis-aligned bounding box for CDP `Page.captureScreenshot` clip. */
export interface ViewportRect {
x: number;
y: number;
width: number;
height: number;
}
/**
* Compute the axis-aligned bounding box of an 8-double CDP content
* quad / box-model polygon. Exported for unit tests.
*/
export function quadBoundingRect(quad: number[]): ViewportRect | null {
if (quad.length !== 8) return null;
const xs = [quad[0], quad[2], quad[4], quad[6]];
const ys = [quad[1], quad[3], quad[5], quad[7]];
const minX = Math.min(...xs);
const maxX = Math.max(...xs);
const minY = Math.min(...ys);
const maxY = Math.max(...ys);
const width = maxX - minX;
const height = maxY - minY;
if (width <= 0 || height <= 0) return null;
return { x: minX, y: minY, width, height };
}
function rectToQuad(rect: ViewportRect): number[] {
return [
rect.x,
rect.y,
rect.x + rect.width,
rect.y,
rect.x + rect.width,
rect.y + rect.height,
rect.x,
rect.y + rect.height,
];
}
/** /**
* Resolve `backendNodeId` → CDP `objectId` so we can invoke * Resolve `backendNodeId` → CDP `objectId` so we can invoke
* `Runtime.callFunctionOn` against the live JS object. * `Runtime.callFunctionOn` against the live JS object.
@@ -79,24 +112,48 @@ export async function scrollNodeIntoView(
} }
} }
function validCdpRegion(quads: number[][] | undefined): Region { /**
return (quads ?? []).flatMap((raw) => { * Viewport-space bounding box for `backendNodeId`. Prefers
const polygon = parseCdpQuad(raw); * `DOM.getContentQuads`, then `DOM.getBoxModel`, then the union of
return polygon && polygonBounds(polygon) ? [polygon] : []; * visible descendant client rects for zero-size overflow containers.
}); */
} export async function nodeBoundingRect(
async function resolveVisibleContentRegion(
cdp: CdpRunner, cdp: CdpRunner,
tabId: number, tabId: number,
backendNodeId: number, backendNodeId: number,
): Promise<Region | RpcError> { ): Promise<ViewportRect | RpcError> {
const polygon = await visibleContentPolygon(cdp, tabId, backendNodeId);
if (isRpcError(polygon)) return polygon;
// visibleContentPolygon only returns quads with positive area.
return quadBoundingRect(polygon)!;
}
/**
* Compute the centroid of a CDP content quad. Quads are reported as
* 8 doubles in clockwise order: (x1, y1, x2, y2, x3, y3, x4, y4).
*/
export function quadCentre(quad: number[]): { x: number; y: number } {
const x = (quad[0] + quad[4]) / 2;
const y = (quad[1] + quad[5]) / 2;
return { x, y };
}
/** Centre of a `DOM.getBoxModel` `content` polygon (also 8 doubles). */
export function boxCentre(box: number[]): { x: number; y: number } {
return quadCentre(box);
}
async function visibleContentPolygon(
cdp: CdpRunner,
tabId: number,
backendNodeId: number,
): Promise<number[] | RpcError> {
try { try {
const quads = await cdp.send<{ quads?: number[][] }>(tabId, "DOM.getContentQuads", { const quads = await cdp.send<{ quads?: number[][] }>(tabId, "DOM.getContentQuads", {
backendNodeId, backendNodeId,
}); });
const visible = validCdpRegion(quads.quads); const quad = quads.quads?.find((q) => q.length === 8 && quadBoundingRect(q) !== null);
if (visible.length > 0) return visible; if (quad) return quad;
} catch (err) { } catch (err) {
console.debug("[bsk element-geometry] getContentQuads failed", err); console.debug("[bsk element-geometry] getContentQuads failed", err);
} }
@@ -105,34 +162,15 @@ async function resolveVisibleContentRegion(
backendNodeId, backendNodeId,
}); });
const content = box.model?.content; const content = box.model?.content;
const fallback = validCdpRegion(content ? [content] : undefined); if (content && content.length === 8 && quadBoundingRect(content)) return content;
if (fallback.length > 0) return fallback;
} catch (err) { } catch (err) {
console.debug("[bsk element-geometry] getBoxModel failed", err); console.debug("[bsk element-geometry] getBoxModel failed", err);
} }
const descendantRect = await descendantBoundingRect(cdp, tabId, backendNodeId); const descendantRect = await descendantBoundingRect(cdp, tabId, backendNodeId);
if (descendantRect) { if (descendantRect) return rectToQuad(descendantRect);
return [
rectPolygon({
x: descendantRect.x,
y: descendantRect.y,
w: descendantRect.width,
h: descendantRect.height,
}),
];
}
return elementNotVisibleError(); return elementNotVisibleError();
} }
/** Resolve every visible content region in the node's target-local viewport. */
export async function nodeContentRegion(
cdp: CdpRunner,
tabId: number,
backendNodeId: number,
): Promise<Region | RpcError> {
return resolveVisibleContentRegion(cdp, tabId, backendNodeId);
}
async function descendantBoundingRect( async function descendantBoundingRect(
cdp: CdpRunner, cdp: CdpRunner,
tabId: number, tabId: number,
@@ -203,3 +241,20 @@ function parseViewportRect(value: unknown): ViewportRect | null {
if (![x, y, width, height].every(Number.isFinite) || width <= 0 || height <= 0) return null; if (![x, y, width, height].every(Number.isFinite) || width <= 0 || height <= 0) return null;
return { x, y, width, height }; return { x, y, width, height };
} }
/**
* Get the click point for `backendNodeId`. Prefers
* `DOM.getContentQuads` (richer for rotated / transformed elements);
* falls back to `DOM.getBoxModel`, then visible descendant bounds.
* Returns `permission_denied` with "element not visible" when every
* geometry path comes back empty / fails.
*/
export async function nodeCentre(
cdp: CdpRunner,
tabId: number,
backendNodeId: number,
): Promise<{ x: number; y: number } | RpcError> {
const polygon = await visibleContentPolygon(cdp, tabId, backendNodeId);
if (isRpcError(polygon)) return polygon;
return quadCentre(polygon);
}
-290
View File
@@ -1,290 +0,0 @@
import { type CdpFrame, type CdpFrameGraph, type CdpTarget } from "@/browser-driver/frame-graph";
import type { RpcError } from "@/transport/types";
import { nodeContentRegion, scrollNodeIntoView } from "./element-geometry";
import {
clipPolygon,
type GeometryProjection,
type Point,
type Polygon,
type ProjectiveEdge,
parseCdpQuad,
polygonArea,
polygonCentroid,
projectRegionToViewport,
type Quad,
type Region,
rectPolygon,
regionBounds,
type Size,
type ViewportRect,
} from "./geometry";
import { type CdpRunner, cdpRunnerForTarget, isRpcError, sendToCdpTarget } from "./shared";
export interface NodeAddress {
target: CdpTarget;
backendNodeId: number;
frameId?: string;
}
export interface ResolvedNodeGeometry {
topVisibleRegions: Region;
topBounds: ViewportRect;
actionPoint: Point;
}
function geometryError(message: string): RpcError {
return { code: "cdp_failed", message };
}
function frameMap(graph: CdpFrameGraph): Map<string, CdpFrame> {
return new Map(graph.frames.map((frame) => [frame.frameId, frame]));
}
function targetRootFrame(byId: Map<string, CdpFrame>, frame: CdpFrame): CdpFrame | null {
const seen = new Set<string>();
let current = frame;
while (current.parentFrameId) {
if (seen.has(current.frameId)) return null;
seen.add(current.frameId);
const parent = byId.get(current.parentFrameId);
if (!parent || parent.target.sessionId !== current.target.sessionId) break;
current = parent;
}
return current;
}
function frameAncestry(graph: CdpFrameGraph, frameId: string): CdpFrame[] | null {
const byId = frameMap(graph);
const path: CdpFrame[] = [];
const seen = new Set<string>();
let current = byId.get(frameId);
if (!current) return null;
while (current.parentFrameId) {
if (seen.has(current.frameId)) return null;
seen.add(current.frameId);
path.push(current);
const parent = byId.get(current.parentFrameId);
if (!parent) return null;
current = parent;
}
return path;
}
async function targetViewport(cdp: CdpRunner, target: CdpTarget): Promise<Size | null> {
const metrics = await sendToCdpTarget<{
cssLayoutViewport?: { clientWidth?: number; clientHeight?: number };
layoutViewport?: { clientWidth?: number; clientHeight?: number };
}>(cdp, target, "Page.getLayoutMetrics", {});
const viewport = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
const width = viewport.clientWidth ?? 0;
const height = viewport.clientHeight ?? 0;
return width > 0 && height > 0 ? { width, height } : null;
}
async function ownerContentQuad(
cdp: CdpRunner,
parent: CdpFrame,
ownerBackendNodeId: number,
): Promise<Quad | null> {
const result = await sendToCdpTarget<{ model?: { content?: number[] } }>(
cdp,
parent.target,
"DOM.getBoxModel",
{ backendNodeId: ownerBackendNodeId },
);
return parseCdpQuad(result.model?.content);
}
async function sameTargetFrameClips(
cdp: CdpRunner,
byId: Map<string, CdpFrame>,
frame: CdpFrame,
targetRoot: CdpFrame,
): Promise<Polygon[] | null> {
const clips: Polygon[] = [];
let current = frame;
const seen = new Set<string>();
while (current.frameId !== targetRoot.frameId) {
if (seen.has(current.frameId) || !current.parentFrameId) return null;
seen.add(current.frameId);
const parent = byId.get(current.parentFrameId);
if (!parent || current.ownerBackendNodeId === undefined) return null;
const clip = await ownerContentQuad(cdp, parent, current.ownerBackendNodeId);
if (!clip) return null;
clips.push(clip);
current = parent;
}
return clips;
}
export async function resolveFrameProjection(
cdp: CdpRunner,
graph: CdpFrameGraph,
frameId: string,
): Promise<GeometryProjection | null> {
const byId = frameMap(graph);
const frame = byId.get(frameId);
if (!frame) return null;
let targetRoot = targetRootFrame(byId, frame);
if (!targetRoot) return null;
const sourceViewport = await targetViewport(cdp, targetRoot.target);
if (!sourceViewport) return null;
const sourceClips = await sameTargetFrameClips(cdp, byId, frame, targetRoot);
if (!sourceClips) return null;
const edges: ProjectiveEdge[] = [];
while (targetRoot.parentFrameId) {
const parent = byId.get(targetRoot.parentFrameId);
if (!parent || targetRoot.ownerBackendNodeId === undefined) return null;
const destinationQuad = await ownerContentQuad(cdp, parent, targetRoot.ownerBackendNodeId);
if (!destinationQuad) return null;
const source =
edges.length === 0 ? sourceViewport : await targetViewport(cdp, targetRoot.target);
if (!source) return null;
const parentTargetRoot = targetRootFrame(byId, parent);
if (!parentTargetRoot) return null;
const destinationClips = await sameTargetFrameClips(cdp, byId, parent, parentTargetRoot);
if (!destinationClips) return null;
edges.push({ sourceViewport: source, destinationQuad, destinationClips });
targetRoot = parentTargetRoot;
}
const topViewport =
edges.length === 0 ? sourceViewport : await targetViewport(cdp, targetRoot.target);
return topViewport ? { sourceClips, edges, topViewport } : null;
}
async function loadFrameGraph(cdp: CdpRunner, tabId: number): Promise<CdpFrameGraph | null> {
if (!cdp.getFrameGraph) return null;
try {
return await cdp.getFrameGraph(tabId);
} catch (error) {
console.debug("[bsk frame-geometry] frame graph resolution failed", error);
return null;
}
}
async function scrollFrameOwners(
cdp: CdpRunner,
tabId: number,
graph: CdpFrameGraph,
frameId: string,
): Promise<RpcError | null> {
const byId = frameMap(graph);
const ancestry = frameAncestry(graph, frameId);
if (!ancestry) return geometryError(`could not resolve frame ancestry for ${frameId}`);
for (const child of [...ancestry].reverse()) {
const parent = child.parentFrameId ? byId.get(child.parentFrameId) : undefined;
if (!parent || child.ownerBackendNodeId === undefined) {
return geometryError(`could not resolve frame owner for ${child.frameId}`);
}
const error = await scrollNodeIntoView(
cdpRunnerForTarget(cdp, parent.target),
tabId,
child.ownerBackendNodeId,
);
if (error) return error;
}
return null;
}
async function scrollElementWithFrameGraph(
cdp: CdpRunner,
tabId: number,
target: CdpTarget,
backendNodeId: number,
frameId: string | undefined,
graph: CdpFrameGraph | null,
): Promise<RpcError | null> {
if (frameId) {
if (!graph) return geometryError(`could not resolve frame graph for ${frameId}`);
const error = await scrollFrameOwners(cdp, tabId, graph, frameId);
if (error) return error;
} else if (target.sessionId) {
return geometryError("an OOPIF node address requires frameId");
}
return scrollNodeIntoView(cdpRunnerForTarget(cdp, target), tabId, backendNodeId);
}
export async function scrollElementAndFramesIntoView(
cdp: CdpRunner,
tabId: number,
target: CdpTarget,
backendNodeId: number,
frameId?: string,
): Promise<RpcError | null> {
const graph = frameId ? await loadFrameGraph(cdp, tabId) : null;
return scrollElementWithFrameGraph(cdp, tabId, target, backendNodeId, frameId, graph);
}
function largestRegion(regions: Region): Polygon | null {
let largest: { polygon: Polygon; area: number } | null = null;
for (const polygon of regions) {
const area = polygonArea(polygon);
if (area <= 0) continue;
if (!largest || area > largest.area) largest = { polygon, area };
}
return largest?.polygon ?? null;
}
export async function resolveNodeGeometry(
cdp: CdpRunner,
tabId: number,
address: NodeAddress,
options: { scrollIntoView?: boolean } = {},
): Promise<ResolvedNodeGeometry | RpcError> {
try {
if (address.target.sessionId && !address.frameId) {
return geometryError("an OOPIF node address requires frameId");
}
const graph = address.frameId ? await loadFrameGraph(cdp, tabId) : null;
if (address.frameId && !graph) {
return geometryError(`could not resolve frame graph for ${address.frameId}`);
}
if (options.scrollIntoView) {
const scrollError = await scrollElementWithFrameGraph(
cdp,
tabId,
address.target,
address.backendNodeId,
address.frameId,
graph,
);
if (scrollError) return scrollError;
}
const localRegion = await nodeContentRegion(
cdpRunnerForTarget(cdp, address.target),
tabId,
address.backendNodeId,
);
if (isRpcError(localRegion)) return localRegion;
let topVisibleRegions: Region;
if (address.frameId && graph) {
const projection = await resolveFrameProjection(cdp, graph, address.frameId);
if (!projection)
return geometryError(`could not resolve frame geometry for ${address.frameId}`);
topVisibleRegions = projectRegionToViewport(localRegion, projection);
} else {
const viewport = await targetViewport(cdp, address.target);
if (!viewport) return geometryError("could not resolve top viewport geometry");
topVisibleRegions = localRegion
.map((polygon) =>
clipPolygon(polygon, rectPolygon({ x: 0, y: 0, w: viewport.width, h: viewport.height })),
)
.filter((polygon) => polygon.length >= 3);
}
const topBounds = regionBounds(topVisibleRegions);
const actionRegion = largestRegion(topVisibleRegions);
const actionPoint = actionRegion ? polygonCentroid(actionRegion) : null;
if (!topBounds || !actionPoint) {
return { code: "permission_denied", message: "element not visible" };
}
return { topVisibleRegions, topBounds, actionPoint };
} catch (error) {
return geometryError(error instanceof Error ? error.message : String(error));
}
}
-266
View File
@@ -1,266 +0,0 @@
export interface Point {
x: number;
y: number;
}
export type Polygon = Point[];
export type Region = Polygon[];
export type Quad = [Point, Point, Point, Point];
export interface Size {
width: number;
height: number;
}
export interface ViewportRect {
x: number;
y: number;
width: number;
height: number;
}
export interface ProjectiveEdge {
sourceViewport: Size;
destinationQuad: Quad;
destinationClips?: Polygon[];
}
export interface GeometryProjection {
sourceClips: Polygon[];
edges: ProjectiveEdge[];
topViewport: Size;
}
export function parseCdpQuad(raw: number[] | undefined): Quad | null {
if (!raw || raw.length !== 8 || !raw.every(Number.isFinite)) return null;
return [
{ x: raw[0], y: raw[1] },
{ x: raw[2], y: raw[3] },
{ x: raw[4], y: raw[5] },
{ x: raw[6], y: raw[7] },
];
}
export function rectPolygon(rect: { x: number; y: number; w: number; h: number }): Quad {
return [
{ x: rect.x, y: rect.y },
{ x: rect.x + rect.w, y: rect.y },
{ x: rect.x + rect.w, y: rect.y + rect.h },
{ x: rect.x, y: rect.y + rect.h },
];
}
export function viewportPolygon(viewport: Size): Quad {
return rectPolygon({ x: 0, y: 0, w: viewport.width, h: viewport.height });
}
/** Project a normalized unit-square point into an arbitrary content quad. */
export function projectUnitPoint(point: Point, quad: Quad): Point {
const [p0, p1, p2, p3] = quad;
const dx1 = p1.x - p2.x;
const dx2 = p3.x - p2.x;
const dx3 = p0.x - p1.x + p2.x - p3.x;
const dy1 = p1.y - p2.y;
const dy2 = p3.y - p2.y;
const dy3 = p0.y - p1.y + p2.y - p3.y;
let a: number;
let b: number;
let c: number;
let d: number;
let e: number;
let f: number;
let g = 0;
let h = 0;
if (Math.abs(dx3) < 1e-9 && Math.abs(dy3) < 1e-9) {
a = p1.x - p0.x;
b = p3.x - p0.x;
c = p0.x;
d = p1.y - p0.y;
e = p3.y - p0.y;
f = p0.y;
} else {
const denominator = dx1 * dy2 - dx2 * dy1;
if (Math.abs(denominator) < 1e-9) {
return {
x: p0.x + (p1.x - p0.x) * point.x + (p3.x - p0.x) * point.y,
y: p0.y + (p1.y - p0.y) * point.x + (p3.y - p0.y) * point.y,
};
}
g = (dx3 * dy2 - dx2 * dy3) / denominator;
h = (dx1 * dy3 - dx3 * dy1) / denominator;
a = p1.x - p0.x + g * p1.x;
b = p3.x - p0.x + h * p3.x;
c = p0.x;
d = p1.y - p0.y + g * p1.y;
e = p3.y - p0.y + h * p3.y;
f = p0.y;
}
const scale = g * point.x + h * point.y + 1;
return {
x: (a * point.x + b * point.y + c) / scale,
y: (d * point.x + e * point.y + f) / scale,
};
}
export function projectPolygon(polygon: Polygon, edge: ProjectiveEdge): Polygon {
const { width, height } = edge.sourceViewport;
if (width <= 0 || height <= 0) return [];
return polygon.map((point) =>
projectUnitPoint({ x: point.x / width, y: point.y / height }, edge.destinationQuad),
);
}
function polygonSignedArea(points: Polygon): number {
let area = 0;
for (let index = 0; index < points.length; index += 1) {
const current = points[index];
const next = points[(index + 1) % points.length];
area += current.x * next.y - next.x * current.y;
}
return area / 2;
}
function edgeIntersection(start: Point, end: Point, clipStart: Point, clipEnd: Point): Point {
const subjectX = end.x - start.x;
const subjectY = end.y - start.y;
const clipX = clipEnd.x - clipStart.x;
const clipY = clipEnd.y - clipStart.y;
const denominator = subjectX * clipY - subjectY * clipX;
if (Math.abs(denominator) < 1e-9) return end;
const offsetX = clipStart.x - start.x;
const offsetY = clipStart.y - start.y;
const scale = (offsetX * clipY - offsetY * clipX) / denominator;
return { x: start.x + scale * subjectX, y: start.y + scale * subjectY };
}
/** Clip a polygon to a convex polygon, preserving every resulting vertex. */
export function clipPolygon(subject: Polygon, clip: Polygon): Polygon {
if (subject.length < 3 || clip.length < 3) return [];
const orientation = polygonSignedArea(clip) >= 0 ? 1 : -1;
let output = [...subject];
for (let edge = 0; edge < clip.length && output.length > 0; edge += 1) {
const clipStart = clip[edge];
const clipEnd = clip[(edge + 1) % clip.length];
const input = output;
output = [];
const inside = (point: Point) =>
orientation *
((clipEnd.x - clipStart.x) * (point.y - clipStart.y) -
(clipEnd.y - clipStart.y) * (point.x - clipStart.x)) >=
-1e-9;
let start = input[input.length - 1];
for (const end of input) {
const startInside = inside(start);
const endInside = inside(end);
if (endInside) {
if (!startInside) output.push(edgeIntersection(start, end, clipStart, clipEnd));
output.push(end);
} else if (startInside) {
output.push(edgeIntersection(start, end, clipStart, clipEnd));
}
start = end;
}
}
return output;
}
export function polygonBounds(points: Polygon): ViewportRect | null {
if (points.length < 3) return null;
const xs = points.map((point) => point.x);
const ys = points.map((point) => point.y);
const x = Math.min(...xs);
const y = Math.min(...ys);
const right = Math.max(...xs);
const bottom = Math.max(...ys);
if (right <= x || bottom <= y) return null;
return { x, y, width: right - x, height: bottom - y };
}
export function regionBounds(region: Region): ViewportRect | null {
return polygonBounds(region.flat());
}
export function polygonArea(points: Polygon): number {
return Math.abs(polygonSignedArea(points));
}
export function polygonCentroid(points: Polygon): Point | null {
if (points.length < 3) return null;
let twiceArea = 0;
let x = 0;
let y = 0;
for (let index = 0; index < points.length; index += 1) {
const current = points[index];
const next = points[(index + 1) % points.length];
const cross = current.x * next.y - next.x * current.y;
twiceArea += cross;
x += (current.x + next.x) * cross;
y += (current.y + next.y) * cross;
}
if (Math.abs(twiceArea) < 1e-9) return null;
return { x: x / (3 * twiceArea), y: y / (3 * twiceArea) };
}
export function projectAndClipRegion(
region: Region,
edges: ProjectiveEdge[],
topViewport: Size,
): Region {
let projected = region;
for (const edge of edges) {
const sourceClip = viewportPolygon(edge.sourceViewport);
projected = projected
.map((polygon) => clipPolygon(polygon, sourceClip))
.filter((polygon) => polygon.length >= 3)
.map((polygon) =>
[edge.destinationQuad, ...(edge.destinationClips ?? [])].reduce(
(current, clip) => clipPolygon(current, clip),
projectPolygon(polygon, edge),
),
)
.filter((polygon) => polygon.length >= 3);
}
const topClip = viewportPolygon(topViewport);
return projected
.map((polygon) => clipPolygon(polygon, topClip))
.filter((polygon) => polygon.length >= 3);
}
export function projectRegionToViewport(region: Region, projection: GeometryProjection): Region {
const clipped = region
.map((polygon) =>
projection.sourceClips.reduce((current, clip) => clipPolygon(current, clip), polygon),
)
.filter((polygon) => polygon.length >= 3);
return projectAndClipRegion(clipped, projection.edges, projection.topViewport);
}
export function projectRectToViewport(
rect: { x: number; y: number; w: number; h: number } | null,
projection: GeometryProjection,
): ViewportRect | null {
if (!rect) return null;
return regionBounds(projectRegionToViewport([rectPolygon(rect)], projection));
}
export function childFrameProjection(
parent: GeometryProjection,
ownerRectInParent: { x: number; y: number; w: number; h: number },
): GeometryProjection {
const destinationQuad = rectPolygon(ownerRectInParent);
return {
sourceClips: [],
edges: [
{
sourceViewport: { width: ownerRectInParent.w, height: ownerRectInParent.h },
destinationQuad,
destinationClips: parent.sourceClips,
},
...parent.edges,
],
topViewport: parent.topViewport,
};
}
+68 -62
View File
@@ -8,7 +8,6 @@ import {
HELP_REQUEST, HELP_REQUEST,
type HelpCancelMessage, type HelpCancelMessage,
type HelpFinishMessage, type HelpFinishMessage,
type HelpHighlightRect,
type HelpQueryResponse, type HelpQueryResponse,
type HelpRequestMessage, type HelpRequestMessage,
isHelpAckMessage, isHelpAckMessage,
@@ -26,7 +25,6 @@ import type {
ResolvedTarget, ResolvedTarget,
RpcError, RpcError,
} from "@/transport/types"; } from "@/transport/types";
import { resolveNodeGeometry } from "./frame-geometry";
import { import {
type CdpRunner, type CdpRunner,
type ChromeTabsApi, type ChromeTabsApi,
@@ -39,6 +37,7 @@ import {
import { lookupSnapshotRef } from "./snapshot-ref"; import { lookupSnapshotRef } from "./snapshot-ref";
const DEFAULT_HELP_TIMEOUT_MS = 300_000; const DEFAULT_HELP_TIMEOUT_MS = 300_000;
const HELP_ATTR = "data-bsk-help";
const HELP_SEND_RETRIES = 3; const HELP_SEND_RETRIES = 3;
const HELP_SEND_RETRY_DELAY_MS = 350; const HELP_SEND_RETRY_DELAY_MS = 350;
const HELP_REARM_DEBOUNCE_MS = 150; const HELP_REARM_DEBOUNCE_MS = 150;
@@ -75,7 +74,6 @@ interface ActiveHelpRequest {
title?: string; title?: string;
targets: HelpTarget[]; targets: HelpTarget[];
selectors: string[]; selectors: string[];
rects: HelpHighlightRect[];
timeoutMs: number; timeoutMs: number;
notificationId: string; notificationId: string;
resolvedTargets?: ResolvedTarget[]; resolvedTargets?: ResolvedTarget[];
@@ -136,7 +134,6 @@ function helpRequestMessage(help: ActiveHelpRequest, tabId: number): HelpRequest
...(help.title ? { title: help.title } : {}), ...(help.title ? { title: help.title } : {}),
displayMode: isPrimaryTab ? "full" : "compact", displayMode: isPrimaryTab ? "full" : "compact",
selectors: isPrimaryTab ? help.selectors : [], selectors: isPrimaryTab ? help.selectors : [],
rects: isPrimaryTab ? help.rects : [],
timeoutMs: help.timeoutMs, timeoutMs: help.timeoutMs,
}; };
} }
@@ -168,6 +165,33 @@ async function sendHelpRequestWithAck(
throw lastError ?? new Error("failed to show help overlay"); throw lastError ?? new Error("failed to show help overlay");
} }
async function tagRefTarget(
cdp: CdpRunner,
tabId: number,
backendNodeId: number,
index: number,
): Promise<string | null> {
let objectId: string | undefined;
try {
const resolved = await cdp.send<{ object?: { objectId?: string } }>(tabId, "DOM.resolveNode", {
backendNodeId,
});
objectId = resolved.object?.objectId;
if (!objectId) return null;
await cdp.send(tabId, "Runtime.callFunctionOn", {
objectId,
functionDeclaration: `function(){ this.setAttribute("${HELP_ATTR}", "${index}"); }`,
});
return `[${HELP_ATTR}="${index}"]`;
} catch {
return null;
} finally {
if (objectId) {
await cdp.send(tabId, "Runtime.releaseObject", { objectId }).catch(() => {});
}
}
}
async function selectorExists( async function selectorExists(
cdp: CdpRunner, cdp: CdpRunner,
tabId: number, tabId: number,
@@ -185,22 +209,36 @@ async function selectorExists(
} }
} }
async function clearRefTags(cdp: CdpRunner | undefined, tabId: number): Promise<void> {
if (!cdp) return;
try {
const doc = await cdp.send<{ root?: { nodeId?: number } }>(tabId, "DOM.getDocument", {
depth: 0,
});
const rootId = doc.root?.nodeId;
if (rootId === undefined) return;
const { nodeIds } = await cdp.send<{ nodeIds: number[] }>(tabId, "DOM.querySelectorAll", {
nodeId: rootId,
selector: `[${HELP_ATTR}]`,
});
for (const nodeId of nodeIds ?? []) {
await cdp.send(tabId, "DOM.removeAttribute", { nodeId, name: HELP_ATTR }).catch(() => {});
}
} catch {
// Best-effort cleanup.
}
}
async function resolveHelpTargets( async function resolveHelpTargets(
ctx: ReturnType<SessionManager["get"]>, ctx: ReturnType<SessionManager["get"]>,
tabId: number, tabId: number,
targets: HelpTarget[], targets: HelpTarget[],
deps: RequestHelpDeps, deps: RequestHelpDeps,
options: { scrollIntoView: boolean }, ): Promise<{ selectors: string[]; resolvedTargets?: ResolvedTarget[] }> {
): Promise<{
selectors: string[];
rects: HelpHighlightRect[];
resolvedTargets?: ResolvedTarget[];
}> {
const selectors: string[] = []; const selectors: string[] = [];
const rects: HelpHighlightRect[] = [];
const resolved: ResolvedTarget[] = []; const resolved: ResolvedTarget[] = [];
let rootNodeId: number | undefined; let rootNodeId: number | undefined;
if (deps.cdp && targets.some((target) => Boolean(target.selector))) { if (deps.cdp && targets.length > 0) {
try { try {
const doc = await deps.cdp.send<{ root?: { nodeId?: number } }>(tabId, "DOM.getDocument", { const doc = await deps.cdp.send<{ root?: { nodeId?: number } }>(tabId, "DOM.getDocument", {
depth: 0, depth: 0,
@@ -224,55 +262,28 @@ async function resolveHelpTargets(
resolved.push({ matched, selector: tgt.selector }); resolved.push({ matched, selector: tgt.selector });
} else if (tgt.ref) { } else if (tgt.ref) {
const looked = ctx ? lookupSnapshotRef(ctx, tgt.ref, tabId) : null; const looked = ctx ? lookupSnapshotRef(ctx, tgt.ref, tabId) : null;
let rect: HelpHighlightRect | null = null; const backendNodeId = looked?.backendNodeId ?? null;
if (looked && deps.cdp) { let sel: string | null = null;
const target = { if (backendNodeId !== null && deps.cdp) {
tabId, sel = await tagRefTarget(deps.cdp, tabId, backendNodeId, i);
...(looked.cdpSessionId ? { sessionId: looked.cdpSessionId } : {}),
};
const geometry = await resolveNodeGeometry(
deps.cdp,
tabId,
{
target,
backendNodeId: looked.backendNodeId,
...(looked.frameId ? { frameId: looked.frameId } : {}),
},
{ scrollIntoView: options.scrollIntoView },
);
if (!isRpcError(geometry)) {
rect = {
top: geometry.topBounds.y,
left: geometry.topBounds.x,
width: geometry.topBounds.width,
height: geometry.topBounds.height,
};
}
} }
if (rect) rects.push(rect); if (sel) selectors.push(sel);
resolved.push({ resolved.push({ matched: sel !== null, ref: tgt.ref });
matched: rect !== null,
ref: tgt.ref,
});
} }
} }
return { selectors, rects, resolvedTargets: resolved.length > 0 ? resolved : undefined }; return { selectors, resolvedTargets: resolved.length > 0 ? resolved : undefined };
} }
async function refreshHelpTargets( async function refreshHelpTargets(help: ActiveHelpRequest): Promise<void> {
help: ActiveHelpRequest, await clearRefTags(help.deps.cdp, help.primaryTabId);
options: { scrollIntoView: boolean } = { scrollIntoView: true }, const { selectors, resolvedTargets } = await resolveHelpTargets(
): Promise<void> {
const { selectors, rects, resolvedTargets } = await resolveHelpTargets(
help.ctx, help.ctx,
help.primaryTabId, help.primaryTabId,
help.targets, help.targets,
help.deps, help.deps,
options,
); );
help.selectors = selectors; help.selectors = selectors;
help.rects = rects;
help.resolvedTargets = resolvedTargets; help.resolvedTargets = resolvedTargets;
} }
@@ -283,7 +294,12 @@ async function cleanupHelp(help: ActiveHelpRequest): Promise<void> {
const tabsToCancel = new Set([help.primaryTabId, ...help.overlayTabIds]); const tabsToCancel = new Set([help.primaryTabId, ...help.overlayTabIds]);
await Promise.all( await Promise.all(
[...tabsToCancel].map((tabId) => [...tabsToCancel].map((tabId) =>
help.deps.sendToTab(tabId, { type: HELP_CANCEL, requestId: help.requestId }).catch(() => {}), Promise.all([
clearRefTags(help.deps.cdp, tabId),
help.deps
.sendToTab(tabId, { type: HELP_CANCEL, requestId: help.requestId })
.catch(() => {}),
]),
), ),
); );
} }
@@ -460,9 +476,7 @@ function attachHelpRuntimeListener(deps: RequestHelpDeps): () => void {
sendResponse({ active: false }); sendResponse({ active: false });
return; return;
} }
if (tabId === help.primaryTabId) { if (tabId === help.primaryTabId) await refreshHelpTargets(help);
await refreshHelpTargets(help, { scrollIntoView: false });
}
help.overlayTabIds.add(tabId); help.overlayTabIds.add(tabId);
sendResponse({ sendResponse({
active: true, active: true,
@@ -472,7 +486,6 @@ function attachHelpRuntimeListener(deps: RequestHelpDeps): () => void {
...(help.title ? { title: help.title } : {}), ...(help.title ? { title: help.title } : {}),
displayMode: tabId === help.primaryTabId ? "full" : "compact", displayMode: tabId === help.primaryTabId ? "full" : "compact",
selectors: tabId === help.primaryTabId ? help.selectors : [], selectors: tabId === help.primaryTabId ? help.selectors : [],
rects: tabId === help.primaryTabId ? help.rects : [],
timeoutMs: help.timeoutMs, timeoutMs: help.timeoutMs,
}, },
}); });
@@ -673,13 +686,7 @@ export async function handleRequestHelp(
if (deps.autoAttachLifecycle !== false) ensureHelpLifecycleListeners(deps); if (deps.autoAttachLifecycle !== false) ensureHelpLifecycleListeners(deps);
const tabId = target.tabId; const tabId = target.tabId;
const initialTargets = params.targets ?? []; const initialTargets = params.targets ?? [];
const { selectors, rects, resolvedTargets } = await resolveHelpTargets( const { selectors, resolvedTargets } = await resolveHelpTargets(ctx, tabId, initialTargets, deps);
ctx,
tabId,
initialTargets,
deps,
{ scrollIntoView: true },
);
await deps.windows.update(target.windowId, { focused: true }).catch(() => {}); await deps.windows.update(target.windowId, { focused: true }).catch(() => {});
await deps.activateTab(tabId).catch(() => {}); await deps.activateTab(tabId).catch(() => {});
@@ -722,7 +729,6 @@ export async function handleRequestHelp(
...(params.title ? { title: params.title } : {}), ...(params.title ? { title: params.title } : {}),
targets: initialTargets, targets: initialTargets,
selectors, selectors,
rects,
timeoutMs, timeoutMs,
notificationId, notificationId,
resolvedTargets, resolvedTargets,
+56 -88
View File
@@ -3,15 +3,14 @@
// //
// All interaction tools: // All interaction tools:
// 1. Resolve target tab (sandbox: must be inside Agent Window). // 1. Resolve target tab (sandbox: must be inside Agent Window).
// 2. Resolve target element by `ref` (compound frame/session identity) // 2. Resolve target element by `ref` (RefStore.resolve with tabId
// or `selector` (DOM.querySelector + describeNode). // binding) or `selector` (DOM.querySelector + describeNode).
// 3. Scroll the node and its frame owners into view, project its live // 3. Scroll the node into view, then dispatch the appropriate
// content quad into the top viewport, then dispatch `Input.*` events. // `Input.*` CDP events.
// 4. Honour `AbortSignal` so canceled calls don't issue follow-up CDP // 4. Honour `AbortSignal` so canceled calls don't issue follow-up CDP
// commands. // commands.
import { ChromiumCdp } from "@/browser-driver/chromium-cdp"; import { ChromiumCdp } from "@/browser-driver/chromium-cdp";
import type { CdpTarget } from "@/browser-driver/frame-graph";
import type { SessionContext, SessionManager } from "@/session-manager/manager"; import type { SessionContext, SessionManager } from "@/session-manager/manager";
import type { import type {
ClickParams, ClickParams,
@@ -29,13 +28,17 @@ import type {
SelectResult, SelectResult,
} from "@/transport/types"; } from "@/transport/types";
import { attachDialogs, markDialogCursor } from "./dialogs"; import { attachDialogs, markDialogCursor } from "./dialogs";
import { backendNodeToObject } from "./element-geometry"; import {
backendNodeToObject,
boxCentre,
nodeCentre,
quadCentre,
scrollNodeIntoView,
} from "./element-geometry";
import { rpcError } from "./errors"; import { rpcError } from "./errors";
import { resolveNodeGeometry, scrollElementAndFramesIntoView } from "./frame-geometry";
import { import {
type CdpRunner, type CdpRunner,
type ChromeTabsApi, type ChromeTabsApi,
cdpRunnerForTarget,
chromeTabsApi, chromeTabsApi,
enforceAgentWindow, enforceAgentWindow,
isRpcError, isRpcError,
@@ -134,16 +137,7 @@ async function resolveBackendNode(
target: { tabId: number }, target: { tabId: number },
params: { ref?: string; selector?: string }, params: { ref?: string; selector?: string },
toolName: string, toolName: string,
): Promise< ): Promise<{ backendNodeId: number; usedRef?: string; usedSelector?: string } | RpcError> {
| {
backendNodeId: number;
cdpTarget: CdpTarget;
frameId?: string;
usedRef?: string;
usedSelector?: string;
}
| RpcError
> {
const hasRef = typeof params.ref === "string" && params.ref.length > 0; const hasRef = typeof params.ref === "string" && params.ref.length > 0;
const hasSelector = typeof params.selector === "string" && params.selector.length > 0; const hasSelector = typeof params.selector === "string" && params.selector.length > 0;
if (hasRef && hasSelector) { if (hasRef && hasSelector) {
@@ -161,15 +155,7 @@ async function resolveBackendNode(
if (hasRef) { if (hasRef) {
const resolved = resolveSnapshotRef(ctx, params.ref as string, target.tabId); const resolved = resolveSnapshotRef(ctx, params.ref as string, target.tabId);
if (isRpcError(resolved)) return resolved; if (isRpcError(resolved)) return resolved;
return { return { backendNodeId: resolved.backendNodeId, usedRef: resolved.refKey };
backendNodeId: resolved.backendNodeId,
cdpTarget: {
tabId: target.tabId,
...(resolved.cdpSessionId ? { sessionId: resolved.cdpSessionId } : {}),
},
...(resolved.frameId ? { frameId: resolved.frameId } : {}),
usedRef: resolved.refKey,
};
} }
// selector path // selector path
try { try {
@@ -206,7 +192,7 @@ async function resolveBackendNode(
message: "DOM.describeNode returned no backendNodeId", message: "DOM.describeNode returned no backendNodeId",
}; };
} }
return { backendNodeId, cdpTarget: { tabId: target.tabId }, usedSelector: params.selector }; return { backendNodeId, usedSelector: params.selector };
} catch (err) { } catch (err) {
return { return {
code: "cdp_failed", code: "cdp_failed",
@@ -242,19 +228,19 @@ export async function handleClick(
return { code: "cancelled", message: "click aborted" }; return { code: "cancelled", message: "click aborted" };
} }
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId); try {
const geometry = await resolveNodeGeometry( deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
deps.cdp, const scrollErr = await scrollNodeIntoView(deps.cdp, target.tabId, node.backendNodeId);
target.tabId, if (scrollErr) return scrollErr;
{ } catch (err) {
target: node.cdpTarget, return {
backendNodeId: node.backendNodeId, code: "cdp_failed",
...(node.frameId ? { frameId: node.frameId } : {}), message: err instanceof Error ? err.message : String(err),
}, };
{ scrollIntoView: true }, }
);
if (isRpcError(geometry)) return geometry; const centre = await nodeCentre(deps.cdp, target.tabId, node.backendNodeId);
const centre = geometry.actionPoint; if (isRpcError(centre)) return centre;
if (throwIfAborted(deps.signal)) { if (throwIfAborted(deps.signal)) {
return { code: "cancelled", message: "click aborted" }; return { code: "cancelled", message: "click aborted" };
@@ -367,19 +353,19 @@ export async function handleHover(
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "hover"); const node = await resolveBackendNode(deps.cdp, ctx, target, params, "hover");
if (isRpcError(node)) return node; if (isRpcError(node)) return node;
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId); try {
const geometry = await resolveNodeGeometry( deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
deps.cdp, const scrollErr = await scrollNodeIntoView(deps.cdp, target.tabId, node.backendNodeId);
target.tabId, if (scrollErr) return scrollErr;
{ } catch (err) {
target: node.cdpTarget, return {
backendNodeId: node.backendNodeId, code: "cdp_failed",
...(node.frameId ? { frameId: node.frameId } : {}), message: err instanceof Error ? err.message : String(err),
}, };
{ scrollIntoView: true }, }
);
if (isRpcError(geometry)) return geometry; const centre = await nodeCentre(deps.cdp, target.tabId, node.backendNodeId);
const centre = geometry.actionPoint; if (isRpcError(centre)) return centre;
if (throwIfAborted(deps.signal)) { if (throwIfAborted(deps.signal)) {
return { code: "cancelled", message: "hover aborted" }; return { code: "cancelled", message: "hover aborted" };
@@ -599,11 +585,10 @@ export async function handleFill(
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "fill"); const node = await resolveBackendNode(deps.cdp, ctx, target, params, "fill");
if (isRpcError(node)) return node; if (isRpcError(node)) return node;
const nodeCdp = cdpRunnerForTarget(deps.cdp, node.cdpTarget);
try { try {
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId); deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
const described = await nodeCdp.send<{ node?: DescribedNode }>( const described = await deps.cdp.send<{ node?: DescribedNode }>(
target.tabId, target.tabId,
"DOM.describeNode", "DOM.describeNode",
{ {
@@ -617,18 +602,12 @@ export async function handleFill(
`element ${described.node?.nodeName ?? "?"} not fillable (need input/textarea/contenteditable)`, `element ${described.node?.nodeName ?? "?"} not fillable (need input/textarea/contenteditable)`,
); );
} }
const scrollErr = await scrollElementAndFramesIntoView( const scrollErr = await scrollNodeIntoView(deps.cdp, target.tabId, node.backendNodeId);
deps.cdp,
target.tabId,
node.cdpTarget,
node.backendNodeId,
node.frameId,
);
if (scrollErr) return scrollErr; if (scrollErr) return scrollErr;
if (throwIfAborted(deps.signal)) { if (throwIfAborted(deps.signal)) {
return { code: "cancelled", message: "fill aborted" }; return { code: "cancelled", message: "fill aborted" };
} }
await nodeCdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId }); await deps.cdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
} catch (err) { } catch (err) {
return { return {
code: "cdp_failed", code: "cdp_failed",
@@ -640,7 +619,7 @@ export async function handleFill(
return { code: "cancelled", message: "fill aborted" }; return { code: "cancelled", message: "fill aborted" };
} }
const objectIdOrErr = await backendNodeToObject(nodeCdp, target.tabId, node.backendNodeId); const objectIdOrErr = await backendNodeToObject(deps.cdp, target.tabId, node.backendNodeId);
if (isRpcError(objectIdOrErr)) return objectIdOrErr; if (isRpcError(objectIdOrErr)) return objectIdOrErr;
const objectId = objectIdOrErr; const objectId = objectIdOrErr;
const clearBefore = params.clear_before ?? true; const clearBefore = params.clear_before ?? true;
@@ -649,7 +628,7 @@ export async function handleFill(
if (clearBefore) { if (clearBefore) {
// Clear input/textarea value or wipe contenteditable innerText, // Clear input/textarea value or wipe contenteditable innerText,
// then fire `input` so frameworks observe the empty state. // then fire `input` so frameworks observe the empty state.
await nodeCdp.send(target.tabId, "Runtime.callFunctionOn", { await deps.cdp.send(target.tabId, "Runtime.callFunctionOn", {
objectId, objectId,
functionDeclaration: `function() { functionDeclaration: `function() {
if (this.isContentEditable) { this.textContent = ''; } if (this.isContentEditable) { this.textContent = ''; }
@@ -676,7 +655,7 @@ export async function handleFill(
return { code: "cancelled", message: "fill aborted" }; return { code: "cancelled", message: "fill aborted" };
} }
// Fire `input` + `change` so React / Vue controlled inputs commit. // Fire `input` + `change` so React / Vue controlled inputs commit.
await nodeCdp.send(target.tabId, "Runtime.callFunctionOn", { await deps.cdp.send(target.tabId, "Runtime.callFunctionOn", {
objectId, objectId,
functionDeclaration: `function() { functionDeclaration: `function() {
this.dispatchEvent(new Event('input', { bubbles: true })); this.dispatchEvent(new Event('input', { bubbles: true }));
@@ -874,21 +853,14 @@ export async function handlePress(
if (params.ref || params.selector) { if (params.ref || params.selector) {
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "press"); const node = await resolveBackendNode(deps.cdp, ctx, target, params, "press");
if (isRpcError(node)) return node; if (isRpcError(node)) return node;
const nodeCdp = cdpRunnerForTarget(deps.cdp, node.cdpTarget);
try { try {
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId); deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
const scrollErr = await scrollElementAndFramesIntoView( const scrollErr = await scrollNodeIntoView(deps.cdp, target.tabId, node.backendNodeId);
deps.cdp,
target.tabId,
node.cdpTarget,
node.backendNodeId,
node.frameId,
);
if (scrollErr) return scrollErr; if (scrollErr) return scrollErr;
if (throwIfAborted(deps.signal)) { if (throwIfAborted(deps.signal)) {
return { code: "cancelled", message: "press aborted" }; return { code: "cancelled", message: "press aborted" };
} }
await nodeCdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId }); await deps.cdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
} catch (err) { } catch (err) {
return { return {
code: "cdp_failed", code: "cdp_failed",
@@ -993,11 +965,10 @@ export async function handleSelect(
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "select"); const node = await resolveBackendNode(deps.cdp, ctx, target, params, "select");
if (isRpcError(node)) return node; if (isRpcError(node)) return node;
const nodeCdp = cdpRunnerForTarget(deps.cdp, node.cdpTarget);
try { try {
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId); deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
const described = await nodeCdp.send<{ node?: DescribedNode }>( const described = await deps.cdp.send<{ node?: DescribedNode }>(
target.tabId, target.tabId,
"DOM.describeNode", "DOM.describeNode",
{ backendNodeId: node.backendNodeId }, { backendNodeId: node.backendNodeId },
@@ -1020,18 +991,12 @@ export async function handleSelect(
"single-select <select> requires exactly one value", "single-select <select> requires exactly one value",
); );
} }
const scrollErr = await scrollElementAndFramesIntoView( const scrollErr = await scrollNodeIntoView(deps.cdp, target.tabId, node.backendNodeId);
deps.cdp,
target.tabId,
node.cdpTarget,
node.backendNodeId,
node.frameId,
);
if (scrollErr) return scrollErr; if (scrollErr) return scrollErr;
if (throwIfAborted(deps.signal)) { if (throwIfAborted(deps.signal)) {
return { code: "cancelled", message: "select aborted" }; return { code: "cancelled", message: "select aborted" };
} }
await nodeCdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId }); await deps.cdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
} catch (err) { } catch (err) {
return { return {
code: "cdp_failed", code: "cdp_failed",
@@ -1043,12 +1008,12 @@ export async function handleSelect(
return { code: "cancelled", message: "select aborted" }; return { code: "cancelled", message: "select aborted" };
} }
const objectIdOrErr = await backendNodeToObject(nodeCdp, target.tabId, node.backendNodeId); const objectIdOrErr = await backendNodeToObject(deps.cdp, target.tabId, node.backendNodeId);
if (isRpcError(objectIdOrErr)) return objectIdOrErr; if (isRpcError(objectIdOrErr)) return objectIdOrErr;
const objectId = objectIdOrErr; const objectId = objectIdOrErr;
try { try {
const evaluated = await nodeCdp.send<{ const evaluated = await deps.cdp.send<{
result?: { value?: SelectMutationResult | null }; result?: { value?: SelectMutationResult | null };
}>(target.tabId, "Runtime.callFunctionOn", { }>(target.tabId, "Runtime.callFunctionOn", {
objectId, objectId,
@@ -1107,6 +1072,9 @@ export async function handleSelect(
export const __testing__ = { export const __testing__ = {
DEFAULT_TIMEOUT_MS, DEFAULT_TIMEOUT_MS,
quadCentre,
boxCentre,
resolveBackendNode, resolveBackendNode,
nodeCentre,
isFillable, isFillable,
}; };
File diff suppressed because it is too large Load Diff
+270 -293
View File
@@ -15,26 +15,31 @@ import {
type RecordQueryResponse, type RecordQueryResponse,
type RecordStartAck, type RecordStartAck,
type RecordStartMessage, type RecordStartMessage,
type RecordStepAck,
type RecordStopMessage, type RecordStopMessage,
} from "@/lib/record-bridge"; } from "@/lib/record-bridge";
import { DEFAULT_MAX_PAGE_TOKENS } from "@/lib/record-constants";
import { import {
type RecordFrameCoordinator, applyTargetMatching,
type RecordingCaptureScope, buildTraceV3,
recordFrameCoordinator, cancelPendingSettles,
} from "@/lib/recording/frame-coordinator"; captureAndRegisterObservation,
import { RecordingObservationRuntime } from "@/lib/recording/recording-runtime"; clearPendingRedirectLanding,
import { createObservationState,
appendRecordedPayload, flushPendingRedirectLanding,
observeRecordedNavigation, flushPendingSettles,
type RecordingStepBuffer, inferMissingPostStates,
} from "@/lib/recording/step-buffer"; type RecordingObservationState,
import { RecordingTabCoordinator, type TabActivation } from "@/lib/recording/tab-coordinator"; rememberStepOnPage,
import { buildTraceV2 } from "@/lib/recording/trace-reducer-v2"; scheduleDraftSettle,
import type { RecordingDraftStep } from "@/lib/recording/types"; scheduleRedirectLandingFlush,
settleUnsettledDrafts,
} from "@/lib/record-observation";
import { appendRecordedPayload, observeRecordedNavigation } from "@/lib/recording-step-buffer";
import { buildTraceV2 } from "@/lib/trace-reducer-v2";
import type { SessionManager } from "@/session-manager/manager"; import type { SessionManager } from "@/session-manager/manager";
import { EXTENSION_VERSION } from "@/transport/handshake"; import { EXTENSION_VERSION } from "@/transport/handshake";
import type { import type {
DraftTraceStep,
RecordAwaitParams, RecordAwaitParams,
RecordAwaitResult, RecordAwaitResult,
RecordedTrace, RecordedTrace,
@@ -45,7 +50,7 @@ import type {
RpcError, RpcError,
StopReason, StopReason,
} from "@/transport/types"; } from "@/transport/types";
import { TRACE_VERSION_V3 } from "@/transport/types"; import { TRACE_VERSION } from "@/transport/types";
import { handleNavigate } from "./navigation"; import { handleNavigate } from "./navigation";
import { import {
type CdpRunner, type CdpRunner,
@@ -58,22 +63,30 @@ import {
interface ActiveRecording { interface ActiveRecording {
requestId: string; requestId: string;
tabs: RecordingTabCoordinator; tabId: number;
agentWindowId: number; agentWindowId: number;
startUrl?: string; startUrl?: string;
purpose?: string; purpose?: string;
steps: RecordingDraftStep[]; steps: DraftTraceStep[];
startedAt: string; startedAt: string;
startedAtMs: number; startedAtMs: number;
traceVersion: 2 | 3; traceVersion: 2 | 3;
supportsTabSwitchSteps: boolean;
finishPromise: Promise<RecordedTrace>; finishPromise: Promise<RecordedTrace>;
resolveFinish: (trace: RecordedTrace) => void; resolveFinish: (trace: RecordedTrace) => void;
rejectFinish: (err: Error) => void; rejectFinish: (err: Error) => void;
settled: boolean; settled: boolean;
finishAttempt: Promise<RecordedTrace | null> | null; finishing: boolean;
observation: RecordingObservationRuntime | null; currentUrl?: string;
pendingNavigation: boolean;
pendingNavigationDeadline?: number;
observation: RecordingObservationState;
stoppedBy: StopReason; stoppedBy: StopReason;
maxPageTokens: number;
redactValues: boolean;
/** Tabs whose content capture acknowledged START and still need STOP. */
armedTabIds: Set<number>;
/** Rearms already sending START when finish begins. */
rearmCallbacks: Set<Promise<void>>;
/** Navigation callbacks tracked from event receipt through action enqueue. */ /** Navigation callbacks tracked from event receipt through action enqueue. */
navigationCallbacks: Set<Promise<void>>; navigationCallbacks: Set<Promise<void>>;
/** Synchronous intake gate closed only after finish drains to stability. */ /** Synchronous intake gate closed only after finish drains to stability. */
@@ -83,21 +96,16 @@ interface ActiveRecording {
* in `steps` before a same-turn `webNavigation` tries to annotate it. * in `steps` before a same-turn `webNavigation` tries to annotate it.
*/ */
actionQueue: Promise<void>; actionQueue: Promise<void>;
/** Last accepted sequence for each content-script document producer. */ /**
lastStepSequenceByProducer: Map<string, number>; * CDP runner captured at `record_start`. The message listeners are attached
* once at service-worker startup, where no runner exists yet, so observation
* capture must go through the recording instead of the listener deps.
*/
cdp?: CdpRunner;
} }
function enqueueRecordingAction( function enqueueRecordingAction(recording: ActiveRecording, task: () => Promise<void>): void {
recording: ActiveRecording, recording.actionQueue = recording.actionQueue.then(task, task).catch(() => {});
task: () => Promise<void>,
): Promise<void> {
const queued = recording.actionQueue.then(task, task);
recording.actionQueue = queued.catch(() => {});
return queued;
}
function isRecordingFinishing(recording: ActiveRecording): boolean {
return recording.settled || recording.finishAttempt !== null;
} }
const recordings = new Map<string, ActiveRecording>(); const recordings = new Map<string, ActiveRecording>();
@@ -180,11 +188,9 @@ async function sendRecordStartWithAck(
tabId: number, tabId: number,
msg: RecordStartMessage, msg: RecordStartMessage,
sendToTab: RecordDeps["sendToTab"], sendToTab: RecordDeps["sendToTab"],
cancelled?: () => boolean,
): Promise<void> { ): Promise<void> {
let lastError: unknown; let lastError: unknown;
for (let attempt = 0; attempt < RECORD_START_RETRIES; attempt += 1) { for (let attempt = 0; attempt < RECORD_START_RETRIES; attempt += 1) {
if (cancelled?.()) throw new Error("record start cancelled");
try { try {
const response = await sendToTab(tabId, msg); const response = await sendToTab(tabId, msg);
if (isRecordStartAck(response)) return; if (isRecordStartAck(response)) return;
@@ -192,7 +198,6 @@ async function sendRecordStartWithAck(
} catch (err) { } catch (err) {
lastError = err; lastError = err;
} }
if (cancelled?.()) throw new Error("record start cancelled");
if (attempt + 1 < RECORD_START_RETRIES) { if (attempt + 1 < RECORD_START_RETRIES) {
await sleep(RECORD_START_RETRY_DELAY_MS); await sleep(RECORD_START_RETRY_DELAY_MS);
} }
@@ -202,28 +207,25 @@ async function sendRecordStartWithAck(
function negotiatedTraceVersion(params: RecordStartParams): 2 | 3 | RpcError { function negotiatedTraceVersion(params: RecordStartParams): 2 | 3 | RpcError {
if (params.trace_version === undefined) return 2; if (params.trace_version === undefined) return 2;
if (params.trace_version === TRACE_VERSION_V3) return 3; if (params.trace_version === TRACE_VERSION) return 3;
return { return {
code: "invalid_params", code: "invalid_params",
message: `unsupported trace_version ${params.trace_version}; supported values are omitted (v2) or ${TRACE_VERSION_V3} (v3)`, message: `unsupported trace_version ${params.trace_version}; supported values are omitted (v2) or ${TRACE_VERSION} (v3)`,
}; };
} }
function buildTrace(recording: ActiveRecording): RecordedTrace { function buildTrace(recording: ActiveRecording): RecordedTrace {
if (recording.traceVersion === 3 && recording.observation) { if (recording.traceVersion === 3) {
return recording.observation.buildTrace({ return buildTraceV3({
drafts: recording.steps, obs: recording.observation,
steps: recording.steps,
startedAt: recording.startedAt, startedAt: recording.startedAt,
purpose: recording.purpose, ...(recording.purpose ? { purpose: recording.purpose } : {}),
startUrl: recording.startUrl, startUrl: recording.startUrl,
stoppedBy: recording.stoppedBy, stoppedBy: recording.stoppedBy,
bskVersion: BSK_TRACE_VERSION, bskVersion: BSK_TRACE_VERSION,
includeTabSwitches: recording.supportsTabSwitchSteps,
}); });
} }
if (recording.traceVersion === 3) {
throw new Error("trace v3 observation runtime is unavailable");
}
return buildTraceV2({ return buildTraceV2({
steps: recording.steps, steps: recording.steps,
startedAt: recording.startedAt, startedAt: recording.startedAt,
@@ -232,49 +234,33 @@ function buildTrace(recording: ActiveRecording): RecordedTrace {
}); });
} }
function stepBufferFor(
recording: ActiveRecording,
tabId: number,
fallbackUrl?: string,
): RecordingStepBuffer {
return {
steps: recording.steps,
navigation: recording.tabs.navigation(tabId, fallbackUrl),
};
}
async function activateRecordingTab(
recording: ActiveRecording,
targetTabId: number,
): Promise<void> {
const previousTabId = recording.tabs.currentTabId;
if (targetTabId === previousTabId) return;
const transition = recording.observation
? await recording.observation.captureTabTransition(previousTabId, targetTabId)
: {};
const { targetUrl, ...stateLinks } = transition;
const draft: Extract<RecordingDraftStep, { op: "switch_tab" }> = {
op: "switch_tab",
...stateLinks,
};
recording.steps.push(draft);
recording.observation?.bindTabTransition(draft, recording.steps.length);
recording.tabs.commit(targetTabId, targetUrl);
}
async function processRecordedStep( async function processRecordedStep(
recording: ActiveRecording, recording: ActiveRecording,
deps: RecordDeps,
draftIndex: number, draftIndex: number,
tabId: number,
producerId?: string,
): Promise<void> { ): Promise<void> {
if (!recording.observation) return; const cdp = recording.cdp ?? deps.cdp;
try { if (!cdp) return;
await recording.observation.processDraft(tabId, recording.steps, draftIndex, producerId); const draft = recording.steps[draftIndex];
} catch (err) { if (!draft) return;
console.warn(`[bsk record] observation failed for step ${draftIndex + 1}`, err); const stepId = draftIndex + 1;
if (draft.op === "navigate") {
draft.preStateId = recording.observation.lastSettled?.stateId;
} else {
applyTargetMatching(recording.observation, draft, stepId);
} }
rememberStepOnPage(recording.observation, draft, stepId);
scheduleDraftSettle(
recording.observation,
draftIndex,
stepId,
cdp,
recording.tabId,
deps.tabsApi,
recording.steps,
);
} }
export interface RecordDeps { export interface RecordDeps {
@@ -284,25 +270,16 @@ export interface RecordDeps {
msg: RecordStartMessage | RecordStopMessage | RecordCancelMessage, msg: RecordStartMessage | RecordStopMessage | RecordCancelMessage,
): Promise<unknown>; ): Promise<unknown>;
bypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>; bypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
frameCoordinator?: Pick<
RecordFrameCoordinator,
"begin" | "armTab" | "sourceFor" | "stop" | "cancel"
>;
cdp?: CdpRunner; cdp?: CdpRunner;
signal?: AbortSignal; signal?: AbortSignal;
} }
export type RecordRuntimeDeps = Omit<RecordDeps, "frameCoordinator" | "signal"> & {
frameCoordinator: NonNullable<RecordDeps["frameCoordinator"]>;
};
let defaultDeps: RecordDeps | null = null; let defaultDeps: RecordDeps | null = null;
function getDefaultDeps(): RecordDeps { function getDefaultDeps(): RecordDeps {
if (!defaultDeps) { if (!defaultDeps) {
defaultDeps = { defaultDeps = {
tabsApi: chromeTabsApi, tabsApi: chromeTabsApi,
sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg), sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
frameCoordinator: recordFrameCoordinator,
}; };
} }
return defaultDeps; return defaultDeps;
@@ -363,55 +340,28 @@ export function releaseBrowserObservationListenersIfIdle(): void {
export function attachRecordStepListener(deps: RecordDeps = getDefaultDeps()): () => void { export function attachRecordStepListener(deps: RecordDeps = getDefaultDeps()): () => void {
const listener = ( const listener = (
message: unknown, message: unknown,
sender: chrome.runtime.MessageSender, _sender: chrome.runtime.MessageSender,
sendResponse: (response: RecordStepAck) => void, _sendResponse: () => void,
) => { ) => {
if (!isRecordStepMessage(message)) return false; if (!isRecordStepMessage(message)) return false;
for (const recording of recordings.values()) { for (const recording of recordings.values()) {
if (recording.requestId !== message.requestId) continue; if (recording.requestId !== message.requestId) continue;
const source: RecordingCaptureScope | null | undefined = deps.frameCoordinator
? deps.frameCoordinator.sourceFor(message.requestId, message.producerId, sender)
: undefined;
if (deps.frameCoordinator && !source) return false;
const sourceTabId = source?.tabId ?? sender.tab?.id ?? recording.tabs.currentTabId;
const sourceWasActive = sender.tab?.active ?? sourceTabId === recording.tabs.activeTabId;
const producerKey = source
? `${source.tabId}:${source.documentId}:${source.producerId}`
: `${sourceTabId}:${message.producerId}`;
const expectedSequence = (recording.lastStepSequenceByProducer.get(producerKey) ?? 0) + 1;
if (message.sequence < expectedSequence) {
sendResponse({ ok: true, sequence: message.sequence });
return false;
}
if (message.sequence > expectedSequence) {
sendResponse({
ok: false,
expectedSequence,
error: `out-of-order recorded step ${message.sequence}`,
});
return false;
}
recording.lastStepSequenceByProducer.set(producerKey, message.sequence);
enqueueRecordingAction(recording, async () => { enqueueRecordingAction(recording, async () => {
if (sourceTabId !== recording.tabs.currentTabId) { const cdp = recording.cdp ?? deps.cdp;
if (!sourceWasActive) return; // Flush before appending so a coalesced OAuth navigate is ordered
if (sourceTabId !== recording.tabs.activeTabId) // ahead of the click that landed on the destination page. The flush
recording.tabs.noteActivation(sourceTabId); // itself waits for that navigate's observation before returning.
await activateRecordingTab(recording, sourceTabId); await flushPendingRedirectLanding(
} recording.observation,
await recording.observation?.flushRedirects(sourceTabId); recording.steps,
const targetHint = message.step.geometry ? { geometry: message.step.geometry } : undefined; cdp,
const draftIndex = appendRecordedPayload( recording.tabId,
stepBufferFor(recording, sourceTabId, message.step.page_url), deps.tabsApi,
message.step, { cancelled: () => recording.settled },
targetHint,
); );
if (draftIndex !== null) { const draftIndex = appendRecordedPayload(recording, message.step);
await processRecordedStep(recording, draftIndex, sourceTabId, source?.producerId); if (draftIndex !== null) await processRecordedStep(recording, deps, draftIndex);
}
}); });
sendResponse({ ok: true, sequence: message.sequence });
return false; return false;
} }
return false; return false;
@@ -438,7 +388,9 @@ export function attachRecordFinishListener(deps: RecordDeps = getDefaultDeps()):
function findRecordingByTabId(tabId: number): ActiveRecording | null { function findRecordingByTabId(tabId: number): ActiveRecording | null {
for (const recording of recordings.values()) { for (const recording of recordings.values()) {
if (recording.tabs.currentTabId === tabId && !recording.settled) return recording; if (!recording.settled && (recording.tabId === tabId || recording.armedTabIds.has(tabId))) {
return recording;
}
} }
return null; return null;
} }
@@ -475,7 +427,7 @@ async function clearRearmTimersForRecording(
recording: ActiveRecording, recording: ActiveRecording,
deps: RecordDeps, deps: RecordDeps,
): Promise<void> { ): Promise<void> {
clearRearmTimer(recording.tabs.currentTabId); clearRearmTimer(recording.tabId);
try { try {
const tabs = await deps.tabsApi.query({ windowId: recording.agentWindowId }); const tabs = await deps.tabsApi.query({ windowId: recording.agentWindowId });
for (const tab of tabs) { for (const tab of tabs) {
@@ -490,33 +442,25 @@ async function stopRecordingOnAllAgentTabs(
recording: ActiveRecording, recording: ActiveRecording,
deps: RecordDeps, deps: RecordDeps,
): Promise<void> { ): Promise<void> {
if (deps.frameCoordinator && !(await deps.frameCoordinator.stop(recording.requestId))) {
throw new Error("failed to flush one or more recording documents");
}
const stopMsg: RecordStopMessage = { type: RECORD_STOP, requestId: recording.requestId }; const stopMsg: RecordStopMessage = { type: RECORD_STOP, requestId: recording.requestId };
let tabIds = [recording.tabs.currentTabId]; let failed = false;
try { for (const tabId of [...recording.armedTabIds]) {
const tabs = await deps.tabsApi.query({ windowId: recording.agentWindowId }); try {
tabIds = [ await deps.tabsApi.get(tabId);
...new Set([ } catch {
recording.tabs.currentTabId, // A closed tab no longer has content capture to flush.
...tabs.flatMap((tab) => (typeof tab.id === "number" ? [tab.id] : [])), recording.armedTabIds.delete(tabId);
]), continue;
]; }
} catch {
// Fall back to the current recording tab.
}
for (const tabId of tabIds) {
try { try {
const response = await deps.sendToTab(tabId, stopMsg); const response = await deps.sendToTab(tabId, stopMsg);
if (tabId === recording.tabs.currentTabId && !isRecordStartAck(response)) { if (!isRecordStartAck(response)) {
throw new Error("content script did not confirm recorded steps"); throw new Error("content script did not confirm recorded steps");
} }
recording.armedTabIds.delete(tabId);
} catch { } catch {
if (tabId === recording.tabs.currentTabId) { failed = true;
throw new Error("failed to flush recorded steps"); continue;
}
} }
if (deps.bypassOverlay) { if (deps.bypassOverlay) {
try { try {
@@ -526,54 +470,53 @@ async function stopRecordingOnAllAgentTabs(
} }
} }
} }
if (failed || recording.armedTabIds.size > 0) {
throw new Error("failed to flush recorded steps");
}
} }
async function rearmRecording( async function rearmRecording(
recording: ActiveRecording, recording: ActiveRecording,
targetTabId: number, targetTabId: number,
deps: RecordDeps, deps: RecordDeps,
activation?: TabActivation,
): Promise<boolean> { ): Promise<boolean> {
let resolveTracked!: () => void;
const tracked = new Promise<void>((resolve) => {
resolveTracked = resolve;
});
recording.rearmCallbacks.add(tracked);
// Do NOT toggle automation-bypass here: each retry used to increment the // Do NOT toggle automation-bypass here: each retry used to increment the
// content-script counter, and a single stop decrement left the ControlOverlay // content-script counter, and a single stop decrement left the ControlOverlay
// stuck with pointer-events:none (page usable, Interrupt dead). RecordOverlay // stuck with pointer-events:none (page usable, Interrupt dead). RecordOverlay
// already hides the control chrome while activeRecord is set. // already hides the control chrome while activeRecord is set.
const isFinishing = () => isRecordingFinishing(recording); try {
for (let attempt = 0; attempt < RECORD_REARM_MAX_ATTEMPTS; attempt += 1) { for (let attempt = 0; attempt < RECORD_REARM_MAX_ATTEMPTS; attempt += 1) {
if (isFinishing()) return false; if (recording.settled || recording.finishing) return false;
const startMsg: RecordStartMessage = { const startMsg: RecordStartMessage = {
type: RECORD_START, type: RECORD_START,
requestId: recording.requestId, requestId: recording.requestId,
startedAtMs: recording.startedAtMs, startedAtMs: recording.startedAtMs,
}; };
try { try {
if (deps.frameCoordinator) { await sendRecordStartWithAck(targetTabId, startMsg, deps.sendToTab);
const frameStarted = await deps.frameCoordinator.armTab(recording.requestId, targetTabId); recording.armedTabIds.add(targetTabId);
if (!frameStarted) throw new Error("recording document did not start"); recording.tabId = targetTabId;
} return true;
await sendRecordStartWithAck(targetTabId, startMsg, deps.sendToTab, isFinishing); } catch {
if (isFinishing()) return false; if (attempt + 1 < RECORD_REARM_MAX_ATTEMPTS) {
if (activation) { await sleep(RECORD_REARM_RETRY_DELAY_MS);
if (!recording.tabs.isLatest(activation)) return true; }
await enqueueRecordingAction(recording, async () => {
if (isFinishing() || !recording.tabs.isLatest(activation)) {
return;
}
await activateRecordingTab(recording, targetTabId);
});
}
return true;
} catch {
if (isFinishing()) return false;
if (attempt + 1 < RECORD_REARM_MAX_ATTEMPTS) {
await sleep(RECORD_REARM_RETRY_DELAY_MS);
} }
} }
return false;
} finally {
recording.rearmCallbacks.delete(tracked);
resolveTracked();
} }
return false;
} }
function scheduleRearmForTab(tabId: number, deps: RecordDeps, activation?: TabActivation): void { function scheduleRearmForTab(tabId: number, deps: RecordDeps): void {
const existing = rearmTimers.get(tabId); const existing = rearmTimers.get(tabId);
if (existing) clearTimeout(existing); if (existing) clearTimeout(existing);
rearmTimers.set( rearmTimers.set(
@@ -582,7 +525,7 @@ function scheduleRearmForTab(tabId: number, deps: RecordDeps, activation?: TabAc
rearmTimers.delete(tabId); rearmTimers.delete(tabId);
void (async () => { void (async () => {
const current = await findRecordingForTab(tabId, deps); const current = await findRecordingForTab(tabId, deps);
if (current) await rearmRecording(current, tabId, deps, activation); if (current) await rearmRecording(current, tabId, deps);
})(); })();
}, RECORD_REARM_DEBOUNCE_MS), }, RECORD_REARM_DEBOUNCE_MS),
); );
@@ -594,20 +537,14 @@ export function attachRecordTabListener(deps: RecordDeps = getDefaultDeps()): ()
const windowId = tab.windowId; const windowId = tab.windowId;
if (tabId === undefined || windowId === undefined) return; if (tabId === undefined || windowId === undefined) return;
for (const recording of recordings.values()) { for (const recording of recordings.values()) {
if (isRecordingFinishing(recording) || recording.agentWindowId !== windowId) continue; if (recording.settled || recording.agentWindowId !== windowId) continue;
scheduleRearmForTab(tabId, deps); scheduleRearmForTab(tabId, deps);
return; return;
} }
}; };
const onActivated = (activeInfo: chrome.tabs.TabActiveInfo) => { const onActivated = (activeInfo: chrome.tabs.TabActiveInfo) => {
for (const recording of recordings.values()) { scheduleRearmForTab(activeInfo.tabId, deps);
if (isRecordingFinishing(recording) || recording.agentWindowId !== activeInfo.windowId)
continue;
const activation = recording.tabs.noteActivation(activeInfo.tabId);
scheduleRearmForTab(activeInfo.tabId, deps, activation);
return;
}
}; };
chrome.tabs.onCreated.addListener(onCreated); chrome.tabs.onCreated.addListener(onCreated);
@@ -627,39 +564,80 @@ export function attachRecordNavigationListener(deps: RecordDeps = getDefaultDeps
transitionQualifiers?: string[], transitionQualifiers?: string[],
) => { ) => {
if (!url) return; if (!url) return;
const candidates = [...recordings.values()].filter( const direct = findRecordingByTabId(tabId);
(recording) => const candidates = direct
!recording.settled && recording.acceptingNavigation && recording.tabs.activeTabId === tabId, ? direct.acceptingNavigation
); ? [direct]
for (const recording of candidates) { : []
const queued = enqueueRecordingAction(recording, async () => { : [...recordings.values()].filter(
if (tabId !== recording.tabs.currentTabId) { (recording) => !recording.settled && recording.acceptingNavigation,
await activateRecordingTab(recording, tabId);
}
const result = observeRecordedNavigation(
stepBufferFor(recording, tabId, url),
url,
causedByAction,
transitionType,
transitionQualifiers,
); );
if (result.kind === "coalesce_redirect") { if (candidates.length === 0) return;
recording.observation?.scheduleRedirect(tabId, recording.steps, result.url);
let resolveTracked!: () => void;
const tracked = new Promise<void>((resolve) => {
resolveTracked = resolve;
});
for (const candidate of candidates) candidate.navigationCallbacks.add(tracked);
void (async () => {
try {
const recording = await findRecordingForTab(tabId, deps);
if (!recording || !recording.acceptingNavigation || !candidates.includes(recording)) {
return; return;
} }
enqueueRecordingAction(recording, async () => {
const result = observeRecordedNavigation(
recording,
url,
causedByAction,
transitionType,
transitionQualifiers,
);
const cdp = recording.cdp ?? deps.cdp;
if (!cdp) return;
if (result.kind === "noop") return; if (result.kind === "coalesce_redirect") {
recording.observation?.clearRedirect(tabId); scheduleRedirectLandingFlush(
if (result.kind === "appended") { recording.observation,
await processRecordedStep(recording, result.index, tabId); recording.steps,
} else { cdp,
recording.observation?.scheduleSettle(tabId, recording.steps, result.index); tabId,
} deps.tabsApi,
}); result.url,
const tracked = queued.catch(() => {}); { cancelled: () => recording.settled },
recording.navigationCallbacks.add(tracked); );
void tracked.finally(() => recording.navigationCallbacks.delete(tracked)); return;
} }
if (result.kind === "noop") return;
// A concrete navigation supersedes any in-flight redirect coalesce.
clearPendingRedirectLanding(recording.observation);
const draftIndex = result.index;
const lastDraft = recording.steps[draftIndex];
if (!lastDraft) return;
if (result.kind === "appended") {
lastDraft.preStateId = recording.observation.lastSettled?.stateId;
rememberStepOnPage(recording.observation, lastDraft, draftIndex + 1);
}
scheduleDraftSettle(
recording.observation,
draftIndex,
draftIndex + 1,
cdp,
tabId,
deps.tabsApi,
recording.steps,
);
});
} finally {
for (const candidate of candidates) candidate.navigationCallbacks.delete(tracked);
resolveTracked();
}
})();
}; };
const onMainFrameComplete = (tabId: number, url?: string) => { const onMainFrameComplete = (tabId: number, url?: string) => {
void (async () => { void (async () => {
@@ -705,14 +683,34 @@ export function attachRecordNavigationListener(deps: RecordDeps = getDefaultDeps
const MAX_FINISH_DRAIN_ROUNDS = 10; const MAX_FINISH_DRAIN_ROUNDS = 10;
async function drainRecordingToStability(recording: ActiveRecording): Promise<boolean> { async function drainRecordingToStability(
recording: ActiveRecording,
deps: RecordDeps,
): Promise<boolean> {
const cdp = recording.cdp ?? deps.cdp;
for (let round = 0; round < MAX_FINISH_DRAIN_ROUNDS; round += 1) { for (let round = 0; round < MAX_FINISH_DRAIN_ROUNDS; round += 1) {
await Promise.all([...recording.navigationCallbacks]); await Promise.all([...recording.navigationCallbacks]);
const actionTail = recording.actionQueue; const actionTail = recording.actionQueue;
await actionTail; await actionTail;
await recording.observation?.flush(); await flushPendingRedirectLanding(
recording.observation,
recording.steps,
cdp,
recording.tabId,
deps.tabsApi,
{ cancelled: () => recording.settled },
);
await flushPendingSettles(recording.observation);
const redirectTail = recording.observation.redirectFlushQueue;
const settleTail = recording.observation.settleQueue;
await Promise.all([redirectTail, settleTail]);
if (recording.navigationCallbacks.size === 0 && recording.actionQueue === actionTail) { if (
recording.navigationCallbacks.size === 0 &&
recording.actionQueue === actionTail &&
recording.observation.redirectFlushQueue === redirectTail &&
recording.observation.settleQueue === settleTail
) {
// No event or promise continuation can interleave with this synchronous // No event or promise continuation can interleave with this synchronous
// check-and-close, so work accepted before the cutoff is fully drained. // check-and-close, so work accepted before the cutoff is fully drained.
recording.acceptingNavigation = false; recording.acceptingNavigation = false;
@@ -741,8 +739,7 @@ export function attachRecordQueryListener(deps: RecordDeps = getDefaultDeps()):
sendResponse({ active: false }); sendResponse({ active: false });
return; return;
} }
const activation = sender.tab?.active ? recording.tabs.noteActivation(tabId) : undefined; await rearmRecording(recording, tabId, deps);
await rearmRecording(recording, tabId, deps, activation);
sendResponse({ sendResponse({
active: true, active: true,
requestId: recording.requestId, requestId: recording.requestId,
@@ -776,38 +773,37 @@ async function finishRecording(
): Promise<RecordedTrace | null> { ): Promise<RecordedTrace | null> {
const recording = recordings.get(sessionId); const recording = recordings.get(sessionId);
if (!recording || recording.settled) return null; if (!recording || recording.settled) return null;
if (recording.finishAttempt) return recording.finishAttempt; if (recording.finishing) return recording.finishPromise;
recording.finishing = true;
recording.stoppedBy = stoppedBy; recording.stoppedBy = stoppedBy;
const attempt = finishRecordingAttempt(sessionId, recording, deps);
recording.finishAttempt = attempt;
try {
return await attempt;
} finally {
if (!recording.settled) {
recording.finishAttempt = null;
}
}
}
async function finishRecordingAttempt(
sessionId: string,
recording: ActiveRecording,
deps: RecordDeps,
): Promise<RecordedTrace | null> {
await clearRearmTimersForRecording(recording, deps); await clearRearmTimersForRecording(recording, deps);
await Promise.all([...recording.rearmCallbacks]);
try { try {
// Disposing content capture may commit a final dirty fill. Flush content // Disposing content capture may commit a final dirty fill. Flush content
// first so all resulting RECORD_STEP messages enter actionQueue before it // first so all resulting RECORD_STEP messages enter actionQueue before it
// and the observation queues are drained. // and the observation queues are drained.
await stopRecordingOnAllAgentTabs(recording, deps); await stopRecordingOnAllAgentTabs(recording, deps);
} catch { } catch {
recording.finishing = false;
return null; return null;
} }
if (!(await drainRecordingToStability(recording))) { if (!(await drainRecordingToStability(recording, deps))) {
recording.finishing = false;
return null; return null;
} }
await recording.observation?.settleTrailing(recording.tabs.currentTabId, recording.steps); cancelPendingSettles(recording.observation);
inferMissingPostStates(recording.steps);
const cdp = recording.cdp ?? deps.cdp;
if (cdp) {
await settleUnsettledDrafts(
recording.observation,
cdp,
recording.tabId,
deps.tabsApi,
recording.steps,
);
}
recording.settled = true; recording.settled = true;
recordings.delete(sessionId); recordings.delete(sessionId);
@@ -824,12 +820,6 @@ export async function handleRecordStart(
): Promise<RecordStartResult | RpcError> { ): Promise<RecordStartResult | RpcError> {
const traceVersionOrErr = negotiatedTraceVersion(params); const traceVersionOrErr = negotiatedTraceVersion(params);
if (typeof traceVersionOrErr !== "number") return traceVersionOrErr; if (typeof traceVersionOrErr !== "number") return traceVersionOrErr;
if (traceVersionOrErr === 3 && !deps.cdp) {
return {
code: "protocol_error",
message: "trace v3 recording requires an active CDP connection",
};
}
const ctxOrErr = lookupSession(manager, params, "record_start"); const ctxOrErr = lookupSession(manager, params, "record_start");
if (isRpcError(ctxOrErr)) return ctxOrErr; if (isRpcError(ctxOrErr)) return ctxOrErr;
@@ -859,7 +849,7 @@ export async function handleRecordStart(
const redactValues = params.redact_values ?? false; const redactValues = params.redact_values ?? false;
recordings.set(params.session_id, { recordings.set(params.session_id, {
requestId, requestId,
tabs: new RecordingTabCoordinator(target.tabId, navigateUrl), tabId: target.tabId,
agentWindowId: ctx.agentWindowId, agentWindowId: ctx.agentWindowId,
startUrl: navigateUrl, startUrl: navigateUrl,
...(params.purpose ? { purpose: params.purpose } : {}), ...(params.purpose ? { purpose: params.purpose } : {}),
@@ -867,44 +857,30 @@ export async function handleRecordStart(
startedAt: new Date(startedAtMs).toISOString(), startedAt: new Date(startedAtMs).toISOString(),
startedAtMs, startedAtMs,
traceVersion: traceVersionOrErr, traceVersion: traceVersionOrErr,
supportsTabSwitchSteps: params.supports_tab_switch_steps === true,
finishPromise, finishPromise,
resolveFinish, resolveFinish,
rejectFinish, rejectFinish,
settled: false, settled: false,
finishAttempt: null, finishing: false,
observation: currentUrl: navigateUrl,
traceVersionOrErr === 3 && deps.cdp pendingNavigation: false,
? new RecordingObservationRuntime({ pendingNavigationDeadline: undefined,
cdp: deps.cdp, observation: createObservationState({ maxPageTokens, redactValues }),
tabsApi: deps.tabsApi,
maxTokens: maxPageTokens,
redactValues,
})
: null,
stoppedBy: "user_finish", stoppedBy: "user_finish",
maxPageTokens: maxPageTokens ?? DEFAULT_MAX_PAGE_TOKENS,
redactValues,
armedTabIds: new Set(),
rearmCallbacks: new Set(),
navigationCallbacks: new Set(), navigationCallbacks: new Set(),
acceptingNavigation: true, acceptingNavigation: true,
actionQueue: Promise.resolve(), actionQueue: Promise.resolve(),
lastStepSequenceByProducer: new Map(), ...(deps.cdp ? { cdp: deps.cdp } : {}),
});
deps.frameCoordinator?.begin(requestId, startedAtMs, target.tabId, async (scope) => {
const recording = recordings.get(params.session_id);
if (!recording || recording.requestId !== requestId || recording.settled) return;
try {
await recording.observation?.refreshDocument(scope.tabId, scope.producerId);
} catch {
// The normal action-time capture remains available if a child Document
// is replaced while its readiness refresh is in flight.
}
}); });
// Observe navigations for the whole recording lifetime; attach before // Observe navigations for the whole recording lifetime; attach before
// optional navigate so the destination load can rearm capture. // optional navigate so the destination load can rearm capture.
ensureBrowserObservationListeners(deps); ensureBrowserObservationListeners(deps);
const abortPending = async (notifyContent: boolean) => { const abortPending = async (notifyContent: boolean) => {
recordings.get(params.session_id)?.observation?.cancel();
deps.frameCoordinator?.cancel(requestId);
recordings.delete(params.session_id); recordings.delete(params.session_id);
releaseBrowserObservationListenersIfIdle(); releaseBrowserObservationListenersIfIdle();
if (notifyContent) { if (notifyContent) {
@@ -986,7 +962,7 @@ export async function handleRecordStart(
return cancelledError(); return cancelledError();
} }
active.startUrl = startUrl; active.startUrl = startUrl;
active.tabs.navigation(target.tabId, startUrl).currentUrl = startUrl; active.currentUrl = startUrl;
if (isContentScriptRestrictedUrl(startUrl)) { if (isContentScriptRestrictedUrl(startUrl)) {
await abortPending(false); await abortPending(false);
@@ -994,7 +970,7 @@ export async function handleRecordStart(
code: "invalid_params", code: "invalid_params",
message: params.url message: params.url
? `cannot record on restricted URL (${startUrl}); use an http(s) page` ? `cannot record on restricted URL (${startUrl}); use an http(s) page`
: `cannot record on restricted URL (${startUrl}); default start page https://example.com/ did not load — pass --url with a page you can open`, : `cannot record on restricted URL (${startUrl}); default start page must be injectable http(s)`,
}; };
} }
@@ -1021,11 +997,8 @@ export async function handleRecordStart(
const startMsg: RecordStartMessage = { type: RECORD_START, requestId, startedAtMs }; const startMsg: RecordStartMessage = { type: RECORD_START, requestId, startedAtMs };
try { try {
if (deps.frameCoordinator) {
const frameStarted = await deps.frameCoordinator.armTab(requestId, target.tabId);
if (!frameStarted) throw new Error("top recording document did not start");
}
await sendRecordStartWithAck(target.tabId, startMsg, deps.sendToTab); await sendRecordStartWithAck(target.tabId, startMsg, deps.sendToTab);
active.armedTabIds.add(target.tabId);
} catch { } catch {
await abortPending(true); await abortPending(true);
return { return {
@@ -1035,11 +1008,17 @@ export async function handleRecordStart(
}; };
} }
if (active.observation) { if (deps.cdp) {
const activeRecording = recordings.get(params.session_id); const activeRecording = recordings.get(params.session_id);
if (activeRecording) { if (activeRecording) {
try { try {
await activeRecording.observation?.captureInitial(target.tabId); await captureAndRegisterObservation(
activeRecording.observation,
deps.cdp,
target.tabId,
deps.tabsApi,
startUrl,
);
} catch { } catch {
// Proceed without initial observation; steps may be dropped by reducer. // Proceed without initial observation; steps may be dropped by reducer.
} }
@@ -1144,8 +1123,6 @@ export function clearRecordingForSession(sessionId: string): void {
} }
void clearRearmTimersForRecording(recording, getDefaultDeps()); void clearRearmTimersForRecording(recording, getDefaultDeps());
if (!recording.settled) { if (!recording.settled) {
getDefaultDeps().frameCoordinator?.cancel(recording.requestId);
recording.observation?.cancel();
recording.settled = true; recording.settled = true;
recording.rejectFinish(new Error("recording cleared")); recording.rejectFinish(new Error("recording cleared"));
} }

Some files were not shown because too many files have changed in this diff Show More