mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
Compare commits
105
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5cd43553ed | ||
|
|
9ad33801c7 | ||
|
|
7aaa5a2d01 | ||
|
|
210b21b637 | ||
|
|
48f64ec0df | ||
|
|
1a70d2200e | ||
|
|
554861d6b1 | ||
|
|
0d7f976335 | ||
|
|
d1d3eb6a94 | ||
|
|
38fd7d3f4c | ||
|
|
b5062ac19b | ||
|
|
41cb576256 | ||
|
|
f060a7c074 | ||
|
|
60b62d5420 | ||
|
|
70b851a3e8 | ||
|
|
da3534d066 | ||
|
|
a004291848 | ||
|
|
4b6cdde168 | ||
|
|
4aade4b94f | ||
|
|
616af1099f | ||
|
|
3656f0dd3c | ||
|
|
9742d83213 | ||
|
|
615e866608 | ||
|
|
c08fed5de3 | ||
|
|
887d1ed0b9 | ||
|
|
1a3d3b7de8 | ||
|
|
72834980d2 | ||
|
|
38b0debba9 | ||
|
|
4677149e5e | ||
|
|
8ea5477e11 | ||
|
|
17af1af65a | ||
|
|
6300eda0aa | ||
|
|
87cdcfe5ca | ||
|
|
21476c3884 | ||
|
|
c2a225377b | ||
|
|
e64f4cac9f | ||
|
|
368507b07d | ||
|
|
b750eed5d4 | ||
|
|
3d159c0254 | ||
|
|
524516cb56 | ||
|
|
f98198949a | ||
|
|
d05279f991 | ||
|
|
a4b6a61d29 | ||
|
|
cad0b5cbc9 | ||
|
|
2719e76438 | ||
|
|
dd9fc9edba | ||
|
|
005a6fd534 | ||
|
|
20554508db | ||
|
|
8352c3c67a | ||
|
|
4eedd98c1d | ||
|
|
6aa931b60a | ||
|
|
3df2738280 | ||
|
|
f86e506d47 | ||
|
|
45a246c41b | ||
|
|
6b78211e79 | ||
|
|
eebf334b40 | ||
|
|
a284ffc5ae | ||
|
|
e720f1ffe5 | ||
|
|
2b7e10ca2b | ||
|
|
1ef93463dd | ||
|
|
c7252529f4 | ||
|
|
0b65b4b295 | ||
|
|
94ec7e73fc | ||
|
|
42403e55f2 | ||
|
|
22019cdf07 | ||
|
|
8d7c0aa66c | ||
|
|
a9489282fa | ||
|
|
d4cddd6224 | ||
|
|
8e4de6cf4b | ||
|
|
fea66ce404 | ||
|
|
8bae85127a | ||
|
|
edb149611b | ||
|
|
14c5884cbc | ||
|
|
e7aad9bc16 | ||
|
|
c8c5c4a7c4 | ||
|
|
3fb9e9f581 | ||
|
|
9854ee2f45 | ||
|
|
7035e46024 | ||
|
|
51085b960b | ||
|
|
51df868008 | ||
|
|
43ebfab7ba | ||
|
|
ce11325de2 | ||
|
|
4d1dcd122f | ||
|
|
a4e668542d | ||
|
|
52aa356153 | ||
|
|
d1001e14b2 | ||
|
|
eb71ddc1da | ||
|
|
b07b8fec65 | ||
|
|
47070fdc14 | ||
|
|
916229dc62 | ||
|
|
dde707b558 | ||
|
|
e2d07574d7 | ||
|
|
461d204ca3 | ||
|
|
bc5d9610a3 | ||
|
|
2955260d11 | ||
|
|
ce89e95211 | ||
|
|
24faa10659 | ||
|
|
a7bc8bacf6 | ||
|
|
3d050e0f26 | ||
|
|
afdc398730 | ||
|
|
6b96fcca32 | ||
|
|
0771292336 | ||
|
|
54869cabd0 | ||
|
|
2de7c5c6ff | ||
|
|
703fc64399 |
@@ -0,0 +1,105 @@
|
||||
name: Release dsh plugin
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- dsh-plugin-v*
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: release-dsh-plugin-${{ github.ref }}
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
PACKAGE_NAME: "@wxg-prc-cpg/browser-skill-dsh-plugin"
|
||||
PACKAGE_DIR: packages/dsh-plugin-browserskill
|
||||
|
||||
jobs:
|
||||
resolve:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
version: ${{ steps.version.outputs.version }}
|
||||
tag: ${{ steps.version.outputs.tag }}
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- name: Resolve plugin version
|
||||
id: version
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
||||
VERSION="$(node -p "require('./${{ env.PACKAGE_DIR }}/package.json').version")"
|
||||
else
|
||||
VERSION="${GITHUB_REF_NAME#dsh-plugin-v}"
|
||||
fi
|
||||
VERSION="${VERSION#v}"
|
||||
echo "version=${VERSION}" >> "$GITHUB_OUTPUT"
|
||||
echo "tag=dsh-plugin-v${VERSION}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
guard:
|
||||
needs: resolve
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- name: Verify tag matches package.json name and version
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
PKG_NAME="$(node -p "require('./${{ env.PACKAGE_DIR }}/package.json').name")"
|
||||
PKG_VERSION="$(node -p "require('./${{ env.PACKAGE_DIR }}/package.json').version")"
|
||||
RELEASE_VERSION="${{ needs.resolve.outputs.version }}"
|
||||
if [ "$PKG_NAME" != "$PACKAGE_NAME" ]; then
|
||||
echo "package.json name (${PKG_NAME}) does not match expected ${PACKAGE_NAME}"
|
||||
exit 1
|
||||
fi
|
||||
if [ "$PKG_VERSION" != "$RELEASE_VERSION" ]; then
|
||||
echo "${PACKAGE_DIR}/package.json version (${PKG_VERSION}) does not match release version (${RELEASE_VERSION})"
|
||||
exit 1
|
||||
fi
|
||||
echo "Version guard passed: ${PKG_NAME}@${RELEASE_VERSION}"
|
||||
|
||||
publish:
|
||||
needs: [resolve, guard]
|
||||
runs-on: ubuntu-latest
|
||||
# Secret NPM_TOKEN is stored on the "npm-publish" GitHub Environment.
|
||||
environment: npm-publish
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- uses: pnpm/action-setup@v6
|
||||
|
||||
- uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: 22
|
||||
cache: pnpm
|
||||
registry-url: https://registry.npmjs.org
|
||||
scope: "@wxg-prc-cpg"
|
||||
|
||||
- name: Install dependencies
|
||||
run: pnpm install --frozen-lockfile
|
||||
|
||||
- name: Typecheck plugin
|
||||
run: pnpm --filter "${{ env.PACKAGE_NAME }}" typecheck
|
||||
|
||||
- name: Test plugin
|
||||
run: pnpm --filter "${{ env.PACKAGE_NAME }}" test
|
||||
|
||||
- name: Publish to npm
|
||||
working-directory: ${{ env.PACKAGE_DIR }}
|
||||
env:
|
||||
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${NODE_AUTH_TOKEN:-}" ]; then
|
||||
echo "::error::Environment secret NPM_TOKEN is not configured"
|
||||
exit 1
|
||||
fi
|
||||
pnpm publish --access public --no-git-checks
|
||||
echo "Published ${PACKAGE_NAME}@${{ needs.resolve.outputs.version }}"
|
||||
@@ -11,6 +11,10 @@ dist/
|
||||
# Logs
|
||||
*.log
|
||||
|
||||
# Local browser evaluation artifacts
|
||||
evals/browser/results/
|
||||
evals/browser/agents.local.json
|
||||
|
||||
# macOS
|
||||
.DS_Store
|
||||
|
||||
|
||||
@@ -3,3 +3,6 @@
|
||||
**/.output/**
|
||||
**/target/**
|
||||
**/node_modules/**
|
||||
# Generated client CSS (tailwind compile output); built by
|
||||
# packages/dsh-plugin-browserskill/scripts/build-client-css.mjs.
|
||||
**/dsh-plugin-browserskill/src/client/bsk-ui.nomodule.css
|
||||
|
||||
+9
-5
@@ -34,13 +34,17 @@ where only `extension connected` fails is expected; go to Step 3.
|
||||
## 3. Open the extension install page
|
||||
|
||||
If `extension connected` is `FAIL` (`0 browsers connected`), the user likely
|
||||
has not installed or enabled the browser extension yet. Open the
|
||||
[Chrome Web Store install page](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi)
|
||||
in the user's browser, then tell the user:
|
||||
has not installed or enabled the browser extension yet. Open the install page
|
||||
matching the user's browser —
|
||||
[Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi)
|
||||
for Chrome and other Chromium browsers,
|
||||
[Edge Add-ons](https://microsoftedge.microsoft.com/addons/detail/browserskill/emacgiaaaiojkkpkddmmdfhmokgmnikg)
|
||||
for Microsoft Edge — then tell the user:
|
||||
|
||||
> Install **BrowserSkill** from the
|
||||
> [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi),
|
||||
> open the popup, and wait until it turns green. Reply when done.
|
||||
> [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi)
|
||||
> (or [Edge Add-ons](https://microsoftedge.microsoft.com/addons/detail/browserskill/emacgiaaaiojkkpkddmmdfhmokgmnikg)
|
||||
> on Edge), open the popup, and wait until it turns green. Reply when done.
|
||||
|
||||
If opening the page fails, give the user the same link. Then run `bsk doctor`
|
||||
once more. All `ok`/`na` → tell the user it's ready.
|
||||
|
||||
Generated
+2
-2
@@ -134,7 +134,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "bsk"
|
||||
version = "0.1.10"
|
||||
version = "0.1.11"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"bsk-protocol",
|
||||
@@ -168,7 +168,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "bsk-protocol"
|
||||
version = "0.1.10"
|
||||
version = "0.1.11"
|
||||
dependencies = [
|
||||
"schemars",
|
||||
"semver",
|
||||
|
||||
+1
-1
@@ -7,7 +7,7 @@ members = [
|
||||
default-members = ["crates/bsk-cli"]
|
||||
|
||||
[workspace.package]
|
||||
version = "0.1.10"
|
||||
version = "0.1.11"
|
||||
edition = "2024"
|
||||
rust-version = "1.85"
|
||||
license = "MIT"
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
</p>
|
||||
|
||||
**BrowserSkill** connects Cursor, Claude Code, Codex, OpenClaw, CodeBuddy,
|
||||
WorkBuddy, Pi, Hermes Agent, and other shell-capable AI agents to your already logged-in
|
||||
WorkBuddy, Pi, Hermes Agent, DeepSeek Harness, and other AI agents to your already logged-in
|
||||
browser.
|
||||
|
||||
Need the agent to touch a tab you already have open? It must borrow that tab
|
||||
@@ -67,7 +67,8 @@ Set up browser-skill on this machine by following https://raw.githubusercontent.
|
||||
|
||||
<br>
|
||||
|
||||
Install the CLI, then install the extension from the [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi).
|
||||
Install the CLI, then install the extension from the [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi)
|
||||
or [Edge Add-ons](https://microsoftedge.microsoft.com/addons/detail/browserskill/emacgiaaaiojkkpkddmmdfhmokgmnikg).
|
||||
|
||||
#### 1. Install the `bsk` CLI
|
||||
|
||||
@@ -91,7 +92,14 @@ bsk --version
|
||||
|
||||
#### 2. Install the browser extension
|
||||
|
||||
Install BrowserSkill from the [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi).
|
||||
Install BrowserSkill from your browser's store:
|
||||
|
||||
| Browser | Store listing |
|
||||
| --- | --- |
|
||||
| Chrome | [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi) |
|
||||
| Microsoft Edge | [Edge Add-ons](https://microsoftedge.microsoft.com/addons/detail/browserskill/emacgiaaaiojkkpkddmmdfhmokgmnikg) |
|
||||
|
||||
On other Chromium-based browsers, install the Chrome Web Store build.
|
||||
|
||||
#### 3. Install the skill
|
||||
|
||||
@@ -123,7 +131,8 @@ internal variants and install paths.
|
||||
|
||||
Other shell-capable agent harnesses are supported too. Copy
|
||||
[`skill/SKILL.md`](skill/SKILL.md) into your harness's skills directory as
|
||||
`browser-skill/SKILL.md` to install the skill manually.
|
||||
`browser-skill/SKILL.md` to install the skill manually. DeepSeek Harness uses a
|
||||
dedicated plugin instead — see [DeepSeek Harness plugin](#deepseek-harness-plugin).
|
||||
|
||||
</details>
|
||||
|
||||
@@ -133,6 +142,26 @@ Start a new Agent session and write a prompt that needs the browser, for example
|
||||
/browser-skill open example.com and summarize what is on the page.
|
||||
```
|
||||
|
||||
## DeepSeek Harness plugin
|
||||
|
||||
Using [DeepSeek Harness](https://github.com/deepseek-ai/deepseek-harness) (`dsh`)?
|
||||
BrowserSkill ships a first-class dsh plugin on npm as
|
||||
[`@wxg-prc-cpg/browser-skill-dsh-plugin`](https://www.npmjs.com/package/@wxg-prc-cpg/browser-skill-dsh-plugin).
|
||||
It injects native `browser_*` tools (no shelling out to `bsk`) and a live Web UI
|
||||
overlay of each Agent Window.
|
||||
|
||||
Add it to a dsh profile, then start that profile:
|
||||
|
||||
```sh
|
||||
dsh plugin --profile web add @wxg-prc-cpg/browser-skill-dsh-plugin
|
||||
dsh --profile web
|
||||
```
|
||||
|
||||
The plugin carries its own copy of the skill, so `bsk install-skill` is not needed
|
||||
for dsh — but the `bsk` CLI and the browser extension are still prerequisites. See
|
||||
the [plugin README](packages/dsh-plugin-browserskill/README.md) for the tool list,
|
||||
configuration, and the observation overlay.
|
||||
|
||||
## How It Works
|
||||
|
||||
BrowserSkill is a local bridge between your agent harness and your browser.
|
||||
@@ -166,7 +195,9 @@ flowchart TB
|
||||
|
||||
The agent never talks to the browser directly. It asks the `bsk` CLI to perform a
|
||||
browser task; the local daemon routes that request to the extension; the
|
||||
extension runs it in an Agent Window.
|
||||
extension runs it in an Agent Window. DeepSeek Harness takes the same path
|
||||
through the [plugin](#deepseek-harness-plugin): the agent calls injected
|
||||
`browser_*` tools, and the plugin invokes `bsk` on its behalf.
|
||||
|
||||
## For Developers
|
||||
|
||||
@@ -176,6 +207,8 @@ The repository is a Cargo + pnpm workspace:
|
||||
- `crates/bsk-protocol` — shared wire types and JSON schemas
|
||||
- `apps/extension` — browser extension
|
||||
- `packages/ui` and `packages/i18n` — shared extension UI support
|
||||
- `packages/dsh-plugin-browserskill` — DeepSeek Harness plugin (`@wxg-prc-cpg/browser-skill-dsh-plugin`)
|
||||
- [`evals/browser`](evals/browser/README.md) — deterministic local pages and agent-neutral browser capability evaluation
|
||||
|
||||
## License
|
||||
|
||||
|
||||
+33
-7
@@ -12,7 +12,7 @@
|
||||
<a href="README.md">English</a> · 中文
|
||||
</p>
|
||||
|
||||
**BrowserSkill** 把 Cursor、Claude Code、Codex、OpenClaw、CodeBuddy、WorkBuddy、Pi、Hermes Agent 等支持 Shell 的 AI Agent 连接到你已登录的浏览器。
|
||||
**BrowserSkill** 把 Cursor、Claude Code、Codex、OpenClaw、CodeBuddy、WorkBuddy、Pi、Hermes Agent、DeepSeek Harness 等 AI Agent 连接到你已登录的浏览器。
|
||||
|
||||
需要 Agent 操作你已打开的标签页?必须显式借用该标签,任务结束后归还,其余浏览器窗口不受影响。
|
||||
|
||||
@@ -54,7 +54,8 @@ BrowserSkill 由两个本地运行组件组成:`bsk` CLI/daemon 和浏览器
|
||||
|
||||
<br>
|
||||
|
||||
先安装 CLI,再从 [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi) 安装浏览器扩展。
|
||||
先安装 CLI,再从 [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi)
|
||||
或 [Edge 加载项商店](https://microsoftedge.microsoft.com/addons/detail/browserskill/emacgiaaaiojkkpkddmmdfhmokgmnikg) 安装浏览器扩展。
|
||||
|
||||
#### 1. 安装 `bsk` CLI
|
||||
|
||||
@@ -64,8 +65,11 @@ BrowserSkill 由两个本地运行组件组成:`bsk` CLI/daemon 和浏览器
|
||||
curl -fsSL https://raw.githubusercontent.com/Tencent/BrowserSkill/main/install.sh | sh
|
||||
```
|
||||
|
||||
**Windows**:从 [最新 CLI release](https://github.com/Tencent/BrowserSkill/releases/latest)
|
||||
下载 `bsk-v<version>-x86_64-pc-windows-msvc.zip`,解压后将 `bsk.exe` 加入 `PATH`。
|
||||
**Windows**(PowerShell,安装到 `~/.local/bin`):
|
||||
|
||||
```powershell
|
||||
irm https://raw.githubusercontent.com/Tencent/BrowserSkill/main/install.ps1 | iex
|
||||
```
|
||||
|
||||
验证二进制:
|
||||
|
||||
@@ -75,7 +79,14 @@ bsk --version
|
||||
|
||||
#### 2. 安装浏览器扩展
|
||||
|
||||
从 [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi) 安装 BrowserSkill。
|
||||
在对应浏览器的商店安装 BrowserSkill:
|
||||
|
||||
| 浏览器 | 商店页面 |
|
||||
| --- | --- |
|
||||
| Chrome | [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi) |
|
||||
| Microsoft Edge | [Edge 加载项商店](https://microsoftedge.microsoft.com/addons/detail/browserskill/emacgiaaaiojkkpkddmmdfhmokgmnikg) |
|
||||
|
||||
其他基于 Chromium 的浏览器,安装 Chrome Web Store 版本即可。
|
||||
|
||||
#### 3. 安装 skill
|
||||
|
||||
@@ -102,7 +113,7 @@ bsk install-skill
|
||||
|
||||
用 <kbd>Space</kbd> 选择需要安装的 Agent harness,然后按 <kbd>Enter</kbd> 安装 skill。运行 `bsk install-skill --list` 可查看 internal 变体及安装路径。
|
||||
|
||||
其他支持 Shell 的 Agent harness 也可使用 BrowserSkill,但需手动将 [`skill/SKILL.md`](skill/SKILL.md) 复制到对应 skills 目录下的 `browser-skill/SKILL.md`。
|
||||
其他支持 Shell 的 Agent harness 也可使用 BrowserSkill,但需手动将 [`skill/SKILL.md`](skill/SKILL.md) 复制到对应 skills 目录下的 `browser-skill/SKILL.md`。DeepSeek Harness 走独立插件,见 [DeepSeek Harness 插件](#deepseek-harness-插件)。
|
||||
|
||||
</details>
|
||||
|
||||
@@ -112,6 +123,19 @@ bsk install-skill
|
||||
/browser-skill open example.com and summarize what is on the page.
|
||||
```
|
||||
|
||||
## DeepSeek Harness 插件
|
||||
|
||||
在用 [DeepSeek Harness](https://github.com/deepseek-ai/deepseek-harness)(`dsh`)?BrowserSkill 提供了官方 dsh 插件,已发布到 npm:[`@wxg-prc-cpg/browser-skill-dsh-plugin`](https://www.npmjs.com/package/@wxg-prc-cpg/browser-skill-dsh-plugin)。它会注入原生 `browser_*` 工具(无需再通过 Shell 调用 `bsk`),并在 Web UI 中实时观察每个 Agent Window。
|
||||
|
||||
把它装进某个 dsh profile,然后启动该 profile:
|
||||
|
||||
```sh
|
||||
dsh plugin --profile web add @wxg-prc-cpg/browser-skill-dsh-plugin
|
||||
dsh --profile web
|
||||
```
|
||||
|
||||
插件自带 skill,所以在 dsh 下无需执行 `bsk install-skill`;但 `bsk` CLI 和浏览器扩展仍是前置条件。工具清单、配置项与观察浮层见[插件 README](packages/dsh-plugin-browserskill/README.md)。
|
||||
|
||||
## 工作原理
|
||||
|
||||
BrowserSkill 是 Agent 运行时与浏览器之间的本地桥接层。
|
||||
@@ -143,7 +167,7 @@ flowchart TB
|
||||
style UserWindows fill:#f8fafc,stroke:#cbd5e1,color:#334155
|
||||
```
|
||||
|
||||
Agent 不直接与浏览器通信。它通过 `bsk` CLI 下发浏览器任务;本地 daemon 把请求路由到扩展;扩展在 Agent Window 中执行。
|
||||
Agent 不直接与浏览器通信。它通过 `bsk` CLI 下发浏览器任务;本地 daemon 把请求路由到扩展;扩展在 Agent Window 中执行。DeepSeek Harness 走同一条链路,只是经由 [插件](#deepseek-harness-插件):Agent 调用注入的 `browser_*` 工具,由插件代为执行 `bsk`。
|
||||
|
||||
## 面向开发者
|
||||
|
||||
@@ -153,6 +177,8 @@ Agent 不直接与浏览器通信。它通过 `bsk` CLI 下发浏览器任务;
|
||||
- `crates/bsk-protocol` — 共享协议类型与 JSON Schema
|
||||
- `apps/extension` — 浏览器扩展
|
||||
- `packages/ui` 和 `packages/i18n` — 扩展 UI 共享支持
|
||||
- `packages/dsh-plugin-browserskill` — DeepSeek Harness 插件(`@wxg-prc-cpg/browser-skill-dsh-plugin`)
|
||||
- [`evals/browser`](evals/browser/README.zh-CN.md) — 确定性本地页面与 Agent 无关的浏览器能力测试台
|
||||
|
||||
## 许可证
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@browser-skill/extension",
|
||||
"version": "0.1.6",
|
||||
"version": "0.1.7",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { type CdpDebuggerApi, ChromiumCdp } from "../chromium-cdp";
|
||||
import { type CdpDebuggee, type CdpDebuggerApi, ChromiumCdp } from "../chromium-cdp";
|
||||
|
||||
function fakeChromeEvent<TArgs extends unknown[]>() {
|
||||
const listeners = new Set<(...args: TArgs) => void>();
|
||||
@@ -14,7 +14,7 @@ function fakeChromeEvent<TArgs extends unknown[]>() {
|
||||
}
|
||||
|
||||
function fakeApi() {
|
||||
const onEvent = fakeChromeEvent<[chrome.debugger.Debuggee, string, unknown]>();
|
||||
const onEvent = fakeChromeEvent<[CdpDebuggee, string, unknown]>();
|
||||
const onDetach = fakeChromeEvent<[chrome.debugger.Debuggee, string]>();
|
||||
const api: CdpDebuggerApi = {
|
||||
attach: vi.fn(async () => {}),
|
||||
@@ -29,6 +29,135 @@ function fakeApi() {
|
||||
}
|
||||
|
||||
describe("ChromiumCdp", () => {
|
||||
it("discovers multiple iframe targets and recursively routes nested OOPIF commands", async () => {
|
||||
const { api, onEvent } = fakeApi();
|
||||
(api.sendCommand as ReturnType<typeof vi.fn>).mockImplementation(
|
||||
async (
|
||||
target: chrome.debugger.Debuggee & { sessionId?: string },
|
||||
method: string,
|
||||
params?: { frameId?: string },
|
||||
) => {
|
||||
if (method === "DOM.getFrameOwner") {
|
||||
return { backendNodeId: params?.frameId === "nested" ? 300 : 200 };
|
||||
}
|
||||
if (method !== "Page.getFrameTree") return {};
|
||||
if (target.sessionId === "right-session") {
|
||||
return {
|
||||
frameTree: {
|
||||
frame: { id: "right", url: "https://right.test" },
|
||||
childFrames: [
|
||||
{ frame: { id: "nested", parentId: "right", url: "https://nested.test" } },
|
||||
],
|
||||
},
|
||||
};
|
||||
}
|
||||
if (target.sessionId === "nested-session") {
|
||||
return { frameTree: { frame: { id: "nested", url: "https://nested.test" } } };
|
||||
}
|
||||
return {
|
||||
frameTree: {
|
||||
frame: { id: "main", url: "https://app.test" },
|
||||
childFrames: [
|
||||
{ frame: { id: "left", parentId: "main", url: "https://left.test" } },
|
||||
{
|
||||
frame: { id: "right", parentId: "main", url: "https://right.test" },
|
||||
childFrames: [
|
||||
{ frame: { id: "nested", parentId: "right", url: "https://nested.test" } },
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
};
|
||||
},
|
||||
);
|
||||
const cdp = new ChromiumCdp(api);
|
||||
await cdp.ensureAttached(4);
|
||||
onEvent.fire({ tabId: 4 }, "Target.attachedToTarget", { sessionId: "right-session" });
|
||||
onEvent.fire({ tabId: 4 } as chrome.debugger.Debuggee, "Target.attachedToTarget", {
|
||||
sessionId: "nested-session",
|
||||
});
|
||||
|
||||
const graph = await cdp.getFrameGraph(4);
|
||||
|
||||
expect(graph.frames).toHaveLength(4);
|
||||
expect(graph.frames.find((frame) => frame.frameId === "left")?.target).toEqual({ tabId: 4 });
|
||||
expect(graph.frames.find((frame) => frame.frameId === "right")?.target.sessionId).toBe(
|
||||
"right-session",
|
||||
);
|
||||
expect(graph.frames.find((frame) => frame.frameId === "nested")?.target.sessionId).toBe(
|
||||
"nested-session",
|
||||
);
|
||||
expect(graph.frames.find((frame) => frame.frameId === "right")?.ownerBackendNodeId).toBe(200);
|
||||
expect(graph.frames.find((frame) => frame.frameId === "nested")?.ownerBackendNodeId).toBe(300);
|
||||
expect(api.sendCommand).toHaveBeenCalledWith(
|
||||
{ tabId: 4, sessionId: "right-session" },
|
||||
"DOM.getFrameOwner",
|
||||
{ frameId: "nested" },
|
||||
);
|
||||
expect(api.sendCommand).toHaveBeenCalledWith(
|
||||
{ tabId: 4, sessionId: "nested-session" },
|
||||
"Target.setAutoAttach",
|
||||
expect.objectContaining({ flatten: true }),
|
||||
);
|
||||
});
|
||||
|
||||
it("waits for recursively attached iframe targets without a depth limit", async () => {
|
||||
const { api, onEvent } = fakeApi();
|
||||
const depth = 6;
|
||||
const scheduled = new Set<string>();
|
||||
const nestedTree = (index: number): Record<string, unknown> => ({
|
||||
frame: {
|
||||
id: index === 0 ? "main" : `frame-${index}`,
|
||||
...(index > 0 ? { parentId: index === 1 ? "main" : `frame-${index - 1}` } : {}),
|
||||
},
|
||||
...(index < depth ? { childFrames: [nestedTree(index + 1)] } : {}),
|
||||
});
|
||||
(api.sendCommand as ReturnType<typeof vi.fn>).mockImplementation(
|
||||
async (target: CdpDebuggee, method: string, params?: { frameId?: string }) => {
|
||||
if (method === "Target.setAutoAttach") {
|
||||
const parentIndex = target.sessionId
|
||||
? Number(target.sessionId.replace("session-", ""))
|
||||
: 0;
|
||||
const nextIndex = parentIndex + 1;
|
||||
const key = `${target.sessionId ?? "root"}:${nextIndex}`;
|
||||
if (nextIndex <= depth && !scheduled.has(key)) {
|
||||
scheduled.add(key);
|
||||
setTimeout(() => {
|
||||
onEvent.fire(
|
||||
{ tabId: 4, ...(target.sessionId ? { sessionId: target.sessionId } : {}) },
|
||||
"Target.attachedToTarget",
|
||||
{
|
||||
sessionId: `session-${nextIndex}`,
|
||||
targetInfo: { type: "iframe" },
|
||||
},
|
||||
);
|
||||
}, 5);
|
||||
}
|
||||
return {};
|
||||
}
|
||||
if (method === "Page.getFrameTree") {
|
||||
if (!target.sessionId) return { frameTree: nestedTree(0) };
|
||||
const index = Number(target.sessionId.replace("session-", ""));
|
||||
return { frameTree: nestedTree(index) };
|
||||
}
|
||||
if (method === "DOM.getFrameOwner") {
|
||||
return { backendNodeId: Number(params?.frameId?.replace("frame-", "")) + 100 };
|
||||
}
|
||||
return {};
|
||||
},
|
||||
);
|
||||
const cdp = new ChromiumCdp(api);
|
||||
|
||||
const graph = await cdp.getFrameGraph(4);
|
||||
|
||||
expect(graph.frames).toHaveLength(depth + 1);
|
||||
for (let index = 1; index <= depth; index += 1) {
|
||||
expect(
|
||||
graph.frames.find((frame) => frame.frameId === `frame-${index}`)?.target.sessionId,
|
||||
).toBe(`session-${index}`);
|
||||
}
|
||||
});
|
||||
|
||||
it("coalesces concurrent attach calls for the same tab", async () => {
|
||||
const { api } = fakeApi();
|
||||
let releaseAttach!: () => void;
|
||||
|
||||
@@ -0,0 +1,136 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { buildFrameGraph, type CdpFrameTreeNode, type CdpFrameTreeSource } from "../frame-graph";
|
||||
|
||||
describe("buildFrameGraph", () => {
|
||||
it("keeps sibling frames distinct and routes nested OOPIFs to their child sessions", () => {
|
||||
const graph = buildFrameGraph([
|
||||
{
|
||||
target: { tabId: 4 },
|
||||
tree: {
|
||||
frame: { id: "main", url: "https://app.test" },
|
||||
childFrames: [
|
||||
{ frame: { id: "left", parentId: "main", url: "https://left.test" } },
|
||||
{
|
||||
frame: { id: "right", parentId: "main", url: "https://right.test" },
|
||||
childFrames: [
|
||||
{ frame: { id: "nested", parentId: "right", url: "https://nested.test" } },
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
target: { tabId: 4, sessionId: "right-session" },
|
||||
tree: {
|
||||
frame: { id: "right", url: "https://right.test" },
|
||||
childFrames: [{ frame: { id: "nested", parentId: "right", url: "https://nested.test" } }],
|
||||
},
|
||||
},
|
||||
{
|
||||
target: { tabId: 4, sessionId: "nested-session" },
|
||||
tree: { frame: { id: "nested", url: "https://nested.test" } },
|
||||
},
|
||||
]);
|
||||
|
||||
expect(graph?.frames).toHaveLength(4);
|
||||
expect(graph?.frames.find((frame) => frame.frameId === "left")?.target).toEqual({ tabId: 4 });
|
||||
expect(graph?.frames.find((frame) => frame.frameId === "right")).toMatchObject({
|
||||
parentFrameId: "main",
|
||||
target: { tabId: 4, sessionId: "right-session" },
|
||||
});
|
||||
expect(graph?.frames.find((frame) => frame.frameId === "nested")).toMatchObject({
|
||||
parentFrameId: "right",
|
||||
target: { tabId: 4, sessionId: "nested-session" },
|
||||
});
|
||||
});
|
||||
|
||||
it("propagates each target boundary to its same-process descendants", () => {
|
||||
const sources: CdpFrameTreeSource[] = [
|
||||
{
|
||||
target: { tabId: 4 },
|
||||
tree: {
|
||||
frame: { id: "main" },
|
||||
childFrames: [
|
||||
{
|
||||
frame: { id: "oopif-a", parentId: "main" },
|
||||
childFrames: [
|
||||
{
|
||||
frame: { id: "same-process-b", parentId: "oopif-a" },
|
||||
childFrames: [
|
||||
{
|
||||
frame: { id: "oopif-c", parentId: "same-process-b" },
|
||||
childFrames: [{ frame: { id: "same-process-d", parentId: "oopif-c" } }],
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
target: { tabId: 4, sessionId: "session-a" },
|
||||
tree: {
|
||||
frame: { id: "oopif-a" },
|
||||
childFrames: [
|
||||
{
|
||||
frame: { id: "same-process-b", parentId: "oopif-a" },
|
||||
childFrames: [{ frame: { id: "oopif-c", parentId: "same-process-b" } }],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
target: { tabId: 4, sessionId: "session-c" },
|
||||
tree: {
|
||||
frame: { id: "oopif-c" },
|
||||
childFrames: [{ frame: { id: "same-process-d", parentId: "oopif-c" } }],
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
const graph = buildFrameGraph(sources);
|
||||
|
||||
expect(graph?.frames.map((frame) => [frame.frameId, frame.target.sessionId])).toEqual([
|
||||
["main", undefined],
|
||||
["oopif-a", "session-a"],
|
||||
["same-process-b", "session-a"],
|
||||
["oopif-c", "session-c"],
|
||||
["same-process-d", "session-c"],
|
||||
]);
|
||||
expect(graph?.frames.find((frame) => frame.frameId === "oopif-a")?.parentFrameId).toBe("main");
|
||||
expect(graph?.frames.find((frame) => frame.frameId === "oopif-c")?.parentFrameId).toBe(
|
||||
"same-process-b",
|
||||
);
|
||||
|
||||
const reordered = buildFrameGraph([sources[2], sources[1], sources[0]]);
|
||||
expect(reordered).toEqual(graph);
|
||||
});
|
||||
|
||||
it("walks deeply nested frame trees without consuming the JavaScript call stack", () => {
|
||||
const depth = 5_000;
|
||||
const root: CdpFrameTreeNode = { frame: { id: "frame-0" } };
|
||||
let parent = root;
|
||||
for (let index = 1; index <= depth; index += 1) {
|
||||
const child: CdpFrameTreeNode = {
|
||||
frame: { id: `frame-${index}`, parentId: `frame-${index - 1}` },
|
||||
};
|
||||
parent.childFrames = [child];
|
||||
parent = child;
|
||||
}
|
||||
|
||||
const graph = buildFrameGraph([{ target: { tabId: 4 }, tree: root }]);
|
||||
|
||||
expect(graph?.frames).toHaveLength(depth + 1);
|
||||
expect(graph?.frames.at(-1)?.frameId).toBe(`frame-${depth}`);
|
||||
});
|
||||
|
||||
it("does not loop when an in-memory frame tree contains an object cycle", () => {
|
||||
const root: CdpFrameTreeNode = { frame: { id: "main" } };
|
||||
root.childFrames = [root];
|
||||
|
||||
expect(buildFrameGraph([{ target: { tabId: 4 }, tree: root }])?.frames).toEqual([
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -31,6 +31,15 @@ import type {
|
||||
NetworkEntryKind,
|
||||
NetworkResult,
|
||||
} from "@/transport/types";
|
||||
import {
|
||||
buildFrameGraph,
|
||||
type CdpFrameGraph,
|
||||
type CdpFrameTreeNode,
|
||||
type CdpFrameTreeSource,
|
||||
type CdpTarget,
|
||||
} from "./frame-graph";
|
||||
|
||||
export type CdpDebuggee = chrome.debugger.Debuggee & { sessionId?: string };
|
||||
|
||||
/**
|
||||
* Minimal slice of `chrome.debugger` the rest of the extension
|
||||
@@ -38,21 +47,15 @@ import type {
|
||||
* fake without monkey-patching the real `chrome` global.
|
||||
*/
|
||||
export interface CdpDebuggerApi {
|
||||
attach(target: chrome.debugger.Debuggee, requiredVersion: string): Promise<void>;
|
||||
detach(target: chrome.debugger.Debuggee): Promise<void>;
|
||||
sendCommand(
|
||||
target: chrome.debugger.Debuggee,
|
||||
method: string,
|
||||
commandParams?: object,
|
||||
): Promise<unknown>;
|
||||
attach(target: CdpDebuggee, requiredVersion: string): Promise<void>;
|
||||
detach(target: CdpDebuggee): Promise<void>;
|
||||
sendCommand(target: CdpDebuggee, method: string, commandParams?: object): Promise<unknown>;
|
||||
/**
|
||||
* Fires for every CDP event (`Page.lifecycleEvent`, `DOM.documentUpdated`,
|
||||
* …). The first callback argument is the source debuggee; the second
|
||||
* is the CDP method name; the third is the payload.
|
||||
*/
|
||||
onEvent: chrome.events.Event<
|
||||
(source: chrome.debugger.Debuggee, method: string, params: unknown) => void
|
||||
>;
|
||||
onEvent: chrome.events.Event<(source: CdpDebuggee, method: string, params: unknown) => void>;
|
||||
/**
|
||||
* Fires when Chrome unilaterally detaches us — most commonly because
|
||||
* the tab navigated to a chrome:// URL or the user clicked
|
||||
@@ -110,6 +113,8 @@ const MAX_CONSOLE_STACK_FRAMES = 20;
|
||||
const MAX_NETWORK_BUFFER = 200;
|
||||
const MAX_NETWORK_FIELD_LENGTH = 4096;
|
||||
const MAX_NETWORK_REQUEST_META = 1024;
|
||||
const FRAME_DISCOVERY_TIMEOUT_MS = 1000;
|
||||
const FRAME_DISCOVERY_QUIET_MS = 20;
|
||||
|
||||
interface ParsedDialogOpening {
|
||||
type: JavaScriptDialogType;
|
||||
@@ -131,6 +136,27 @@ interface NetworkRequestMeta {
|
||||
truncated: boolean;
|
||||
}
|
||||
|
||||
interface FrameDiscoveryState {
|
||||
sessions: Set<string>;
|
||||
pending: Set<Promise<void>>;
|
||||
generation: number;
|
||||
}
|
||||
|
||||
async function settleBeforeDeadline(promises: Promise<void>[], deadline: number): Promise<boolean> {
|
||||
if (promises.length === 0) return true;
|
||||
const remaining = deadline - Date.now();
|
||||
if (remaining <= 0) return false;
|
||||
let timer: ReturnType<typeof setTimeout> | undefined;
|
||||
const timedOut = await Promise.race([
|
||||
Promise.allSettled(promises).then(() => false),
|
||||
new Promise<boolean>((resolve) => {
|
||||
timer = setTimeout(() => resolve(true), remaining);
|
||||
}),
|
||||
]);
|
||||
if (timer) clearTimeout(timer);
|
||||
return !timedOut;
|
||||
}
|
||||
|
||||
/**
|
||||
* Wrapper around `chrome.debugger` that owns the "attach once per
|
||||
* tabId" cache and exposes typed `send<T>()`.
|
||||
@@ -149,10 +175,12 @@ export class ChromiumCdp {
|
||||
private readonly networkSequences = new Map<number, number>();
|
||||
private readonly networkDomainsEnabledTabs = new Set<number>();
|
||||
private readonly networkRequestMeta = new Map<number, Map<string, NetworkRequestMeta>>();
|
||||
private readonly frameDiscovery = new Map<number, FrameDiscoveryState>();
|
||||
private detachSubscription: { dispose(): void } | null = null;
|
||||
private dialogSubscription: { dispose(): void } | null = null;
|
||||
private consoleSubscription: { dispose(): void } | null = null;
|
||||
private networkSubscription: { dispose(): void } | null = null;
|
||||
private frameTargetSubscription: { dispose(): void } | null = null;
|
||||
|
||||
constructor(api: CdpDebuggerApi = chromeDebuggerApi) {
|
||||
this.api = api;
|
||||
@@ -160,6 +188,7 @@ export class ChromiumCdp {
|
||||
this.bindDialogHandler();
|
||||
this.bindConsoleHandler();
|
||||
this.bindNetworkHandler();
|
||||
this.bindFrameTargetHandler();
|
||||
}
|
||||
|
||||
/** Attach to `tabId` if we haven't already in this driver. */
|
||||
@@ -177,6 +206,9 @@ export class ChromiumCdp {
|
||||
await this.enableConsoleDomains(tabId);
|
||||
await this.enableNetworkDomainBestEffort(tabId);
|
||||
this.attachedTabs.add(tabId);
|
||||
await this.enableFrameDiscovery({ tabId }).catch((err) => {
|
||||
console.debug("[bsk cdp] frame discovery unavailable", { tabId, err });
|
||||
});
|
||||
} catch (err) {
|
||||
// A CDP domain enable failed after the raw attach succeeded
|
||||
// (e.g. `Page.enable` rejects because the tab just navigated to
|
||||
@@ -222,6 +254,73 @@ export class ChromiumCdp {
|
||||
}
|
||||
}
|
||||
|
||||
async sendToTarget<T = unknown>(target: CdpTarget, method: string, params?: object): Promise<T> {
|
||||
if (!this.attachedTabs.has(target.tabId)) {
|
||||
await this.ensureAttached(target.tabId);
|
||||
}
|
||||
try {
|
||||
return (await this.api.sendCommand(target, method, params ?? {})) as T;
|
||||
} catch (err) {
|
||||
throw normalizeError(err);
|
||||
}
|
||||
}
|
||||
|
||||
async getFrameGraph(tabId: number): Promise<CdpFrameGraph> {
|
||||
await this.ensureAttached(tabId);
|
||||
await this.enableFrameDiscovery({ tabId }).catch(() => {});
|
||||
await this.drainFrameAttachTasks(tabId);
|
||||
|
||||
const sources: CdpFrameTreeSource[] = [];
|
||||
const root = await this.sendToTarget<{ frameTree?: CdpFrameTreeNode }>(
|
||||
{ tabId },
|
||||
"Page.getFrameTree",
|
||||
{},
|
||||
);
|
||||
if (root.frameTree) sources.push({ target: { tabId }, tree: root.frameTree });
|
||||
|
||||
const sessions = [...(this.frameDiscovery.get(tabId)?.sessions ?? [])];
|
||||
const childTrees = await Promise.all(
|
||||
sessions.map(async (sessionId): Promise<CdpFrameTreeSource | null> => {
|
||||
const target = { tabId, sessionId };
|
||||
try {
|
||||
const reply = await this.sendToTarget<{ frameTree?: CdpFrameTreeNode }>(
|
||||
target,
|
||||
"Page.getFrameTree",
|
||||
{},
|
||||
);
|
||||
return reply.frameTree ? { target, tree: reply.frameTree } : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}),
|
||||
);
|
||||
for (const source of childTrees) {
|
||||
if (source) sources.push(source);
|
||||
}
|
||||
|
||||
const graph = buildFrameGraph(sources);
|
||||
if (!graph) throw new Error("Page.getFrameTree returned no root frame");
|
||||
const frameById = new Map(graph.frames.map((frame) => [frame.frameId, frame]));
|
||||
await Promise.all(
|
||||
graph.frames.map(async (frame) => {
|
||||
if (!frame.parentFrameId) return;
|
||||
const parent = frameById.get(frame.parentFrameId);
|
||||
if (!parent) return;
|
||||
try {
|
||||
const owner = await this.sendToTarget<{ backendNodeId?: number }>(
|
||||
parent.target,
|
||||
"DOM.getFrameOwner",
|
||||
{ frameId: frame.frameId },
|
||||
);
|
||||
if (owner.backendNodeId !== undefined) frame.ownerBackendNodeId = owner.backendNodeId;
|
||||
} catch {
|
||||
// The frame may have navigated between tree capture and owner lookup.
|
||||
}
|
||||
}),
|
||||
);
|
||||
return graph;
|
||||
}
|
||||
|
||||
/** Return a cursor marking the current dialog sequence for `tabId`. */
|
||||
dialogCursor(tabId: number): DialogCursor {
|
||||
return this.dialogSequences.get(tabId) ?? 0;
|
||||
@@ -332,6 +431,7 @@ export class ChromiumCdp {
|
||||
this.clearDialogState(tabId);
|
||||
this.clearConsoleState(tabId);
|
||||
this.clearNetworkState(tabId);
|
||||
this.clearFrameState(tabId);
|
||||
try {
|
||||
await this.api.detach({ tabId });
|
||||
} catch (err) {
|
||||
@@ -354,7 +454,7 @@ export class ChromiumCdp {
|
||||
}
|
||||
|
||||
/** Subscribe to all CDP events. Returned disposable removes the listener. */
|
||||
onEvent(handler: (source: chrome.debugger.Debuggee, method: string, params: unknown) => void): {
|
||||
onEvent(handler: (source: CdpDebuggee, method: string, params: unknown) => void): {
|
||||
dispose(): void;
|
||||
} {
|
||||
this.api.onEvent.addListener(handler);
|
||||
@@ -378,6 +478,7 @@ export class ChromiumCdp {
|
||||
this.networkSequences.clear();
|
||||
this.networkDomainsEnabledTabs.clear();
|
||||
this.networkRequestMeta.clear();
|
||||
this.frameDiscovery.clear();
|
||||
await Promise.all(
|
||||
tabs.map(async (tabId) => {
|
||||
try {
|
||||
@@ -428,9 +529,104 @@ export class ChromiumCdp {
|
||||
}
|
||||
}
|
||||
|
||||
private async enableFrameDiscovery(target: CdpTarget): Promise<void> {
|
||||
await this.api.sendCommand(target, "Target.setAutoAttach", {
|
||||
autoAttach: true,
|
||||
waitForDebuggerOnStart: false,
|
||||
flatten: true,
|
||||
filter: [{ type: "iframe", exclude: false }],
|
||||
});
|
||||
}
|
||||
|
||||
private bindFrameTargetHandler(): void {
|
||||
if (this.frameTargetSubscription) return;
|
||||
const listener = (source: CdpDebuggee, method: string, params: unknown) => {
|
||||
const tabId = source.tabId;
|
||||
if (typeof tabId !== "number") return;
|
||||
const raw = (params ?? {}) as Record<string, unknown>;
|
||||
const sessionId = typeof raw.sessionId === "string" ? raw.sessionId : undefined;
|
||||
if (!sessionId) return;
|
||||
|
||||
if (method === "Target.detachedFromTarget") {
|
||||
const state = this.frameDiscovery.get(tabId);
|
||||
if (state?.sessions.delete(sessionId)) state.generation += 1;
|
||||
return;
|
||||
}
|
||||
if (method !== "Target.attachedToTarget") return;
|
||||
const targetInfo = raw.targetInfo as { type?: string } | undefined;
|
||||
if (targetInfo?.type && targetInfo.type !== "iframe") return;
|
||||
|
||||
const state = this.frameDiscoveryState(tabId);
|
||||
if (state.sessions.has(sessionId)) return;
|
||||
state.sessions.add(sessionId);
|
||||
state.generation += 1;
|
||||
|
||||
const task = this.initializeFrameTarget({ tabId, sessionId });
|
||||
state.pending.add(task);
|
||||
void task.finally(() => {
|
||||
state.pending.delete(task);
|
||||
});
|
||||
};
|
||||
this.api.onEvent.addListener(listener);
|
||||
this.frameTargetSubscription = {
|
||||
dispose: () => this.api.onEvent.removeListener(listener),
|
||||
};
|
||||
}
|
||||
|
||||
private async initializeFrameTarget(target: CdpTarget): Promise<void> {
|
||||
try {
|
||||
await this.enableFrameDiscovery(target);
|
||||
} catch (err) {
|
||||
const state = this.frameDiscovery.get(target.tabId);
|
||||
if (state?.sessions.delete(target.sessionId as string)) state.generation += 1;
|
||||
console.debug("[bsk cdp] child frame target initialization failed", { target, err });
|
||||
}
|
||||
}
|
||||
|
||||
private async drainFrameAttachTasks(tabId: number): Promise<void> {
|
||||
const deadline = Date.now() + FRAME_DISCOVERY_TIMEOUT_MS;
|
||||
while (Date.now() < deadline) {
|
||||
const state = this.frameDiscovery.get(tabId);
|
||||
const generation = state?.generation ?? 0;
|
||||
if (state?.pending.size && !(await settleBeforeDeadline([...state.pending], deadline))) break;
|
||||
|
||||
// Target.setAutoAttach may enqueue the next attachedToTarget event after
|
||||
// its command promise settles. Wait for a short quiet window, then finish
|
||||
// only if neither the state object nor its generation changed.
|
||||
const quietTime = Math.min(FRAME_DISCOVERY_QUIET_MS, deadline - Date.now());
|
||||
if (quietTime <= 0) break;
|
||||
await new Promise<void>((resolve) => setTimeout(resolve, quietTime));
|
||||
const current = this.frameDiscovery.get(tabId);
|
||||
if (
|
||||
current === state &&
|
||||
(current?.generation ?? 0) === generation &&
|
||||
!current?.pending.size
|
||||
) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
console.debug("[bsk cdp] frame discovery did not reach quiescence before timeout", { tabId });
|
||||
}
|
||||
|
||||
private frameDiscoveryState(tabId: number): FrameDiscoveryState {
|
||||
const existing = this.frameDiscovery.get(tabId);
|
||||
if (existing) return existing;
|
||||
const created: FrameDiscoveryState = {
|
||||
sessions: new Set(),
|
||||
pending: new Set(),
|
||||
generation: 0,
|
||||
};
|
||||
this.frameDiscovery.set(tabId, created);
|
||||
return created;
|
||||
}
|
||||
|
||||
private clearFrameState(tabId: number): void {
|
||||
this.frameDiscovery.delete(tabId);
|
||||
}
|
||||
|
||||
private bindDialogHandler(): void {
|
||||
if (this.dialogSubscription) return;
|
||||
const listener = (source: chrome.debugger.Debuggee, method: string, params: unknown) => {
|
||||
const listener = (source: CdpDebuggee, method: string, params: unknown) => {
|
||||
if (method !== "Page.javascriptDialogOpening") return;
|
||||
const tabId = source.tabId;
|
||||
if (typeof tabId !== "number") return;
|
||||
@@ -614,6 +810,7 @@ export class ChromiumCdp {
|
||||
this.clearDialogState(source.tabId);
|
||||
this.clearConsoleState(source.tabId);
|
||||
this.clearNetworkState(source.tabId);
|
||||
this.clearFrameState(source.tabId);
|
||||
}
|
||||
};
|
||||
this.api.onDetach.addListener(listener);
|
||||
@@ -632,6 +829,8 @@ export class ChromiumCdp {
|
||||
this.consoleSubscription = null;
|
||||
this.networkSubscription?.dispose();
|
||||
this.networkSubscription = null;
|
||||
this.frameTargetSubscription?.dispose();
|
||||
this.frameTargetSubscription = null;
|
||||
}
|
||||
|
||||
/** Detach tabs only when no other live session has claimed them. */
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
export interface CdpTarget {
|
||||
tabId: number;
|
||||
sessionId?: string;
|
||||
}
|
||||
|
||||
export interface CdpFrame {
|
||||
frameId: string;
|
||||
parentFrameId?: string;
|
||||
ownerBackendNodeId?: number;
|
||||
url?: string;
|
||||
target: CdpTarget;
|
||||
}
|
||||
|
||||
export interface CdpFrameGraph {
|
||||
rootFrameId: string;
|
||||
frames: CdpFrame[];
|
||||
}
|
||||
|
||||
export interface CdpFrameTreeNode {
|
||||
frame: {
|
||||
id: string;
|
||||
parentId?: string;
|
||||
url?: string;
|
||||
};
|
||||
childFrames?: CdpFrameTreeNode[];
|
||||
}
|
||||
|
||||
export interface CdpFrameTreeSource {
|
||||
target: CdpTarget;
|
||||
tree: CdpFrameTreeNode;
|
||||
}
|
||||
|
||||
export function cdpTargetKey(target: CdpTarget): string {
|
||||
return `${target.tabId}:${target.sessionId ?? "root"}`;
|
||||
}
|
||||
|
||||
function mergeFrame(
|
||||
frames: Map<string, CdpFrame>,
|
||||
order: string[],
|
||||
node: CdpFrameTreeNode,
|
||||
target: CdpTarget,
|
||||
): void {
|
||||
const existing = frames.get(node.frame.id);
|
||||
if (!existing) order.push(node.frame.id);
|
||||
const parentFrameId = node.frame.parentId || existing?.parentFrameId;
|
||||
const url = node.frame.url || existing?.url;
|
||||
frames.set(node.frame.id, {
|
||||
frameId: node.frame.id,
|
||||
...(parentFrameId ? { parentFrameId } : {}),
|
||||
...(url ? { url } : {}),
|
||||
target,
|
||||
});
|
||||
}
|
||||
|
||||
function walkFrameTree(
|
||||
source: CdpFrameTreeSource,
|
||||
targetByRootFrameId: ReadonlyMap<string, CdpTarget>,
|
||||
frames: Map<string, CdpFrame>,
|
||||
order: string[],
|
||||
): void {
|
||||
const stack: Array<{ node: CdpFrameTreeNode; inheritedTarget: CdpTarget }> = [
|
||||
{ node: source.tree, inheritedTarget: source.target },
|
||||
];
|
||||
const expanded = new WeakSet<CdpFrameTreeNode>();
|
||||
|
||||
while (stack.length > 0) {
|
||||
const current = stack.pop();
|
||||
if (!current || expanded.has(current.node)) continue;
|
||||
expanded.add(current.node);
|
||||
|
||||
const target = targetByRootFrameId.get(current.node.frame.id) ?? current.inheritedTarget;
|
||||
mergeFrame(frames, order, current.node, target);
|
||||
|
||||
const children = current.node.childFrames ?? [];
|
||||
for (let index = children.length - 1; index >= 0; index -= 1) {
|
||||
stack.push({ node: children[index], inheritedTarget: target });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function buildFrameGraph(sources: CdpFrameTreeSource[]): CdpFrameGraph | null {
|
||||
const rootSource = sources.find((source) => source.target.sessionId === undefined) ?? sources[0];
|
||||
if (!rootSource) return null;
|
||||
|
||||
const frames = new Map<string, CdpFrame>();
|
||||
const order: string[] = [];
|
||||
const targetByRootFrameId = new Map(
|
||||
sources.map((source) => [source.tree.frame.id, source.target] as const),
|
||||
);
|
||||
// The top-level source is authoritative if malformed input reports the same
|
||||
// frame as both a root target and a child target.
|
||||
targetByRootFrameId.set(rootSource.tree.frame.id, rootSource.target);
|
||||
|
||||
walkFrameTree(rootSource, targetByRootFrameId, frames, order);
|
||||
for (const source of sources) {
|
||||
if (source === rootSource) continue;
|
||||
walkFrameTree(source, targetByRootFrameId, frames, order);
|
||||
}
|
||||
|
||||
return {
|
||||
rootFrameId: rootSource.tree.frame.id,
|
||||
frames: order.flatMap((frameId) => {
|
||||
const frame = frames.get(frameId);
|
||||
return frame ? [frame] : [];
|
||||
}),
|
||||
};
|
||||
}
|
||||
@@ -8,6 +8,7 @@ import {
|
||||
useRef,
|
||||
useState,
|
||||
} from "react";
|
||||
import type { HelpHighlightRect } from "@/lib/help-bridge";
|
||||
import logoUrl from "../../assets/logo.png";
|
||||
|
||||
export interface HelpRequestData {
|
||||
@@ -19,6 +20,10 @@ export interface HelpRequestData {
|
||||
displayMode?: "full" | "compact";
|
||||
/** CSS selectors to scroll to + flash-highlight. */
|
||||
selectors: string[];
|
||||
/** Pre-resolved top-viewport rectangles, used for cross-frame targets. */
|
||||
rects?: HelpHighlightRect[];
|
||||
/** Re-resolve cross-frame rectangles after scroll, resize, or layout changes. */
|
||||
refreshRects?: () => Promise<HelpHighlightRect[] | undefined>;
|
||||
onContinue: (note: string) => void;
|
||||
onCancel: () => void;
|
||||
}
|
||||
@@ -49,6 +54,7 @@ const PANEL_WIDTH = 420;
|
||||
const FALLBACK_PANEL_H = 180;
|
||||
const FALLBACK_PANEL_H_COLLAPSED = 44;
|
||||
const EMPTY_SELECTORS: string[] = [];
|
||||
const EMPTY_RECTS: HelpHighlightRect[] = [];
|
||||
|
||||
function clamp(value: number, min: number, max: number): number {
|
||||
return Math.max(min, Math.min(max, value));
|
||||
@@ -179,6 +185,8 @@ export function HelpRequestOverlay({ request }: Props) {
|
||||
const { t } = useTranslation("extension");
|
||||
const isCompact = request?.displayMode === "compact";
|
||||
const effectiveSelectors = isCompact ? EMPTY_SELECTORS : (request?.selectors ?? EMPTY_SELECTORS);
|
||||
const [liveRects, setLiveRects] = useState<HelpHighlightRect[] | null>(null);
|
||||
const effectiveRects = isCompact ? EMPTY_RECTS : (liveRects ?? request?.rects ?? EMPTY_RECTS);
|
||||
const [note, setNote] = useState("");
|
||||
const [collapsed, setCollapsed] = useState(false);
|
||||
const [boxes, setBoxes] = useState<Box[]>([]);
|
||||
@@ -201,6 +209,7 @@ export function HelpRequestOverlay({ request }: Props) {
|
||||
// Reset the note and collapse state whenever a new request appears.
|
||||
useEffect(() => {
|
||||
setNote("");
|
||||
setLiveRects(null);
|
||||
setCollapsed(false);
|
||||
placementRef.current = null;
|
||||
setDragPos(null);
|
||||
@@ -209,6 +218,32 @@ export function HelpRequestOverlay({ request }: Props) {
|
||||
dragStartRef.current = null;
|
||||
}, [request?.id]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!request || isCompact || !request.refreshRects) return;
|
||||
let disposed = false;
|
||||
let resolving = false;
|
||||
const refresh = async () => {
|
||||
if (resolving) return;
|
||||
resolving = true;
|
||||
try {
|
||||
const rects = await request.refreshRects?.();
|
||||
if (!disposed && rects) setLiveRects(rects);
|
||||
} finally {
|
||||
resolving = false;
|
||||
}
|
||||
};
|
||||
const onViewportChange = () => void refresh();
|
||||
window.addEventListener("scroll", onViewportChange, true);
|
||||
window.addEventListener("resize", onViewportChange);
|
||||
const interval = window.setInterval(onViewportChange, 500);
|
||||
return () => {
|
||||
disposed = true;
|
||||
window.removeEventListener("scroll", onViewportChange, true);
|
||||
window.removeEventListener("resize", onViewportChange);
|
||||
window.clearInterval(interval);
|
||||
};
|
||||
}, [request, isCompact]);
|
||||
|
||||
// Scroll the first matched target into view once per distinct request id.
|
||||
useLayoutEffect(() => {
|
||||
if (!request) return;
|
||||
@@ -231,7 +266,7 @@ export function HelpRequestOverlay({ request }: Props) {
|
||||
return;
|
||||
}
|
||||
const update = () => {
|
||||
const nextBoxes = measure(effectiveSelectors);
|
||||
const nextBoxes = [...measure(effectiveSelectors), ...effectiveRects];
|
||||
setBoxes((prev) => (boxesEqual(prev, nextBoxes) ? prev : nextBoxes));
|
||||
};
|
||||
update();
|
||||
@@ -243,7 +278,7 @@ export function HelpRequestOverlay({ request }: Props) {
|
||||
window.removeEventListener("resize", update);
|
||||
window.clearInterval(interval);
|
||||
};
|
||||
}, [request, effectiveSelectors]);
|
||||
}, [request, effectiveSelectors, effectiveRects]);
|
||||
|
||||
const onHeaderPointerDown = useCallback(
|
||||
(e: ReactPointerEvent<HTMLDivElement>) => {
|
||||
@@ -305,7 +340,7 @@ export function HelpRequestOverlay({ request }: Props) {
|
||||
return;
|
||||
}
|
||||
if (userMovedRef.current) return;
|
||||
const measured = boxes.length > 0 ? boxes : measure(effectiveSelectors);
|
||||
const measured = boxes.length > 0 ? boxes : [...measure(effectiveSelectors), ...effectiveRects];
|
||||
const anchor = unionBox(measured);
|
||||
const banner = bannerRef.current;
|
||||
if (!anchor || !banner) {
|
||||
@@ -329,7 +364,7 @@ export function HelpRequestOverlay({ request }: Props) {
|
||||
}
|
||||
const nextPos = placePanel(anchor, panelW, panelH, placementRef.current.placement);
|
||||
setPanelPos((prev) => (panelPosEqual(prev, nextPos) ? prev : nextPos));
|
||||
}, [request, boxes, collapsed, isCompact, effectiveSelectors]);
|
||||
}, [request, boxes, collapsed, isCompact, effectiveSelectors, effectiveRects]);
|
||||
|
||||
if (!request) return null;
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { i18n } from "@browser-skill/i18n";
|
||||
import { I18nextProvider } from "@browser-skill/i18n/react";
|
||||
import { cleanup, fireEvent, render, screen } from "@testing-library/react";
|
||||
import { cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react";
|
||||
import { createElement } from "react";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { type HelpRequestData, HelpRequestOverlay } from "../HelpRequestOverlay";
|
||||
@@ -38,6 +38,36 @@ describe("HelpRequestOverlay", () => {
|
||||
expect(screen.getByText("Please complete the captcha")).toBeTruthy();
|
||||
});
|
||||
|
||||
it("renders explicit viewport rectangles for cross-frame targets", () => {
|
||||
const { container } = renderOverlay(
|
||||
baseRequest({ rects: [{ top: 30, left: 40, width: 120, height: 50 }] }),
|
||||
);
|
||||
const highlight = container.querySelector<HTMLElement>("[data-slot='help-highlight']");
|
||||
expect(highlight?.style.top).toBe("30px");
|
||||
expect(highlight?.style.left).toBe("40px");
|
||||
expect(highlight?.style.width).toBe("120px");
|
||||
expect(highlight?.style.height).toBe("50px");
|
||||
});
|
||||
|
||||
it("re-resolves cross-frame rectangles after the viewport changes", async () => {
|
||||
const refreshRects = vi.fn(async () => [{ top: 80, left: 90, width: 140, height: 60 }]);
|
||||
const { container } = renderOverlay(
|
||||
baseRequest({
|
||||
rects: [{ top: 30, left: 40, width: 120, height: 50 }],
|
||||
refreshRects,
|
||||
}),
|
||||
);
|
||||
|
||||
window.dispatchEvent(new Event("scroll"));
|
||||
|
||||
await waitFor(() => {
|
||||
const highlight = container.querySelector<HTMLElement>("[data-slot='help-highlight']");
|
||||
expect(highlight?.style.top).toBe("80px");
|
||||
expect(highlight?.style.left).toBe("90px");
|
||||
});
|
||||
expect(refreshRects).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("keeps the inactive render stable", () => {
|
||||
const { container, rerender } = renderOverlay(null);
|
||||
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { createHelpRequestData } from "../help-request";
|
||||
|
||||
describe("createHelpRequestData", () => {
|
||||
it("gives live delivery and recovery the same rectangle refresh mechanism", async () => {
|
||||
const finish = vi.fn();
|
||||
const query = vi.fn(async () => ({
|
||||
active: true,
|
||||
request: {
|
||||
requestId: "help-1",
|
||||
prompt: "complete the challenge",
|
||||
selectors: [],
|
||||
rects: [{ top: 80, left: 90, width: 140, height: 60 }],
|
||||
timeoutMs: 1_000,
|
||||
},
|
||||
}));
|
||||
const request = createHelpRequestData(
|
||||
{
|
||||
requestId: "help-1",
|
||||
prompt: "complete the challenge",
|
||||
selectors: [],
|
||||
rects: [{ top: 10, left: 20, width: 30, height: 40 }],
|
||||
timeoutMs: 1_000,
|
||||
},
|
||||
{ finish, query },
|
||||
);
|
||||
|
||||
await expect(request.refreshRects?.()).resolves.toEqual([
|
||||
{ top: 80, left: 90, width: 140, height: 60 },
|
||||
]);
|
||||
request.onContinue(" done ");
|
||||
request.onCancel();
|
||||
|
||||
expect(query).toHaveBeenCalledOnce();
|
||||
expect(finish).toHaveBeenNthCalledWith(1, "help-1", "continued", " done ");
|
||||
expect(finish).toHaveBeenNthCalledWith(2, "help-1", "cancelled");
|
||||
});
|
||||
|
||||
it("ignores rectangle replies for a replaced request", async () => {
|
||||
const request = createHelpRequestData(
|
||||
{
|
||||
requestId: "help-1",
|
||||
prompt: "complete the challenge",
|
||||
selectors: [],
|
||||
timeoutMs: 1_000,
|
||||
},
|
||||
{
|
||||
finish: vi.fn(),
|
||||
query: vi.fn(async () => ({
|
||||
active: true,
|
||||
request: {
|
||||
requestId: "help-2",
|
||||
prompt: "new challenge",
|
||||
selectors: [],
|
||||
rects: [{ top: 1, left: 2, width: 3, height: 4 }],
|
||||
timeoutMs: 1_000,
|
||||
},
|
||||
})),
|
||||
},
|
||||
);
|
||||
|
||||
await expect(request.refreshRects?.()).resolves.toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -1,10 +1,12 @@
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { RECORD_STOP, type RecordStepPayload } from "@/lib/record-bridge";
|
||||
import { handleRecordContentMessage, startRecordCapture } from "../record-capture";
|
||||
import type { RecordStepPayload } from "@/lib/record-bridge";
|
||||
import { startRecordCapture } from "../record-capture";
|
||||
|
||||
vi.stubGlobal("chrome", {
|
||||
runtime: {
|
||||
sendMessage: vi.fn(() => Promise.resolve()),
|
||||
sendMessage: vi.fn((message: { sequence?: number }) =>
|
||||
Promise.resolve({ ok: true, sequence: message.sequence }),
|
||||
),
|
||||
},
|
||||
});
|
||||
|
||||
@@ -46,58 +48,6 @@ function click(el: Element): void {
|
||||
el.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 }));
|
||||
}
|
||||
|
||||
describe("handleRecordContentMessage stop/cancel", () => {
|
||||
it("ignores STOP when no recording is active", () => {
|
||||
const dispose = vi.fn();
|
||||
const onStop = vi.fn();
|
||||
const setActiveRequestId = vi.fn();
|
||||
const setCapture = vi.fn();
|
||||
const sendResponse = vi.fn();
|
||||
|
||||
const needsAsync = handleRecordContentMessage(
|
||||
{ type: RECORD_STOP, requestId: "rec-stale" },
|
||||
{
|
||||
activeRequestId: null,
|
||||
capture: { dispose },
|
||||
setActiveRequestId,
|
||||
setCapture,
|
||||
onStart: vi.fn(),
|
||||
onStop,
|
||||
},
|
||||
sendResponse,
|
||||
);
|
||||
|
||||
expect(needsAsync).toBe(false);
|
||||
expect(dispose).not.toHaveBeenCalled();
|
||||
expect(onStop).not.toHaveBeenCalled();
|
||||
expect(setActiveRequestId).not.toHaveBeenCalled();
|
||||
expect(setCapture).not.toHaveBeenCalled();
|
||||
expect(sendResponse).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("ignores STOP for a mismatched requestId", () => {
|
||||
const dispose = vi.fn();
|
||||
const onStop = vi.fn();
|
||||
|
||||
const needsAsync = handleRecordContentMessage(
|
||||
{ type: RECORD_STOP, requestId: "rec-other" },
|
||||
{
|
||||
activeRequestId: "rec-1",
|
||||
capture: { dispose },
|
||||
setActiveRequestId: vi.fn(),
|
||||
setCapture: vi.fn(),
|
||||
onStart: vi.fn(),
|
||||
onStop,
|
||||
},
|
||||
vi.fn(),
|
||||
);
|
||||
|
||||
expect(needsAsync).toBe(false);
|
||||
expect(dispose).not.toHaveBeenCalled();
|
||||
expect(onStop).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe("record-capture semantic", () => {
|
||||
let steps: RecordStepPayload[];
|
||||
|
||||
@@ -174,6 +124,19 @@ describe("record-capture semantic", () => {
|
||||
]);
|
||||
});
|
||||
|
||||
it("does not emit page navigation from a child Document capture", () => {
|
||||
const originalUrl = location.href;
|
||||
const capture = startRecordCapture("rec-child", (step) => steps.push(step), {
|
||||
captureNavigation: false,
|
||||
});
|
||||
|
||||
history.pushState({}, "", "#inside-frame");
|
||||
expect(steps).toEqual([]);
|
||||
|
||||
capture.dispose();
|
||||
history.replaceState({}, "", originalUrl);
|
||||
});
|
||||
|
||||
it("does not record clicks on anonymous layout divs", () => {
|
||||
document.body.innerHTML = `
|
||||
<div id="chrome">page chrome</div>
|
||||
@@ -228,6 +191,7 @@ describe("record-capture semantic", () => {
|
||||
expect(steps[0]).toMatchObject({
|
||||
op: "hover",
|
||||
target: { role: "button", name: "Open user navigation menu" },
|
||||
geometry: { tag: "button", rect: { x: 900, y: 8, w: 32, h: 32 } },
|
||||
});
|
||||
expect(steps[1]).toMatchObject({
|
||||
op: "click",
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import type { RecordFramePortMessage } from "@/lib/recording/frame-bridge";
|
||||
import { RECORD_FRAME_START } from "@/lib/recording/frame-bridge";
|
||||
import { RECORD_DOCUMENT_ATTRIBUTE } from "@/shared/recording-document-identity";
|
||||
import { RecordFrameAgent } from "../recording/frame-agent";
|
||||
|
||||
class PortListeners<T extends (...args: never[]) => unknown> {
|
||||
readonly values = new Set<T>();
|
||||
addListener = (listener: T) => this.values.add(listener);
|
||||
removeListener = (listener: T) => this.values.delete(listener);
|
||||
}
|
||||
|
||||
function portHarness() {
|
||||
const onMessage = new PortListeners<(message: unknown) => void>();
|
||||
const onDisconnect = new PortListeners<() => void>();
|
||||
const outbound: RecordFramePortMessage[] = [];
|
||||
const port = {
|
||||
name: "bsk-record-frame",
|
||||
onMessage,
|
||||
onDisconnect,
|
||||
postMessage(message: RecordFramePortMessage) {
|
||||
outbound.push(message);
|
||||
if (message.type === "ready") {
|
||||
queueMicrotask(() => {
|
||||
for (const listener of onMessage.values) {
|
||||
listener({
|
||||
type: "ready_ack",
|
||||
requestId: message.requestId,
|
||||
producerId: message.producerId,
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
},
|
||||
disconnect: vi.fn(),
|
||||
} as unknown as chrome.runtime.Port;
|
||||
return {
|
||||
port,
|
||||
outbound,
|
||||
receive(message: RecordFramePortMessage) {
|
||||
for (const listener of onMessage.values) listener(message);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe("RecordFrameAgent", () => {
|
||||
afterEach(() => {
|
||||
document.documentElement.removeAttribute(RECORD_DOCUMENT_ATTRIBUTE);
|
||||
document.body.replaceChildren();
|
||||
});
|
||||
|
||||
it("keeps a failed stop retryable and flushes the final dirty fill", async () => {
|
||||
const harness = portHarness();
|
||||
const sendMessage = vi
|
||||
.fn<(message: { sequence: number }) => Promise<unknown>>()
|
||||
.mockRejectedValueOnce(new Error("offline"))
|
||||
.mockRejectedValueOnce(new Error("still offline"))
|
||||
.mockImplementation(async (message) => ({ ok: true, sequence: message.sequence }));
|
||||
vi.stubGlobal("chrome", {
|
||||
runtime: {
|
||||
connect: vi.fn(() => harness.port),
|
||||
sendMessage,
|
||||
},
|
||||
});
|
||||
document.body.innerHTML = `<label for="draft">Draft</label><input id="draft" />`;
|
||||
const agent = new RecordFrameAgent();
|
||||
|
||||
await expect(
|
||||
agent.start({ type: RECORD_FRAME_START, requestId: "rec-1", startedAtMs: 10 }),
|
||||
).resolves.toEqual({ ok: true });
|
||||
expect(document.documentElement.hasAttribute(RECORD_DOCUMENT_ATTRIBUTE)).toBe(true);
|
||||
|
||||
const input = document.querySelector("input")!;
|
||||
input.dispatchEvent(new FocusEvent("focusin", { bubbles: true }));
|
||||
input.value = "final value";
|
||||
input.dispatchEvent(new Event("input", { bubbles: true }));
|
||||
|
||||
harness.receive({ type: "stop", requestId: "rec-1", commandId: "stop-1" });
|
||||
await vi.waitFor(() =>
|
||||
expect(harness.outbound).toContainEqual({
|
||||
type: "stopped",
|
||||
requestId: "rec-1",
|
||||
commandId: "stop-1",
|
||||
ok: false,
|
||||
error: "failed to deliver one or more recorded steps",
|
||||
}),
|
||||
);
|
||||
expect(document.documentElement.hasAttribute(RECORD_DOCUMENT_ATTRIBUTE)).toBe(true);
|
||||
|
||||
harness.receive({ type: "stop", requestId: "rec-1", commandId: "stop-2" });
|
||||
await vi.waitFor(() =>
|
||||
expect(harness.outbound).toContainEqual({
|
||||
type: "stopped",
|
||||
requestId: "rec-1",
|
||||
commandId: "stop-2",
|
||||
ok: true,
|
||||
}),
|
||||
);
|
||||
expect(sendMessage).toHaveBeenCalledTimes(3);
|
||||
expect(document.documentElement.hasAttribute(RECORD_DOCUMENT_ATTRIBUTE)).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,36 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import type { RecordStepMessage } from "@/lib/record-bridge";
|
||||
import { RecordStepDelivery } from "../record-step-delivery";
|
||||
|
||||
describe("RecordStepDelivery", () => {
|
||||
it("retries from the first unacknowledged sequence before sending later steps", async () => {
|
||||
const sent: RecordStepMessage[] = [];
|
||||
const send = vi.fn(async (message: RecordStepMessage) => {
|
||||
sent.push(message);
|
||||
if (sent.length === 1) throw new Error("service worker unavailable");
|
||||
return { ok: true, sequence: message.sequence };
|
||||
});
|
||||
const delivery = new RecordStepDelivery("rec-ordered", send, "document-1");
|
||||
|
||||
delivery.enqueue({ op: "click", target: { tag: "button", name: "First" } });
|
||||
delivery.enqueue({ op: "click", target: { tag: "button", name: "Second" } });
|
||||
|
||||
await expect(delivery.flush()).resolves.toBe(true);
|
||||
expect(sent.map((message) => message.sequence)).toEqual([1, 1, 2]);
|
||||
expect(sent.map((message) => message.step.target?.name)).toEqual(["First", "First", "Second"]);
|
||||
});
|
||||
|
||||
it("keeps unacknowledged steps pending for a later flush", async () => {
|
||||
const send = vi
|
||||
.fn<(message: RecordStepMessage) => Promise<unknown>>()
|
||||
.mockRejectedValueOnce(new Error("offline"))
|
||||
.mockRejectedValueOnce(new Error("still offline"))
|
||||
.mockImplementation(async (message) => ({ ok: true, sequence: message.sequence }));
|
||||
const delivery = new RecordStepDelivery("rec-retry", send, "document-1");
|
||||
delivery.enqueue({ op: "click", target: { tag: "button", name: "Save" } });
|
||||
|
||||
await expect(delivery.flush()).resolves.toBe(false);
|
||||
await expect(delivery.flush()).resolves.toBe(true);
|
||||
expect(send).toHaveBeenCalledTimes(3);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,31 @@
|
||||
import type { HelpQueryResponse, HelpRequestMessage } from "@/lib/help-bridge";
|
||||
import type { HelpRequestData } from "./HelpRequestOverlay";
|
||||
|
||||
export interface HelpRequestDataDeps {
|
||||
finish(requestId: string, outcome: "continued" | "cancelled", note?: string): void;
|
||||
query(): Promise<HelpQueryResponse | undefined>;
|
||||
}
|
||||
|
||||
/** Build the content-side representation shared by live delivery and recovery. */
|
||||
export function createHelpRequestData(
|
||||
message: Omit<HelpRequestMessage, "type">,
|
||||
deps: HelpRequestDataDeps,
|
||||
): HelpRequestData {
|
||||
return {
|
||||
id: message.requestId,
|
||||
prompt: message.prompt,
|
||||
...(message.title ? { title: message.title } : {}),
|
||||
...(message.displayMode ? { displayMode: message.displayMode } : {}),
|
||||
selectors: message.selectors,
|
||||
rects: message.rects,
|
||||
refreshRects: async () => {
|
||||
const response = await deps.query();
|
||||
return response?.active && response.request?.requestId === message.requestId
|
||||
? response.request.rects
|
||||
: undefined;
|
||||
},
|
||||
onContinue: (note: string) =>
|
||||
deps.finish(message.requestId, "continued", note.trim() ? note : undefined),
|
||||
onCancel: () => deps.finish(message.requestId, "cancelled"),
|
||||
};
|
||||
}
|
||||
@@ -1,9 +1,9 @@
|
||||
import {
|
||||
type CaptureTargetDescriptor,
|
||||
describeEventTarget,
|
||||
describeTarget,
|
||||
resolveClickableElement,
|
||||
resolveHoverElement,
|
||||
type TargetDescriptor,
|
||||
} from "@/lib/describe-target";
|
||||
import {
|
||||
evaluateHoverTrigger,
|
||||
@@ -12,22 +12,8 @@ import {
|
||||
hasDirectHoverInteractiveSignal,
|
||||
hasStrongHoverExpansionSignal,
|
||||
} from "@/lib/hover-trigger-policy";
|
||||
import {
|
||||
isRecordCancelMessage,
|
||||
isRecordStartMessage,
|
||||
isRecordStopMessage,
|
||||
RECORD_CANCEL,
|
||||
RECORD_START,
|
||||
RECORD_STEP,
|
||||
RECORD_STOP,
|
||||
type RecordCancelMessage,
|
||||
type RecordStartAck,
|
||||
type RecordStartMessage,
|
||||
type RecordStepPayload,
|
||||
type RecordStopAck,
|
||||
type RecordStopMessage,
|
||||
} from "@/lib/record-bridge";
|
||||
import { shouldRecordPress } from "@/lib/trace-reducer";
|
||||
import type { RecordStepPayload } from "@/lib/record-bridge";
|
||||
import { shouldRecordPress } from "@/lib/recording/draft-policy";
|
||||
import {
|
||||
closestHoverSurfaceCandidate,
|
||||
collectHoverSurfaceStates,
|
||||
@@ -37,42 +23,21 @@ import {
|
||||
isLikelyHoverSurfaceOwner,
|
||||
} from "./record-hover-surface";
|
||||
|
||||
const pendingStepSends = new Map<string, Set<Promise<boolean>>>();
|
||||
const failedStepDeliveries = new Set<string>();
|
||||
const knownRecordRequests = new Set<string>();
|
||||
|
||||
function sendRecordStep(requestId: string, step: RecordStepPayload): void {
|
||||
const payload = { type: RECORD_STEP, requestId, step };
|
||||
const pending = Promise.resolve(chrome.runtime.sendMessage(payload)).then(
|
||||
() => true,
|
||||
() => {
|
||||
failedStepDeliveries.add(requestId);
|
||||
return false;
|
||||
},
|
||||
);
|
||||
const sends = pendingStepSends.get(requestId) ?? new Set<Promise<boolean>>();
|
||||
sends.add(pending);
|
||||
pendingStepSends.set(requestId, sends);
|
||||
void pending.then(() => {
|
||||
sends.delete(pending);
|
||||
if (sends.size === 0) pendingStepSends.delete(requestId);
|
||||
});
|
||||
}
|
||||
|
||||
export interface RecordCaptureController {
|
||||
dispose(): void;
|
||||
}
|
||||
|
||||
interface FillSession {
|
||||
element: FillableElement;
|
||||
target: TargetDescriptor;
|
||||
target: CaptureTargetDescriptor;
|
||||
baselineValue: string;
|
||||
lastValue: string;
|
||||
pendingCommit?: "enter" | "suggestion" | "blur";
|
||||
}
|
||||
|
||||
interface HoverCandidate {
|
||||
element: Element;
|
||||
target: TargetDescriptor;
|
||||
target: CaptureTargetDescriptor;
|
||||
recordedAt: number;
|
||||
score: number;
|
||||
eligible: boolean;
|
||||
@@ -157,6 +122,24 @@ function nearbyFillableFromSearchChrome(target: Element): FillableElement | null
|
||||
return null;
|
||||
}
|
||||
|
||||
function captureGeometry(el: Element): RecordStepPayload["geometry"] {
|
||||
const rect = el.getBoundingClientRect();
|
||||
return {
|
||||
rect: { x: rect.x, y: rect.y, w: rect.width, h: rect.height },
|
||||
tag: el.tagName.toLowerCase(),
|
||||
};
|
||||
}
|
||||
|
||||
function geometryForEventTarget(
|
||||
target: EventTarget | null,
|
||||
): RecordStepPayload["geometry"] | undefined {
|
||||
if (!(target instanceof Element)) return undefined;
|
||||
const clickable = target.closest(
|
||||
'a,button,input,select,textarea,[role="button"],[role="link"],[role="menuitem"],[contenteditable="true"]',
|
||||
);
|
||||
return captureGeometry(clickable ?? target);
|
||||
}
|
||||
|
||||
function fillableValue(el: FillableElement): string {
|
||||
if (el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement) {
|
||||
return el.value;
|
||||
@@ -232,7 +215,10 @@ function collectHoverTriggerLabelText(root: Element): string {
|
||||
return normalizeLabelText(text);
|
||||
}
|
||||
|
||||
function compactHoverTargetName(el: Element, desc: TargetDescriptor): TargetDescriptor {
|
||||
function compactHoverTargetName(
|
||||
el: Element,
|
||||
desc: CaptureTargetDescriptor,
|
||||
): CaptureTargetDescriptor {
|
||||
if (!desc.name) return desc;
|
||||
const fullText = normalizeLabelText(el.textContent ?? "");
|
||||
const compactName = collectHoverTriggerLabelText(el);
|
||||
@@ -250,7 +236,7 @@ function compactHoverTargetName(el: Element, desc: TargetDescriptor): TargetDesc
|
||||
return desc;
|
||||
}
|
||||
|
||||
function isWeakHoverTarget(target: TargetDescriptor): boolean {
|
||||
function isWeakHoverTarget(target: CaptureTargetDescriptor): boolean {
|
||||
return !target.role && !target.name && target.tag === "div";
|
||||
}
|
||||
|
||||
@@ -261,9 +247,9 @@ function looksLikeAvatarElement(el: Element): boolean {
|
||||
|
||||
function normalizeHoverTarget(
|
||||
el: Element,
|
||||
desc: TargetDescriptor,
|
||||
desc: CaptureTargetDescriptor,
|
||||
decision: HoverTriggerDecision,
|
||||
): TargetDescriptor {
|
||||
): CaptureTargetDescriptor {
|
||||
if (desc.role === "img" && !desc.name && looksLikeAvatarElement(el)) {
|
||||
return { ...desc, name: "image" };
|
||||
}
|
||||
@@ -277,7 +263,7 @@ function normalizeHoverTarget(
|
||||
return desc;
|
||||
}
|
||||
|
||||
function hoverTriggerSignals(el: Element, desc: TargetDescriptor) {
|
||||
function hoverTriggerSignals(el: Element, desc: CaptureTargetDescriptor) {
|
||||
if (!(el instanceof HTMLElement)) return null;
|
||||
const style = hoverTriggerStyle(el);
|
||||
return {
|
||||
@@ -373,6 +359,7 @@ function scheduleInputCompletionCommit(
|
||||
export function startRecordCapture(
|
||||
_requestId: string,
|
||||
sendStep: (step: RecordStepPayload) => void,
|
||||
options: { captureNavigation?: boolean } = {},
|
||||
): RecordCaptureController {
|
||||
const emitStep = (step: RecordStepPayload) => {
|
||||
sendStep({ page_url: location.href, ...step });
|
||||
@@ -411,14 +398,17 @@ export function startRecordCapture(
|
||||
emitStep({
|
||||
op: "fill",
|
||||
target: session.target,
|
||||
geometry: captureGeometry(session.element),
|
||||
value,
|
||||
commit: session.pendingCommit ?? "blur",
|
||||
...(isPassword ? { redacted: true } : {}),
|
||||
});
|
||||
};
|
||||
|
||||
const commitFillSession = () => {
|
||||
const commitFillSession = (commit: "enter" | "suggestion" | "blur" = "blur") => {
|
||||
if (!fillSession || composing) return;
|
||||
const session = fillSession;
|
||||
session.pendingCommit = commit;
|
||||
fillSession = null;
|
||||
emitFill(session);
|
||||
committedValues.set(session.element, session.lastValue);
|
||||
@@ -624,6 +614,7 @@ export function startRecordCapture(
|
||||
emitStep({
|
||||
op: "hover",
|
||||
target: hover.target,
|
||||
geometry: captureGeometry(hover.element),
|
||||
});
|
||||
emittedHoverElements.add(hover.element);
|
||||
};
|
||||
@@ -678,6 +669,7 @@ export function startRecordCapture(
|
||||
emitStep({
|
||||
op: "click",
|
||||
target,
|
||||
geometry: geometryForEventTarget(eventTarget(event)),
|
||||
expects_navigation: true,
|
||||
});
|
||||
};
|
||||
@@ -779,7 +771,7 @@ export function startRecordCapture(
|
||||
scheduleInputCompletionCommit(
|
||||
sessionElement,
|
||||
syncFillSessionValue,
|
||||
commitFillSession,
|
||||
() => commitFillSession("suggestion"),
|
||||
(el) => fillSession?.element === el,
|
||||
);
|
||||
return;
|
||||
@@ -839,6 +831,7 @@ export function startRecordCapture(
|
||||
emitStep({
|
||||
op: "select",
|
||||
target: desc,
|
||||
geometry: captureGeometry(target),
|
||||
values,
|
||||
labels,
|
||||
expects_navigation: true,
|
||||
@@ -879,7 +872,7 @@ export function startRecordCapture(
|
||||
};
|
||||
}
|
||||
if (fillable) {
|
||||
commitFillSession();
|
||||
commitFillSession(event.key === "Enter" ? "enter" : "blur");
|
||||
}
|
||||
const desc = describeEventTarget(target);
|
||||
if (!desc && !event.key) return;
|
||||
@@ -887,7 +880,7 @@ export function startRecordCapture(
|
||||
emitStep({
|
||||
op: "press",
|
||||
key: event.key,
|
||||
...(desc ? { target: desc } : {}),
|
||||
...(desc ? { target: desc, geometry: geometryForEventTarget(target) } : {}),
|
||||
...(modifiers.length ? { modifiers } : {}),
|
||||
expects_navigation: event.key === "Enter",
|
||||
});
|
||||
@@ -903,23 +896,33 @@ export function startRecordCapture(
|
||||
document.addEventListener("change", onChange, true);
|
||||
document.addEventListener("keydown", onKeyDown, true);
|
||||
|
||||
const urlObserver = new MutationObserver(() => emitNavigateIfChanged());
|
||||
urlObserver.observe(document, { subtree: true, childList: true });
|
||||
const captureNavigation = options.captureNavigation ?? true;
|
||||
const urlObserver = captureNavigation
|
||||
? new MutationObserver(() => emitNavigateIfChanged())
|
||||
: null;
|
||||
urlObserver?.observe(document, { subtree: true, childList: true });
|
||||
const onUrlEvent = () => emitNavigateIfChanged();
|
||||
window.addEventListener("hashchange", onUrlEvent);
|
||||
window.addEventListener("popstate", onUrlEvent);
|
||||
window.addEventListener("pagehide", commitFillSession);
|
||||
// Wrap: passing commitFillSession directly would forward the DOM event as
|
||||
// the `commit` argument and stamp it onto the recorded fill step.
|
||||
const onPageHide = () => commitFillSession();
|
||||
if (captureNavigation) {
|
||||
window.addEventListener("hashchange", onUrlEvent);
|
||||
window.addEventListener("popstate", onUrlEvent);
|
||||
}
|
||||
window.addEventListener("pagehide", onPageHide);
|
||||
|
||||
const originalPushState = history.pushState;
|
||||
const originalReplaceState = history.replaceState;
|
||||
history.pushState = function (...args: Parameters<History["pushState"]>) {
|
||||
originalPushState.apply(this, args);
|
||||
emitNavigateIfChanged(navigationActionPending ? true : undefined);
|
||||
};
|
||||
history.replaceState = function (...args: Parameters<History["replaceState"]>) {
|
||||
originalReplaceState.apply(this, args);
|
||||
emitNavigateIfChanged(navigationActionPending ? true : undefined);
|
||||
};
|
||||
if (captureNavigation) {
|
||||
history.pushState = function (...args: Parameters<History["pushState"]>) {
|
||||
originalPushState.apply(this, args);
|
||||
emitNavigateIfChanged(navigationActionPending ? true : undefined);
|
||||
};
|
||||
history.replaceState = function (...args: Parameters<History["replaceState"]>) {
|
||||
originalReplaceState.apply(this, args);
|
||||
emitNavigateIfChanged(navigationActionPending ? true : undefined);
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
dispose() {
|
||||
@@ -933,93 +936,16 @@ export function startRecordCapture(
|
||||
document.removeEventListener("compositionend", onCompositionEnd, true);
|
||||
document.removeEventListener("change", onChange, true);
|
||||
document.removeEventListener("keydown", onKeyDown, true);
|
||||
urlObserver.disconnect();
|
||||
window.removeEventListener("hashchange", onUrlEvent);
|
||||
window.removeEventListener("popstate", onUrlEvent);
|
||||
window.removeEventListener("pagehide", commitFillSession);
|
||||
history.pushState = originalPushState;
|
||||
history.replaceState = originalReplaceState;
|
||||
urlObserver?.disconnect();
|
||||
if (captureNavigation) {
|
||||
window.removeEventListener("hashchange", onUrlEvent);
|
||||
window.removeEventListener("popstate", onUrlEvent);
|
||||
}
|
||||
window.removeEventListener("pagehide", onPageHide);
|
||||
if (captureNavigation) {
|
||||
history.pushState = originalPushState;
|
||||
history.replaceState = originalReplaceState;
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export type RecordContentMessage = RecordStartMessage | RecordStopMessage | RecordCancelMessage;
|
||||
|
||||
export function isRecordContentMessage(msg: unknown): msg is RecordContentMessage {
|
||||
return isRecordStartMessage(msg) || isRecordStopMessage(msg) || isRecordCancelMessage(msg);
|
||||
}
|
||||
|
||||
export function handleRecordContentMessage(
|
||||
message: RecordContentMessage,
|
||||
state: {
|
||||
activeRequestId: string | null;
|
||||
capture: RecordCaptureController | null;
|
||||
setActiveRequestId(id: string | null): void;
|
||||
setCapture(capture: RecordCaptureController | null): void;
|
||||
onStart(requestId: string, startedAtMs?: number): void;
|
||||
onStop(): void;
|
||||
},
|
||||
sendResponse?: (response: RecordStartAck | RecordStopAck) => void,
|
||||
): boolean {
|
||||
if (isRecordStartMessage(message)) {
|
||||
if (!knownRecordRequests.has(message.requestId)) {
|
||||
knownRecordRequests.add(message.requestId);
|
||||
failedStepDeliveries.delete(message.requestId);
|
||||
}
|
||||
state.capture?.dispose();
|
||||
state.setCapture(
|
||||
startRecordCapture(message.requestId, (step) => {
|
||||
sendRecordStep(message.requestId, step);
|
||||
}),
|
||||
);
|
||||
state.setActiveRequestId(message.requestId);
|
||||
state.onStart(message.requestId, message.startedAtMs);
|
||||
sendResponse?.({ ok: true });
|
||||
return sendResponse !== undefined;
|
||||
}
|
||||
|
||||
if (isRecordStopMessage(message) || isRecordCancelMessage(message)) {
|
||||
// Require an active recording that matches this requestId — otherwise a
|
||||
// stray STOP/CANCEL (e.g. after teardown) would still run finishStop and
|
||||
// clear overlay state even though nothing was capturing.
|
||||
if (!state.activeRequestId || state.activeRequestId !== message.requestId) {
|
||||
return false;
|
||||
}
|
||||
state.capture?.dispose();
|
||||
state.setCapture(null);
|
||||
const finishStop = () => {
|
||||
state.onStop();
|
||||
state.setActiveRequestId(null);
|
||||
};
|
||||
if (isRecordStopMessage(message) && sendResponse) {
|
||||
const pending = [...(pendingStepSends.get(message.requestId) ?? [])];
|
||||
void Promise.all(pending).then((delivered) => {
|
||||
finishStop();
|
||||
const succeeded = delivered.every(Boolean) && !failedStepDeliveries.has(message.requestId);
|
||||
if (succeeded) {
|
||||
failedStepDeliveries.delete(message.requestId);
|
||||
knownRecordRequests.delete(message.requestId);
|
||||
}
|
||||
sendResponse(
|
||||
succeeded
|
||||
? { ok: true }
|
||||
: {
|
||||
ok: false,
|
||||
error: "failed to deliver one or more recorded steps",
|
||||
},
|
||||
);
|
||||
});
|
||||
return true;
|
||||
}
|
||||
if (isRecordCancelMessage(message)) {
|
||||
failedStepDeliveries.delete(message.requestId);
|
||||
knownRecordRequests.delete(message.requestId);
|
||||
}
|
||||
finishStop();
|
||||
return false;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
export { RECORD_CANCEL, RECORD_START, RECORD_STEP, RECORD_STOP };
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
import {
|
||||
isAcceptedRecordStepAck,
|
||||
RECORD_STEP,
|
||||
type RecordStepMessage,
|
||||
type RecordStepPayload,
|
||||
} from "@/lib/record-bridge";
|
||||
|
||||
export type SendRecordStepMessage = (message: RecordStepMessage) => Promise<unknown>;
|
||||
|
||||
export class RecordStepDelivery {
|
||||
readonly #requestId: string;
|
||||
readonly #producerId: string;
|
||||
readonly #send: SendRecordStepMessage;
|
||||
readonly #pending = new Map<number, RecordStepPayload>();
|
||||
#nextSequence = 1;
|
||||
#tail = Promise.resolve();
|
||||
|
||||
constructor(
|
||||
requestId: string,
|
||||
send: SendRecordStepMessage = (message) => chrome.runtime.sendMessage(message),
|
||||
producerId: string = crypto.randomUUID(),
|
||||
) {
|
||||
this.#requestId = requestId;
|
||||
this.#producerId = producerId;
|
||||
this.#send = send;
|
||||
}
|
||||
|
||||
enqueue(step: RecordStepPayload): void {
|
||||
const sequence = this.#nextSequence;
|
||||
this.#nextSequence += 1;
|
||||
this.#pending.set(sequence, step);
|
||||
void this.#schedule();
|
||||
}
|
||||
|
||||
async flush(): Promise<boolean> {
|
||||
await this.#tail;
|
||||
await this.#schedule();
|
||||
return this.#pending.size === 0;
|
||||
}
|
||||
|
||||
#schedule(): Promise<void> {
|
||||
const drain = () => this.#drain();
|
||||
this.#tail = this.#tail.then(drain, drain);
|
||||
return this.#tail;
|
||||
}
|
||||
|
||||
async #drain(): Promise<void> {
|
||||
for (const [sequence, step] of this.#pending) {
|
||||
const message: RecordStepMessage = {
|
||||
type: RECORD_STEP,
|
||||
requestId: this.#requestId,
|
||||
producerId: this.#producerId,
|
||||
sequence,
|
||||
step,
|
||||
};
|
||||
try {
|
||||
const response = await this.#send(message);
|
||||
if (!isAcceptedRecordStepAck(response, sequence)) return;
|
||||
this.#pending.delete(sequence);
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,209 @@
|
||||
import {
|
||||
markRecordingDocument,
|
||||
type RecordingDocumentMarker,
|
||||
waitForDocumentElement,
|
||||
} from "@/lib/recording/document-marker";
|
||||
import {
|
||||
isRecordFrameStartMessage,
|
||||
RECORD_FRAME_PORT,
|
||||
RECORD_FRAME_QUERY,
|
||||
RECORD_FRAME_START,
|
||||
type RecordFramePortMessage,
|
||||
type RecordFrameQueryResponse,
|
||||
type RecordFrameStartMessage,
|
||||
} from "@/lib/recording/frame-bridge";
|
||||
import { type RecordCaptureController, startRecordCapture } from "../record-capture";
|
||||
import { RecordStepDelivery } from "../record-step-delivery";
|
||||
|
||||
interface ActiveFrameRecording {
|
||||
requestId: string;
|
||||
producerId: string;
|
||||
port: chrome.runtime.Port;
|
||||
delivery: RecordStepDelivery;
|
||||
capture: RecordCaptureController;
|
||||
marker: RecordingDocumentMarker;
|
||||
}
|
||||
|
||||
const RECORD_FRAME_REGISTER_TIMEOUT_MS = 5_000;
|
||||
|
||||
export class RecordFrameAgent {
|
||||
#active: ActiveFrameRecording | null = null;
|
||||
#startPromise: Promise<{ ok: true } | { ok: false; error: string }> | null = null;
|
||||
#disposed = false;
|
||||
|
||||
async start(
|
||||
message: RecordFrameStartMessage,
|
||||
): Promise<{ ok: true } | { ok: false; error: string }> {
|
||||
if (this.#active?.requestId === message.requestId) return { ok: true };
|
||||
if (this.#disposed) return { ok: false, error: "recording document is disposed" };
|
||||
if (this.#startPromise) {
|
||||
await this.#startPromise;
|
||||
return this.start(message);
|
||||
}
|
||||
if (this.#active) this.cancel(this.#active.requestId);
|
||||
const starting = this.#start(message);
|
||||
this.#startPromise = starting;
|
||||
try {
|
||||
return await starting;
|
||||
} finally {
|
||||
if (this.#startPromise === starting) this.#startPromise = null;
|
||||
}
|
||||
}
|
||||
|
||||
cancel(requestId: string): void {
|
||||
const active = this.#active;
|
||||
if (!active || active.requestId !== requestId) return;
|
||||
active.capture.dispose();
|
||||
active.marker.restore();
|
||||
active.port.disconnect();
|
||||
this.#active = null;
|
||||
}
|
||||
|
||||
dispose(): void {
|
||||
this.#disposed = true;
|
||||
if (this.#active) this.cancel(this.#active.requestId);
|
||||
}
|
||||
|
||||
async #start(
|
||||
message: RecordFrameStartMessage,
|
||||
): Promise<{ ok: true } | { ok: false; error: string }> {
|
||||
const producerId = crypto.randomUUID();
|
||||
const root = await waitForDocumentElement();
|
||||
if (this.#disposed) return { ok: false, error: "recording document is disposed" };
|
||||
const marker = markRecordingDocument(producerId, root);
|
||||
const port = chrome.runtime.connect({ name: RECORD_FRAME_PORT });
|
||||
try {
|
||||
await registerPort(port, message.requestId, producerId);
|
||||
} catch {
|
||||
marker.restore();
|
||||
port.disconnect();
|
||||
return { ok: false, error: "failed to register recording document" };
|
||||
}
|
||||
if (this.#disposed) {
|
||||
marker.restore();
|
||||
port.disconnect();
|
||||
return { ok: false, error: "recording document is disposed" };
|
||||
}
|
||||
|
||||
const delivery = new RecordStepDelivery(message.requestId, undefined, producerId);
|
||||
const capture = startRecordCapture(
|
||||
message.requestId,
|
||||
(step) => {
|
||||
marker.ensure();
|
||||
delivery.enqueue(step);
|
||||
},
|
||||
{ captureNavigation: window.top === window },
|
||||
);
|
||||
const active: ActiveFrameRecording = {
|
||||
requestId: message.requestId,
|
||||
producerId,
|
||||
port,
|
||||
delivery,
|
||||
capture,
|
||||
marker,
|
||||
};
|
||||
this.#active = active;
|
||||
port.onMessage.addListener((raw: unknown) => {
|
||||
const command = raw as Partial<RecordFramePortMessage>;
|
||||
if (command.type === "stop" && command.requestId === active.requestId) {
|
||||
void this.#stop(active, command.commandId);
|
||||
} else if (command.type === "cancel" && command.requestId === active.requestId) {
|
||||
this.cancel(active.requestId);
|
||||
}
|
||||
});
|
||||
port.onDisconnect.addListener(() => {
|
||||
if (this.#active !== active) return;
|
||||
active.capture.dispose();
|
||||
active.marker.restore();
|
||||
this.#active = null;
|
||||
});
|
||||
return { ok: true };
|
||||
}
|
||||
|
||||
async #stop(active: ActiveFrameRecording, commandId: string | undefined): Promise<void> {
|
||||
if (!commandId || this.#active !== active) return;
|
||||
active.capture.dispose();
|
||||
const succeeded = await active.delivery.flush();
|
||||
try {
|
||||
active.port.postMessage({
|
||||
type: "stopped",
|
||||
requestId: active.requestId,
|
||||
commandId,
|
||||
ok: succeeded,
|
||||
...(succeeded ? {} : { error: "failed to deliver one or more recorded steps" }),
|
||||
} satisfies RecordFramePortMessage);
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
if (!succeeded || this.#active !== active) return;
|
||||
active.marker.restore();
|
||||
this.#active = null;
|
||||
active.port.disconnect();
|
||||
}
|
||||
}
|
||||
|
||||
function registerPort(
|
||||
port: chrome.runtime.Port,
|
||||
requestId: string,
|
||||
producerId: string,
|
||||
): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const timer = setTimeout(() => {
|
||||
cleanup();
|
||||
reject(new Error("recording document registration timed out"));
|
||||
}, RECORD_FRAME_REGISTER_TIMEOUT_MS);
|
||||
const cleanup = () => {
|
||||
clearTimeout(timer);
|
||||
port.onMessage.removeListener(onMessage);
|
||||
port.onDisconnect.removeListener(onDisconnect);
|
||||
};
|
||||
const onMessage = (raw: unknown) => {
|
||||
const message = raw as Partial<RecordFramePortMessage>;
|
||||
if (
|
||||
message.type !== "ready_ack" ||
|
||||
message.requestId !== requestId ||
|
||||
message.producerId !== producerId
|
||||
) {
|
||||
return;
|
||||
}
|
||||
cleanup();
|
||||
resolve();
|
||||
};
|
||||
const onDisconnect = () => {
|
||||
cleanup();
|
||||
reject(new Error("recording document port disconnected"));
|
||||
};
|
||||
port.onMessage.addListener(onMessage);
|
||||
port.onDisconnect.addListener(onDisconnect);
|
||||
port.postMessage({ type: "ready", requestId, producerId } satisfies RecordFramePortMessage);
|
||||
});
|
||||
}
|
||||
|
||||
export function attachRecordFrameAgent(): () => void {
|
||||
const agent = new RecordFrameAgent();
|
||||
const onMessage = (
|
||||
message: unknown,
|
||||
_sender: chrome.runtime.MessageSender,
|
||||
sendResponse: (response: { ok: true } | { ok: false; error: string }) => void,
|
||||
) => {
|
||||
if (!isRecordFrameStartMessage(message)) return false;
|
||||
void agent.start(message).then(sendResponse);
|
||||
return true;
|
||||
};
|
||||
chrome.runtime.onMessage.addListener(onMessage);
|
||||
void chrome.runtime
|
||||
.sendMessage({ type: RECORD_FRAME_QUERY })
|
||||
.then((response: RecordFrameQueryResponse | undefined) => {
|
||||
if (!response?.active || !response.requestId || response.startedAtMs === undefined) return;
|
||||
return agent.start({
|
||||
type: RECORD_FRAME_START,
|
||||
requestId: response.requestId,
|
||||
startedAtMs: response.startedAtMs,
|
||||
});
|
||||
})
|
||||
.catch(() => {});
|
||||
return () => {
|
||||
chrome.runtime.onMessage.removeListener(onMessage);
|
||||
agent.dispose();
|
||||
};
|
||||
}
|
||||
@@ -21,6 +21,7 @@ import {
|
||||
type OverlayMode,
|
||||
} from "@/lib/overlay-bridge";
|
||||
import { POPUP_PORT_NAME, type PopupInbound, type PopupOutbound } from "@/lib/popup-bridge";
|
||||
import { recordFrameCoordinator } from "@/lib/recording/frame-coordinator";
|
||||
import { attachSessionsLiveFlag } from "@/lib/sessions-live-flag";
|
||||
import { createDisconnectCleanup } from "@/session-manager/disconnect-cleanup";
|
||||
import { attachSessionEventHandler } from "@/session-manager/event-handler";
|
||||
@@ -38,6 +39,7 @@ import {
|
||||
attachRecordFinishListener,
|
||||
attachRecordQueryListener,
|
||||
attachRecordStepListener,
|
||||
type RecordRuntimeDeps,
|
||||
} from "@/tools/record";
|
||||
import { detectBrowserMeta } from "@/transport/handshake";
|
||||
import type { Transport } from "@/transport/transport";
|
||||
@@ -145,10 +147,33 @@ export default defineBackground(() => {
|
||||
void sessionsLive.syncFromManager();
|
||||
},
|
||||
});
|
||||
const recordDeps = {
|
||||
tabsApi: chrome.tabs,
|
||||
cdp,
|
||||
frameCoordinator: recordFrameCoordinator,
|
||||
sendToTab: (tabId: number, msg: Parameters<typeof chrome.tabs.sendMessage>[1]) =>
|
||||
chrome.tabs.sendMessage(tabId, msg),
|
||||
bypassOverlay: async (tabId: number, enabled: boolean) => {
|
||||
try {
|
||||
await chrome.tabs.sendMessage(tabId, {
|
||||
type: OVERLAY_AUTOMATION_BYPASS,
|
||||
enabled,
|
||||
});
|
||||
} catch {
|
||||
// Content script may be unavailable on restricted pages.
|
||||
}
|
||||
},
|
||||
} satisfies RecordRuntimeDeps;
|
||||
recordFrameCoordinator.attach();
|
||||
attachRecordStepListener(recordDeps);
|
||||
attachRecordFinishListener(recordDeps);
|
||||
attachRecordQueryListener(recordDeps);
|
||||
|
||||
const dispatcher = new ToolDispatcher({
|
||||
transport,
|
||||
sessions,
|
||||
cdp,
|
||||
recording: recordDeps,
|
||||
onSessionsChanged: onOverlaySessionStateChanged,
|
||||
onBrowserControlResumed,
|
||||
approveBorrow: (ctx) =>
|
||||
@@ -170,27 +195,6 @@ export default defineBackground(() => {
|
||||
}),
|
||||
});
|
||||
dispatcher.start();
|
||||
const recordDeps = {
|
||||
tabsApi: chrome.tabs,
|
||||
sendToTab: (tabId: number, msg: Parameters<typeof chrome.tabs.sendMessage>[1]) =>
|
||||
chrome.tabs.sendMessage(tabId, msg),
|
||||
bypassOverlay: async (tabId: number, enabled: boolean) => {
|
||||
try {
|
||||
await chrome.tabs.sendMessage(tabId, {
|
||||
type: OVERLAY_AUTOMATION_BYPASS,
|
||||
enabled,
|
||||
});
|
||||
} catch {
|
||||
// Content script may be unavailable on restricted pages.
|
||||
}
|
||||
},
|
||||
};
|
||||
// Message listeners stay up (cheap; fire only for record message types).
|
||||
// Tab / webNavigation observation attaches lazily while a recording is
|
||||
// active — see ensureBrowserObservationListeners in tools/record.ts.
|
||||
attachRecordStepListener();
|
||||
attachRecordFinishListener(recordDeps);
|
||||
attachRecordQueryListener(recordDeps);
|
||||
if (typeof chrome.notifications?.onClicked?.addListener === "function") {
|
||||
attachBorrowNotificationClickHandler({
|
||||
onClicked: chrome.notifications.onClicked,
|
||||
|
||||
@@ -7,14 +7,10 @@ import { BorrowConfirmationOverlay } from "@/content/BorrowConfirmationOverlay";
|
||||
import { ControlOverlay } from "@/content/ControlOverlay";
|
||||
import { createCaptureSuppressController } from "@/content/capture-suppress";
|
||||
import { HelpRequestOverlay } from "@/content/HelpRequestOverlay";
|
||||
import { createHelpRequestData } from "@/content/help-request";
|
||||
import overlayCss from "@/content/overlay.css?inline";
|
||||
import { OverlayController, shouldShowAgentControlOverlay } from "@/content/overlay-controller";
|
||||
import { RecordOverlay } from "@/content/RecordOverlay";
|
||||
import {
|
||||
handleRecordContentMessage,
|
||||
isRecordContentMessage,
|
||||
type RecordCaptureController,
|
||||
} from "@/content/record-capture";
|
||||
import {
|
||||
type CaptureSuppressAck,
|
||||
type CaptureSuppressMessage,
|
||||
@@ -44,11 +40,17 @@ import {
|
||||
} from "@/lib/overlay-bridge";
|
||||
import { sendInterrupt } from "@/lib/overlay-interrupt-client";
|
||||
import {
|
||||
isRecordCancelMessage,
|
||||
isRecordStartMessage,
|
||||
isRecordStopMessage,
|
||||
RECORD_FINISH,
|
||||
RECORD_QUERY,
|
||||
type RecordCancelMessage,
|
||||
type RecordQueryResponse,
|
||||
type RecordStartAck,
|
||||
type RecordStartMessage,
|
||||
type RecordStopAck,
|
||||
type RecordStopMessage,
|
||||
} from "@/lib/record-bridge";
|
||||
import type {
|
||||
BorrowCancelMessage,
|
||||
@@ -68,7 +70,6 @@ export default defineContentScript({
|
||||
if (window.top !== window) return;
|
||||
|
||||
const overlays = new OverlayController();
|
||||
let recordCapture: RecordCaptureController | null = null;
|
||||
let activeRecordRequestId: string | null = null;
|
||||
let reactRoot: ReactDOM.Root | null = null;
|
||||
let overlayHost: HTMLElement | null = null;
|
||||
@@ -227,8 +228,6 @@ export default defineContentScript({
|
||||
if (previousHelp) {
|
||||
void sendHelpFinish(previousHelp.id, "cancelled");
|
||||
}
|
||||
recordCapture?.dispose();
|
||||
recordCapture = null;
|
||||
activeRecordRequestId = null;
|
||||
renderAll();
|
||||
}
|
||||
@@ -258,6 +257,9 @@ export default defineContentScript({
|
||||
| HelpRequestMessage
|
||||
| HelpCancelMessage
|
||||
| CaptureSuppressMessage
|
||||
| RecordStartMessage
|
||||
| RecordStopMessage
|
||||
| RecordCancelMessage
|
||||
| OverlayAgentOverlayResetMessage
|
||||
| OverlayAgentStateMessage
|
||||
| OverlayAutomationBypassMessage,
|
||||
@@ -268,41 +270,32 @@ export default defineContentScript({
|
||||
return captureSuppress.handleMessage(message, sendResponse);
|
||||
}
|
||||
|
||||
if (isRecordContentMessage(message)) {
|
||||
const needsAsync = handleRecordContentMessage(
|
||||
message,
|
||||
{
|
||||
activeRequestId: activeRecordRequestId,
|
||||
capture: recordCapture,
|
||||
setActiveRequestId: (id) => {
|
||||
activeRecordRequestId = id;
|
||||
},
|
||||
setCapture: (capture) => {
|
||||
recordCapture = capture;
|
||||
},
|
||||
onStart: (requestId, startedAtMs) => {
|
||||
overlays.setAgentRecordRequest({
|
||||
id: requestId,
|
||||
...(typeof startedAtMs === "number" ? { startedAtMs } : {}),
|
||||
onFinish: () => {
|
||||
void chrome.runtime.sendMessage({
|
||||
type: RECORD_FINISH,
|
||||
requestId,
|
||||
});
|
||||
},
|
||||
});
|
||||
renderAll();
|
||||
},
|
||||
onStop: () => {
|
||||
overlays.clearAgentRecordRequest(activeRecordRequestId ?? undefined);
|
||||
renderAll();
|
||||
},
|
||||
if (isRecordStartMessage(message)) {
|
||||
activeRecordRequestId = message.requestId;
|
||||
overlays.setAgentRecordRequest({
|
||||
id: message.requestId,
|
||||
...(typeof message.startedAtMs === "number" ? { startedAtMs: message.startedAtMs } : {}),
|
||||
onFinish: () => {
|
||||
void chrome.runtime.sendMessage({
|
||||
type: RECORD_FINISH,
|
||||
requestId: message.requestId,
|
||||
});
|
||||
},
|
||||
sendResponse as unknown as
|
||||
| ((response: RecordStartAck | RecordStopAck) => void)
|
||||
| undefined,
|
||||
);
|
||||
return needsAsync;
|
||||
});
|
||||
renderAll();
|
||||
(sendResponse as unknown as (response: RecordStartAck) => void)({ ok: true });
|
||||
return false;
|
||||
}
|
||||
|
||||
if (isRecordStopMessage(message) || isRecordCancelMessage(message)) {
|
||||
if (activeRecordRequestId !== message.requestId) return false;
|
||||
overlays.clearAgentRecordRequest(message.requestId);
|
||||
activeRecordRequestId = null;
|
||||
renderAll();
|
||||
if (isRecordStopMessage(message)) {
|
||||
(sendResponse as unknown as (response: RecordStopAck) => void)({ ok: true });
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
if (
|
||||
@@ -344,16 +337,7 @@ export default defineContentScript({
|
||||
|
||||
if (isHelpRequestMessage(message)) {
|
||||
const helpMsg = message as HelpRequestMessage;
|
||||
const previousHelp = overlays.setAgentHelpRequest({
|
||||
id: helpMsg.requestId,
|
||||
prompt: helpMsg.prompt,
|
||||
...(helpMsg.title ? { title: helpMsg.title } : {}),
|
||||
...(helpMsg.displayMode ? { displayMode: helpMsg.displayMode } : {}),
|
||||
selectors: helpMsg.selectors,
|
||||
onContinue: (note: string) =>
|
||||
void sendHelpFinish(helpMsg.requestId, "continued", note.trim() ? note : undefined),
|
||||
onCancel: () => void sendHelpFinish(helpMsg.requestId, "cancelled"),
|
||||
});
|
||||
const previousHelp = mountHelpRequest(helpMsg);
|
||||
if (previousHelp && previousHelp.id !== helpMsg.requestId) {
|
||||
void sendHelpFinish(previousHelp.id, "cancelled");
|
||||
}
|
||||
@@ -406,17 +390,18 @@ export default defineContentScript({
|
||||
});
|
||||
}
|
||||
|
||||
function mountHelpRequest(helpMsg: Omit<HelpRequestMessage, "type">): void {
|
||||
overlays.setAgentHelpRequest({
|
||||
id: helpMsg.requestId,
|
||||
prompt: helpMsg.prompt,
|
||||
...(helpMsg.title ? { title: helpMsg.title } : {}),
|
||||
...(helpMsg.displayMode ? { displayMode: helpMsg.displayMode } : {}),
|
||||
selectors: helpMsg.selectors,
|
||||
onContinue: (note: string) =>
|
||||
void sendHelpFinish(helpMsg.requestId, "continued", note.trim() ? note : undefined),
|
||||
onCancel: () => void sendHelpFinish(helpMsg.requestId, "cancelled"),
|
||||
});
|
||||
function mountHelpRequest(helpMsg: Omit<HelpRequestMessage, "type">) {
|
||||
return overlays.setAgentHelpRequest(
|
||||
createHelpRequestData(helpMsg, {
|
||||
finish: (requestId, outcome, note) => {
|
||||
void sendHelpFinish(requestId, outcome, note);
|
||||
},
|
||||
query: () =>
|
||||
chrome.runtime.sendMessage({ type: HELP_QUERY }) as Promise<
|
||||
HelpQueryResponse | undefined
|
||||
>,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
async function queryActiveHelpWithRetry(): Promise<void> {
|
||||
@@ -533,9 +518,6 @@ export default defineContentScript({
|
||||
chrome.runtime.onMessage.removeListener(onMessage);
|
||||
chrome.storage.onChanged.removeListener(onStorageChange);
|
||||
window.removeEventListener("pageshow", onPageShow);
|
||||
// Restore history hooks / remove capture listeners before the CS unloads.
|
||||
recordCapture?.dispose();
|
||||
recordCapture = null;
|
||||
activeRecordRequestId = null;
|
||||
});
|
||||
},
|
||||
|
||||
@@ -200,6 +200,9 @@ describe("App", () => {
|
||||
expect(copied).toContain("--browser 03c3e47f");
|
||||
expect(copied).toContain('--purpose "发布 wiki 文档"');
|
||||
expect(copied).not.toMatch(/bsk record start[^\n]*--url/);
|
||||
expect(copied).toContain("./trace");
|
||||
expect(copied).toContain("trace.json");
|
||||
expect(copied).toContain("states/");
|
||||
// Button label stays static; a transient toast confirms the copy.
|
||||
expect(copyButton.textContent).toContain("复制录制指令");
|
||||
await waitFor(() => expect(screen.getByRole("status").textContent).toContain("已复制"));
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
import { attachRecordFrameAgent } from "@/content/recording/frame-agent";
|
||||
|
||||
export default defineContentScript({
|
||||
matches: ["<all_urls>"],
|
||||
runAt: "document_start",
|
||||
allFrames: true,
|
||||
matchAboutBlank: true,
|
||||
matchOriginAsFallback: true,
|
||||
|
||||
main(ctx) {
|
||||
const dispose = attachRecordFrameAgent();
|
||||
ctx.onInvalidated(dispose);
|
||||
},
|
||||
});
|
||||
@@ -26,20 +26,20 @@ function handshake(
|
||||
}
|
||||
|
||||
describe("computeConnectedState (protocol-based compat)", () => {
|
||||
it("returns connected when protocol strings match", () => {
|
||||
expect(computeConnectedState(handshake("1.0", "1.0"), MIN_COMPATIBLE_PROTOCOL)).toEqual({
|
||||
it("returns connected when daemon protocol equals extension protocol", () => {
|
||||
expect(computeConnectedState(handshake("1.1", "1.0"), MIN_COMPATIBLE_PROTOCOL)).toEqual({
|
||||
kind: "connected",
|
||||
});
|
||||
});
|
||||
|
||||
it("returns version_skew when daemon protocol minor is newer", () => {
|
||||
expect(computeConnectedState(handshake("1.1", "1.0"))).toEqual({
|
||||
expect(computeConnectedState(handshake("1.2", "1.0"))).toEqual({
|
||||
kind: "version_skew",
|
||||
});
|
||||
});
|
||||
|
||||
it("returns version_skew when daemon protocol string differs but floor is satisfied", () => {
|
||||
expect(computeConnectedState(handshake("1", "1.0"))).toEqual({
|
||||
expect(computeConnectedState(handshake("1.1.0", "1.0"))).toEqual({
|
||||
kind: "version_skew",
|
||||
});
|
||||
});
|
||||
@@ -53,7 +53,7 @@ describe("computeConnectedState (protocol-based compat)", () => {
|
||||
});
|
||||
|
||||
it("rejects when extension is below daemon min_compatible_protocol", () => {
|
||||
const result = computeConnectedState(handshake("1.0", "1.5"));
|
||||
const result = computeConnectedState(handshake("1.1", "1.5"));
|
||||
expect(result.kind).toBe("rejected");
|
||||
if (result.kind === "rejected") {
|
||||
expect(result.reason).toContain("min_compatible_protocol");
|
||||
@@ -65,7 +65,7 @@ describe("computeConnectedState (protocol-based compat)", () => {
|
||||
const result = computeConnectedState({
|
||||
server: "browser-skill-daemon",
|
||||
version: "0.1.0",
|
||||
protocol_version: "1.0",
|
||||
protocol_version: "1.1",
|
||||
min_compatible_peer: "0.1.0",
|
||||
});
|
||||
expect(result).toEqual({ kind: "connected" });
|
||||
@@ -80,7 +80,7 @@ describe("computeConnectedState (protocol-based compat)", () => {
|
||||
});
|
||||
|
||||
it("rejects malformed daemon min_compatible_protocol with a daemon-floor reason", () => {
|
||||
const result = computeConnectedState(handshake("1.0", "not-a-protocol"));
|
||||
const result = computeConnectedState(handshake("1.1", "not-a-protocol"));
|
||||
expect(result.kind).toBe("rejected");
|
||||
if (result.kind === "rejected") {
|
||||
expect(result.reason).toContain("daemon min_compatible_protocol");
|
||||
@@ -242,11 +242,11 @@ describe("ConnectionController connectionEnabled", () => {
|
||||
const second = transport.send.mock.calls[1]?.[0] as { id: string };
|
||||
expect(second.id).not.toBe(first.id);
|
||||
|
||||
transport.emitMessage({ id: first.id, result: handshake("1.0", "1.0") });
|
||||
transport.emitMessage({ id: first.id, result: handshake("1.1", "1.0") });
|
||||
await Promise.resolve();
|
||||
expect(controller.snapshot().state).not.toBe("connected");
|
||||
|
||||
transport.emitMessage({ id: second.id, result: handshake("1.0", "1.0") });
|
||||
transport.emitMessage({ id: second.id, result: handshake("1.1", "1.0") });
|
||||
await vi.waitFor(() => expect(controller.snapshot().state).toBe("connected"));
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,101 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import type { CdpRunner } from "@/tools/shared";
|
||||
import { waitForDocumentSettled } from "../recording/document-settle";
|
||||
|
||||
function quietCdp(): { cdp: CdpRunner; send: ReturnType<typeof vi.fn> } {
|
||||
const send = vi.fn(async () => ({
|
||||
result: { value: { idleMs: 1_000, readyState: "complete" } },
|
||||
}));
|
||||
return {
|
||||
cdp: { send: send as unknown as CdpRunner["send"] },
|
||||
send,
|
||||
};
|
||||
}
|
||||
|
||||
describe("waitForDocumentSettled", () => {
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it("waits through the minimum observation floor before accepting a quiet page", async () => {
|
||||
vi.useFakeTimers();
|
||||
const { cdp, send } = quietCdp();
|
||||
|
||||
const settled = waitForDocumentSettled(cdp, { target: { tabId: 7 } });
|
||||
await vi.advanceTimersByTimeAsync(180);
|
||||
|
||||
await expect(settled).resolves.toBe("quiet");
|
||||
expect(send).toHaveBeenCalledTimes(3);
|
||||
});
|
||||
|
||||
it("cancels before probing a superseded observation", async () => {
|
||||
vi.useFakeTimers();
|
||||
const { cdp, send } = quietCdp();
|
||||
|
||||
const controller = new AbortController();
|
||||
const settled = waitForDocumentSettled(
|
||||
cdp,
|
||||
{ target: { tabId: 7 } },
|
||||
{
|
||||
signal: controller.signal,
|
||||
},
|
||||
);
|
||||
controller.abort();
|
||||
await vi.advanceTimersByTimeAsync(60);
|
||||
|
||||
await expect(settled).resolves.toBe("cancelled");
|
||||
expect(send).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("probes a same-process iframe through its isolated execution context", async () => {
|
||||
vi.useFakeTimers();
|
||||
const send = vi.fn(async (_tabId: number, method: string) => {
|
||||
if (method === "Page.createIsolatedWorld") return { executionContextId: 91 };
|
||||
return { result: { value: { idleMs: 1_000, readyState: "complete" } } };
|
||||
});
|
||||
const cdp: CdpRunner = { send: send as unknown as CdpRunner["send"] };
|
||||
|
||||
const settled = waitForDocumentSettled(cdp, {
|
||||
target: { tabId: 7 },
|
||||
frameId: "child-frame",
|
||||
});
|
||||
await vi.advanceTimersByTimeAsync(180);
|
||||
|
||||
await expect(settled).resolves.toBe("quiet");
|
||||
expect(send).toHaveBeenCalledWith(
|
||||
7,
|
||||
"Page.createIsolatedWorld",
|
||||
expect.objectContaining({ frameId: "child-frame" }),
|
||||
);
|
||||
expect(send).toHaveBeenCalledWith(
|
||||
7,
|
||||
"Runtime.evaluate",
|
||||
expect.objectContaining({ contextId: 91 }),
|
||||
);
|
||||
});
|
||||
|
||||
it("routes an OOPIF probe through its CDP target session", async () => {
|
||||
vi.useFakeTimers();
|
||||
const send = vi.fn();
|
||||
const sendToTarget = vi.fn(async (_target, method: string) => {
|
||||
if (method === "Page.createIsolatedWorld") return { executionContextId: 27 };
|
||||
return { result: { value: { idleMs: 1_000, readyState: "complete" } } };
|
||||
});
|
||||
const cdp: CdpRunner = {
|
||||
send: send as unknown as CdpRunner["send"],
|
||||
sendToTarget: sendToTarget as unknown as NonNullable<CdpRunner["sendToTarget"]>,
|
||||
};
|
||||
|
||||
const target = { tabId: 7, sessionId: "oopif-session" };
|
||||
const settled = waitForDocumentSettled(cdp, { target, frameId: "oopif-frame" });
|
||||
await vi.advanceTimersByTimeAsync(180);
|
||||
|
||||
await expect(settled).resolves.toBe("quiet");
|
||||
expect(send).not.toHaveBeenCalled();
|
||||
expect(sendToTarget).toHaveBeenCalledWith(
|
||||
target,
|
||||
"Runtime.evaluate",
|
||||
expect.objectContaining({ contextId: 27 }),
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -32,6 +32,7 @@ describe("help-bridge", () => {
|
||||
prompt: "log in",
|
||||
displayMode: "compact",
|
||||
selectors: ["#login"],
|
||||
rects: [{ top: 10, left: 20, width: 100, height: 40 }],
|
||||
timeoutMs: 1000,
|
||||
}),
|
||||
).toBe(true);
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
isAcceptedRecordStepAck,
|
||||
isRecordStepMessage,
|
||||
RECORD_STEP,
|
||||
type RecordStepMessage,
|
||||
} from "../record-bridge";
|
||||
|
||||
const message: RecordStepMessage = {
|
||||
type: RECORD_STEP,
|
||||
requestId: "rec-1",
|
||||
producerId: "document-1",
|
||||
sequence: 1,
|
||||
step: { op: "click", target: { tag: "button", name: "Submit" } },
|
||||
};
|
||||
|
||||
describe("record step delivery protocol", () => {
|
||||
it("requires a producer and a positive integer sequence", () => {
|
||||
expect(isRecordStepMessage(message)).toBe(true);
|
||||
expect(isRecordStepMessage({ ...message, producerId: "" })).toBe(false);
|
||||
expect(isRecordStepMessage({ ...message, sequence: 0 })).toBe(false);
|
||||
expect(isRecordStepMessage({ ...message, sequence: 1.5 })).toBe(false);
|
||||
});
|
||||
|
||||
it("accepts only the acknowledgement for the sent sequence", () => {
|
||||
expect(isAcceptedRecordStepAck({ ok: true, sequence: 2 }, 2)).toBe(true);
|
||||
expect(isAcceptedRecordStepAck({ ok: true, sequence: 1 }, 2)).toBe(false);
|
||||
expect(isAcceptedRecordStepAck({ ok: false, expectedSequence: 2, error: "gap" }, 2)).toBe(
|
||||
false,
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,213 @@
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { RECORD_FRAME_PORT, RECORD_FRAME_QUERY } from "../recording/frame-bridge";
|
||||
import { RecordFrameCoordinator } from "../recording/frame-coordinator";
|
||||
|
||||
class ListenerSet<T extends (...args: never[]) => unknown> {
|
||||
readonly listeners = new Set<T>();
|
||||
addListener = (listener: T) => this.listeners.add(listener);
|
||||
removeListener = (listener: T) => this.listeners.delete(listener);
|
||||
}
|
||||
|
||||
function fakePort(sender: chrome.runtime.MessageSender) {
|
||||
const messages: unknown[] = [];
|
||||
const onMessage = new ListenerSet<(message: unknown) => void>();
|
||||
const onDisconnect = new ListenerSet<() => void>();
|
||||
return {
|
||||
port: {
|
||||
name: RECORD_FRAME_PORT,
|
||||
sender,
|
||||
onMessage,
|
||||
onDisconnect,
|
||||
postMessage: vi.fn((message: unknown) => messages.push(message)),
|
||||
disconnect: vi.fn(),
|
||||
} as unknown as chrome.runtime.Port,
|
||||
messages,
|
||||
receive(message: unknown) {
|
||||
for (const listener of onMessage.listeners) listener(message);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe("RecordFrameCoordinator", () => {
|
||||
const onConnect = new ListenerSet<(port: chrome.runtime.Port) => void>();
|
||||
const onMessage = new ListenerSet<
|
||||
(
|
||||
message: unknown,
|
||||
sender: chrome.runtime.MessageSender,
|
||||
sendResponse: (response: unknown) => void,
|
||||
) => boolean
|
||||
>();
|
||||
|
||||
beforeEach(() => {
|
||||
onConnect.listeners.clear();
|
||||
onMessage.listeners.clear();
|
||||
vi.stubGlobal("chrome", {
|
||||
runtime: { onConnect, onMessage },
|
||||
});
|
||||
});
|
||||
|
||||
it("answers child-document queries for the initial tab before it is armed", () => {
|
||||
const coordinator = new RecordFrameCoordinator({
|
||||
getAllFrames: async () => [],
|
||||
sendToDocument: async () => ({ ok: true }),
|
||||
});
|
||||
coordinator.attach();
|
||||
coordinator.begin("rec-1", 10, 3);
|
||||
|
||||
const query = (tabId: number) => {
|
||||
let response: unknown;
|
||||
for (const listener of onMessage.listeners) {
|
||||
listener(
|
||||
{ type: RECORD_FRAME_QUERY },
|
||||
{
|
||||
tab: { id: tabId },
|
||||
frameId: 7,
|
||||
documentId: "child-document",
|
||||
} as chrome.runtime.MessageSender,
|
||||
(value) => {
|
||||
response = value;
|
||||
},
|
||||
);
|
||||
}
|
||||
return response;
|
||||
};
|
||||
|
||||
expect(query(3)).toEqual({ active: true, requestId: "rec-1", startedAtMs: 10 });
|
||||
expect(query(4)).toEqual({ active: false });
|
||||
});
|
||||
|
||||
it("starts a child Document that appears after the initial frame snapshot", async () => {
|
||||
let notifyFrameNavigation:
|
||||
| ((frame: {
|
||||
tabId: number;
|
||||
frameId: number;
|
||||
documentId?: string;
|
||||
lifecycle: "committed" | "completed";
|
||||
}) => void)
|
||||
| undefined;
|
||||
const sendToDocument = vi.fn(async () => ({ ok: true }));
|
||||
const coordinator = new RecordFrameCoordinator({
|
||||
getAllFrames: async () => [{ frameId: 0, documentId: "top-document" }],
|
||||
sendToDocument,
|
||||
subscribeFrameNavigation(listener) {
|
||||
notifyFrameNavigation = listener;
|
||||
return () => {};
|
||||
},
|
||||
});
|
||||
coordinator.attach();
|
||||
const onDocumentReady = vi.fn();
|
||||
coordinator.begin("rec-1", 10, 3, onDocumentReady);
|
||||
await coordinator.armTab("rec-1", 3);
|
||||
sendToDocument.mockClear();
|
||||
|
||||
notifyFrameNavigation?.({
|
||||
tabId: 3,
|
||||
frameId: 7,
|
||||
documentId: "late-child",
|
||||
lifecycle: "completed",
|
||||
});
|
||||
await vi.waitFor(() => {
|
||||
expect(sendToDocument).toHaveBeenCalledWith(
|
||||
3,
|
||||
{ type: "bsk-record-frame-start", requestId: "rec-1", startedAtMs: 10 },
|
||||
{ documentId: "late-child" },
|
||||
);
|
||||
});
|
||||
expect(onDocumentReady).not.toHaveBeenCalled();
|
||||
|
||||
const child = fakePort({
|
||||
tab: { id: 3 },
|
||||
frameId: 7,
|
||||
documentId: "late-child",
|
||||
} as chrome.runtime.MessageSender);
|
||||
for (const listener of onConnect.listeners) listener(child.port);
|
||||
child.receive({ type: "ready", requestId: "rec-1", producerId: "producer-1" });
|
||||
notifyFrameNavigation?.({
|
||||
tabId: 3,
|
||||
frameId: 7,
|
||||
documentId: "late-child",
|
||||
lifecycle: "completed",
|
||||
});
|
||||
await vi.waitFor(() => {
|
||||
expect(onDocumentReady).toHaveBeenCalledWith({
|
||||
tabId: 3,
|
||||
documentId: "late-child",
|
||||
browserFrameId: 7,
|
||||
producerId: "producer-1",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it("binds a producer to its sender Document and keeps final steps valid while stopping", async () => {
|
||||
const sendToDocument = vi.fn(async () => ({ ok: true }));
|
||||
const coordinator = new RecordFrameCoordinator({
|
||||
getAllFrames: async () => [
|
||||
{ frameId: 0, documentId: "top-document" },
|
||||
{ frameId: 7, documentId: "child-document" },
|
||||
],
|
||||
sendToDocument,
|
||||
});
|
||||
coordinator.attach();
|
||||
coordinator.begin("rec-1", 10, 3);
|
||||
await expect(coordinator.armTab("rec-1", 3)).resolves.toBe(true);
|
||||
|
||||
const sender = {
|
||||
tab: { id: 3 },
|
||||
frameId: 7,
|
||||
documentId: "child-document",
|
||||
} as chrome.runtime.MessageSender;
|
||||
const frame = fakePort(sender);
|
||||
for (const listener of onConnect.listeners) listener(frame.port);
|
||||
frame.receive({ type: "ready", requestId: "rec-1", producerId: "producer-1" });
|
||||
|
||||
expect(coordinator.sourceFor("rec-1", "producer-1", sender)).toEqual({
|
||||
tabId: 3,
|
||||
documentId: "child-document",
|
||||
browserFrameId: 7,
|
||||
producerId: "producer-1",
|
||||
});
|
||||
|
||||
const stopping = coordinator.stop("rec-1");
|
||||
expect(coordinator.sourceFor("rec-1", "producer-1", sender)).not.toBeNull();
|
||||
const stop = frame.messages.find(
|
||||
(message): message is { type: "stop"; commandId: string } =>
|
||||
typeof message === "object" &&
|
||||
message !== null &&
|
||||
(message as { type?: string }).type === "stop",
|
||||
);
|
||||
expect(stop).toBeDefined();
|
||||
frame.receive({
|
||||
type: "stopped",
|
||||
requestId: "rec-1",
|
||||
commandId: stop!.commandId,
|
||||
ok: true,
|
||||
});
|
||||
await expect(stopping).resolves.toBe(true);
|
||||
expect(coordinator.sourceFor("rec-1", "producer-1", sender)).toBeNull();
|
||||
});
|
||||
|
||||
it("rejects a producer used from another Document", async () => {
|
||||
const coordinator = new RecordFrameCoordinator({
|
||||
getAllFrames: async () => [{ frameId: 0, documentId: "top-document" }],
|
||||
sendToDocument: async () => ({ ok: true }),
|
||||
});
|
||||
coordinator.attach();
|
||||
coordinator.begin("rec-1", 10, 3);
|
||||
await coordinator.armTab("rec-1", 3);
|
||||
const sender = {
|
||||
tab: { id: 3 },
|
||||
frameId: 0,
|
||||
documentId: "top-document",
|
||||
} as chrome.runtime.MessageSender;
|
||||
const frame = fakePort(sender);
|
||||
for (const listener of onConnect.listeners) listener(frame.port);
|
||||
frame.receive({ type: "ready", requestId: "rec-1", producerId: "producer-1" });
|
||||
|
||||
expect(
|
||||
coordinator.sourceFor("rec-1", "producer-1", {
|
||||
...sender,
|
||||
documentId: "other-document",
|
||||
}),
|
||||
).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,70 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
RECORD_DOCUMENT_ATTRIBUTE,
|
||||
recordingDocumentMarkerValue,
|
||||
} from "@/shared/recording-document-identity";
|
||||
import { markRecordingDocument } from "../recording/document-marker";
|
||||
import { ObservationNodeIndex } from "../recording/observation-capture";
|
||||
|
||||
describe("recording document marker", () => {
|
||||
it("rejects a non-random Document identity", () => {
|
||||
expect(() => markRecordingDocument("user@example.com")).toThrow(
|
||||
"recording Document identity must be a random UUID",
|
||||
);
|
||||
});
|
||||
|
||||
it("restores the Document attribute exactly", () => {
|
||||
document.documentElement.setAttribute(RECORD_DOCUMENT_ATTRIBUTE, "page-value");
|
||||
const producerId = "123e4567-e89b-42d3-a456-426614174000";
|
||||
const marker = markRecordingDocument(producerId);
|
||||
|
||||
expect(document.documentElement.getAttribute(RECORD_DOCUMENT_ATTRIBUTE)).toBe(
|
||||
recordingDocumentMarkerValue(producerId),
|
||||
);
|
||||
marker.restore();
|
||||
expect(document.documentElement.getAttribute(RECORD_DOCUMENT_ATTRIBUTE)).toBe("page-value");
|
||||
document.documentElement.removeAttribute(RECORD_DOCUMENT_ATTRIBUTE);
|
||||
});
|
||||
|
||||
it("indexes the CDP scope from the safe frame identity", () => {
|
||||
const index = new ObservationNodeIndex({
|
||||
rootFrameId: "root",
|
||||
frames: [
|
||||
{
|
||||
frameId: "child",
|
||||
target: { tabId: 3, sessionId: "oopif-session" },
|
||||
recordingDocumentId: "producer-1",
|
||||
},
|
||||
],
|
||||
matchNodes: [],
|
||||
refs: [],
|
||||
});
|
||||
|
||||
expect(index.documentScope("producer-1")).toEqual({
|
||||
frameId: "child",
|
||||
target: { tabId: 3, sessionId: "oopif-session" },
|
||||
});
|
||||
});
|
||||
|
||||
it("fails closed when a recording identity maps to multiple Documents", () => {
|
||||
const index = new ObservationNodeIndex({
|
||||
rootFrameId: "root",
|
||||
frames: [
|
||||
{
|
||||
frameId: "left",
|
||||
target: { tabId: 3, sessionId: "left-session" },
|
||||
recordingDocumentId: "duplicate",
|
||||
},
|
||||
{
|
||||
frameId: "right",
|
||||
target: { tabId: 3, sessionId: "right-session" },
|
||||
recordingDocumentId: "duplicate",
|
||||
},
|
||||
],
|
||||
matchNodes: [],
|
||||
refs: [],
|
||||
});
|
||||
|
||||
expect(index.documentScope("duplicate")).toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,146 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { ObservationNodeIndex } from "../recording/observation-capture";
|
||||
import { RecordingObservationSession } from "../recording/observation-session";
|
||||
import { RecordingStateRegistry } from "../recording/state-registry";
|
||||
import { buildTraceV3 } from "../recording/trace-builder-v3";
|
||||
import type { RecordingDraftStep } from "../recording/types";
|
||||
|
||||
const URL = "https://example.com/login";
|
||||
|
||||
function sessionWithInput(redactValues = false): RecordingObservationSession {
|
||||
const session = new RecordingObservationSession({ redactValues });
|
||||
const state = session.registry.register({
|
||||
url: URL,
|
||||
vomText: '@vom 1\ntextbox "Password" value="••••••" [ref=e1]',
|
||||
});
|
||||
session.cursor.lastSettled = {
|
||||
stateId: state.id,
|
||||
rootFrameId: "root",
|
||||
index: new ObservationNodeIndex({
|
||||
rootFrameId: "root",
|
||||
matchNodes: [
|
||||
{
|
||||
frameId: "root",
|
||||
backendNodeId: 42,
|
||||
tag: "input",
|
||||
rect: { x: 20, y: 40, w: 200, h: 30 },
|
||||
localRect: { x: 20, y: 40, w: 200, h: 30 },
|
||||
},
|
||||
],
|
||||
refs: [{ ref: "e1", backendNodeId: 42, role: "textbox", name: "Password", line: 1 }],
|
||||
}),
|
||||
url: URL,
|
||||
};
|
||||
return session;
|
||||
}
|
||||
|
||||
function finalizedFillTrace(value: string, redactValues: boolean) {
|
||||
const session = sessionWithInput(redactValues);
|
||||
const draft: RecordingDraftStep = {
|
||||
op: "fill",
|
||||
captureTarget: { tag: "input", role: "textbox", name: "Password" },
|
||||
value,
|
||||
targetHint: {
|
||||
geometry: { rect: { x: 20, y: 40, w: 200, h: 30 }, tag: "input" },
|
||||
},
|
||||
};
|
||||
session.bindDraft(draft, 1);
|
||||
draft.postStateId = draft.preStateId;
|
||||
return buildTraceV3({
|
||||
registry: session.registry,
|
||||
drafts: [draft],
|
||||
annotations: session.annotations,
|
||||
startedAt: "2026-08-12T00:00:00.000Z",
|
||||
stoppedBy: "user_finish",
|
||||
bskVersion: "test",
|
||||
redactValues,
|
||||
});
|
||||
}
|
||||
|
||||
function finalizedFillBody(value: string, redactValues: boolean): string {
|
||||
return finalizedFillTrace(value, redactValues).states[0]!.body;
|
||||
}
|
||||
|
||||
describe("record observation annotations", () => {
|
||||
it("omits a fill literal when values are redacted", () => {
|
||||
const secret = "hunter2-private";
|
||||
const dumped = JSON.stringify(finalizedFillTrace(secret, true));
|
||||
expect(dumped).toContain("step 1: fill");
|
||||
expect(dumped).not.toContain(secret);
|
||||
});
|
||||
|
||||
it("keeps ordinary fill details", () => {
|
||||
expect(finalizedFillBody("ordinary text", false)).toContain('step 1: fill: "ordinary text"');
|
||||
});
|
||||
|
||||
it("encodes title and URL so line breaks cannot corrupt state metadata", () => {
|
||||
const registry = new RecordingStateRegistry();
|
||||
const state = registry.register({
|
||||
url: "https://example.com/a\nb",
|
||||
title: "hello\nworld",
|
||||
vomText: "@vom 1",
|
||||
});
|
||||
const trace = buildTraceV3({
|
||||
registry,
|
||||
drafts: [{ op: "scroll", preStateId: state.id, postStateId: state.id }],
|
||||
startedAt: "2026-08-12T00:00:00.000Z",
|
||||
stoppedBy: "user_finish",
|
||||
bskVersion: "test",
|
||||
});
|
||||
expect(trace.states[0]?.body).toContain('url: "https://example.com/a\\nb"');
|
||||
expect(trace.states[0]?.body).toContain('title: "hello\\nworld"');
|
||||
});
|
||||
});
|
||||
|
||||
describe("recording state ownership", () => {
|
||||
it("deduplicates within one recording and isolates ids between recordings", () => {
|
||||
const first = new RecordingStateRegistry();
|
||||
const second = new RecordingStateRegistry();
|
||||
expect(first.register({ url: URL, vomText: "same" }).id).toBe("s1");
|
||||
expect(first.register({ url: URL, vomText: "same" }).id).toBe("s1");
|
||||
expect(second.register({ url: URL, vomText: "other" }).id).toBe("s1");
|
||||
});
|
||||
|
||||
it("enriches metadata when a deduplicated observation becomes more complete", () => {
|
||||
const registry = new RecordingStateRegistry();
|
||||
registry.register({ url: URL, vomText: "same" });
|
||||
const state = registry.register({
|
||||
url: URL,
|
||||
title: "Login",
|
||||
vomText: "same",
|
||||
truncated: true,
|
||||
});
|
||||
expect(state).toMatchObject({ id: "s1", title: "Login", truncated: true });
|
||||
});
|
||||
});
|
||||
|
||||
describe("draft binding", () => {
|
||||
it("keeps capture semantics when no observation exists", () => {
|
||||
const session = new RecordingObservationSession();
|
||||
const draft: RecordingDraftStep = {
|
||||
op: "hover",
|
||||
captureTarget: { tag: "button", role: "button", name: "新建" },
|
||||
};
|
||||
session.bindDraft(draft, 1);
|
||||
expect(draft.matchedTarget).toEqual({ role: "button", name: "新建", unmatched: true });
|
||||
expect(draft.preStateId).toBeUndefined();
|
||||
});
|
||||
|
||||
it("does not bind an unmatched new action to a stale observation", () => {
|
||||
const session = sessionWithInput();
|
||||
const draft: RecordingDraftStep = {
|
||||
op: "click",
|
||||
captureTarget: { tag: "button", role: "button", name: "Confirm" },
|
||||
targetHint: {
|
||||
geometry: { rect: { x: 400, y: 300, w: 80, h: 30 }, tag: "button" },
|
||||
},
|
||||
};
|
||||
session.bindDraft(draft, 2, true);
|
||||
expect(draft.preStateId).toBeUndefined();
|
||||
expect("matchedTarget" in draft ? draft.matchedTarget : undefined).toEqual({
|
||||
role: "button",
|
||||
name: "Confirm",
|
||||
unmatched: true,
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,289 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
|
||||
|
||||
const captureRecordingObservation = vi.hoisted(() => vi.fn());
|
||||
const waitForDocumentSettled = vi.hoisted(() => vi.fn(async () => "quiet" as const));
|
||||
|
||||
vi.mock("../recording/observation-capture", async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import("../recording/observation-capture")>();
|
||||
return { ...actual, captureRecordingObservation };
|
||||
});
|
||||
|
||||
vi.mock("../recording/document-settle", async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import("../recording/document-settle")>();
|
||||
return { ...actual, waitForDocumentSettled };
|
||||
});
|
||||
|
||||
import { ObservationNodeIndex } from "../recording/observation-capture";
|
||||
import { RecordingObservationRuntime } from "../recording/recording-runtime";
|
||||
import type { RecordingDraftStep } from "../recording/types";
|
||||
|
||||
function observation(url = "https://example.com/") {
|
||||
return {
|
||||
rootFrameId: "root",
|
||||
index: new ObservationNodeIndex({ rootFrameId: "root", matchNodes: [], refs: [] }),
|
||||
url,
|
||||
title: "Example",
|
||||
vomText: '@vom 1\nRootWebArea "Example"',
|
||||
truncated: false,
|
||||
};
|
||||
}
|
||||
|
||||
function runtime(): RecordingObservationRuntime {
|
||||
return new RecordingObservationRuntime({
|
||||
cdp: { send: vi.fn() as unknown as CdpRunner["send"] },
|
||||
tabsApi: { get: vi.fn(), query: vi.fn() } as unknown as ChromeTabsApi,
|
||||
});
|
||||
}
|
||||
|
||||
describe("RecordingObservationRuntime", () => {
|
||||
afterEach(() => {
|
||||
captureRecordingObservation.mockReset();
|
||||
waitForDocumentSettled.mockClear();
|
||||
});
|
||||
|
||||
it("shares one initial capture and includes it in flush", async () => {
|
||||
let release!: (value: ReturnType<typeof observation>) => void;
|
||||
captureRecordingObservation.mockReturnValueOnce(
|
||||
new Promise((resolve) => {
|
||||
release = resolve;
|
||||
}),
|
||||
);
|
||||
const recording = runtime();
|
||||
|
||||
const first = recording.captureInitial(7);
|
||||
const second = recording.captureInitial(7);
|
||||
const flushed = recording.flush();
|
||||
let flushCompleted = false;
|
||||
void flushed.then(() => {
|
||||
flushCompleted = true;
|
||||
});
|
||||
|
||||
await Promise.resolve();
|
||||
expect(captureRecordingObservation).toHaveBeenCalledTimes(1);
|
||||
expect(flushCompleted).toBe(false);
|
||||
|
||||
release(observation());
|
||||
await Promise.all([first, second, flushed]);
|
||||
expect(flushCompleted).toBe(true);
|
||||
});
|
||||
|
||||
it("cancels an in-flight initial capture", async () => {
|
||||
captureRecordingObservation.mockImplementationOnce(
|
||||
(input: { signal?: AbortSignal }) =>
|
||||
new Promise((_, reject) => {
|
||||
input.signal?.addEventListener(
|
||||
"abort",
|
||||
() => reject(new DOMException("observation aborted", "AbortError")),
|
||||
{ once: true },
|
||||
);
|
||||
}),
|
||||
);
|
||||
const recording = runtime();
|
||||
|
||||
const pending = recording.captureInitial(7);
|
||||
recording.cancel();
|
||||
|
||||
await expect(pending).rejects.toMatchObject({ name: "AbortError" });
|
||||
});
|
||||
|
||||
it("retries a failed initial capture when the recording settles at stop", async () => {
|
||||
captureRecordingObservation
|
||||
.mockRejectedValueOnce(new Error("document swapped"))
|
||||
.mockResolvedValueOnce(observation());
|
||||
const recording = runtime();
|
||||
|
||||
await expect(recording.captureInitial(7)).rejects.toThrow("document swapped");
|
||||
await recording.settleTrailing(7, []);
|
||||
const trace = recording.buildTrace({
|
||||
drafts: [],
|
||||
startedAt: "2026-08-19T00:00:00.000Z",
|
||||
stoppedBy: "user_finish",
|
||||
bskVersion: "0.1.6",
|
||||
});
|
||||
|
||||
expect(captureRecordingObservation).toHaveBeenCalledTimes(2);
|
||||
expect(trace.states).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("keeps runtime matching data out of the serialized trace", async () => {
|
||||
const captured = observation();
|
||||
captured.index = new ObservationNodeIndex({
|
||||
rootFrameId: "root",
|
||||
matchNodes: [
|
||||
{
|
||||
frameId: "root",
|
||||
backendNodeId: 42,
|
||||
tag: "button",
|
||||
rect: { x: 10, y: 20, w: 100, h: 30 },
|
||||
localRect: { x: 10, y: 20, w: 100, h: 30 },
|
||||
},
|
||||
],
|
||||
refs: [{ ref: "e1", backendNodeId: 42, role: "button", name: "Submit", line: 1 }],
|
||||
});
|
||||
captureRecordingObservation.mockResolvedValueOnce(captured);
|
||||
const recording = runtime();
|
||||
|
||||
await recording.captureInitial(7);
|
||||
const dumped = JSON.stringify(
|
||||
recording.buildTrace({
|
||||
drafts: [],
|
||||
startedAt: "2026-08-19T00:00:00.000Z",
|
||||
stoppedBy: "user_finish",
|
||||
bskVersion: "0.1.6",
|
||||
}),
|
||||
);
|
||||
|
||||
expect(dumped).toContain("RootWebArea");
|
||||
expect(dumped).not.toContain("matchNodes");
|
||||
expect(dumped).not.toContain("localRect");
|
||||
expect(dumped).not.toContain("backendNodeId");
|
||||
expect(dumped).not.toContain("ObservationNodeIndex");
|
||||
});
|
||||
|
||||
it("captures a tab transition without sharing observation cursors", async () => {
|
||||
captureRecordingObservation
|
||||
.mockResolvedValueOnce(observation("https://example.com/first"))
|
||||
.mockResolvedValueOnce(observation("https://example.com/second"));
|
||||
const recording = runtime();
|
||||
|
||||
await recording.captureInitial(4);
|
||||
const transition = await recording.captureTabTransition(4, 5);
|
||||
await recording.captureInitial(4);
|
||||
await recording.captureInitial(5);
|
||||
|
||||
expect(transition).toEqual({
|
||||
preStateId: "s1",
|
||||
postStateId: "s2",
|
||||
targetUrl: "https://example.com/second",
|
||||
});
|
||||
expect(captureRecordingObservation.mock.calls.map(([input]) => input.tabId)).toEqual([4, 5]);
|
||||
});
|
||||
|
||||
it("binds an iframe draft to its marked CDP Document scope", async () => {
|
||||
captureRecordingObservation.mockResolvedValueOnce({
|
||||
...observation(),
|
||||
index: new ObservationNodeIndex({
|
||||
rootFrameId: "root",
|
||||
frames: [
|
||||
{
|
||||
frameId: "child",
|
||||
target: { tabId: 7, sessionId: "oopif-session" },
|
||||
recordingDocumentId: "producer-1",
|
||||
},
|
||||
],
|
||||
matchNodes: [
|
||||
{
|
||||
backendNodeId: 42,
|
||||
frameId: "child",
|
||||
tag: "button",
|
||||
rect: { x: 410, y: 20, w: 100, h: 30 },
|
||||
localRect: { x: 10, y: 20, w: 100, h: 30 },
|
||||
},
|
||||
],
|
||||
refs: [{ ref: "e1", backendNodeId: 42, frameId: "child", line: 1 }],
|
||||
}),
|
||||
});
|
||||
const recording = runtime();
|
||||
const drafts: RecordingDraftStep[] = [
|
||||
{
|
||||
op: "click" as const,
|
||||
captureTarget: { tag: "button", role: "button", name: "Save" },
|
||||
targetHint: {
|
||||
geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" },
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
await recording.captureInitial(7);
|
||||
await recording.processDraft(7, drafts, 0, "producer-1");
|
||||
recording.cancel();
|
||||
|
||||
const draft = drafts[0];
|
||||
expect(draft?.op).toBe("click");
|
||||
if (!draft || draft.op !== "click") throw new Error("expected click draft");
|
||||
expect(draft.targetHint).toMatchObject({
|
||||
frameId: "child",
|
||||
geometrySpace: "local",
|
||||
});
|
||||
expect(draft.matchedTarget?.ref).toBe("e1");
|
||||
});
|
||||
|
||||
it("refreshes a late iframe before matching its first action", async () => {
|
||||
const iframeObservation = {
|
||||
...observation(),
|
||||
index: new ObservationNodeIndex({
|
||||
rootFrameId: "root",
|
||||
frames: [
|
||||
{
|
||||
frameId: "child",
|
||||
target: { tabId: 7, sessionId: "oopif-session" },
|
||||
recordingDocumentId: "producer-1",
|
||||
},
|
||||
],
|
||||
matchNodes: [
|
||||
{
|
||||
backendNodeId: 42,
|
||||
frameId: "child",
|
||||
tag: "button",
|
||||
rect: { x: 410, y: 20, w: 100, h: 30 },
|
||||
localRect: { x: 10, y: 20, w: 100, h: 30 },
|
||||
},
|
||||
],
|
||||
refs: [
|
||||
{
|
||||
ref: "e1",
|
||||
backendNodeId: 42,
|
||||
frameId: "child",
|
||||
role: "button",
|
||||
name: "表格视图",
|
||||
line: 1,
|
||||
},
|
||||
],
|
||||
}),
|
||||
};
|
||||
captureRecordingObservation
|
||||
.mockResolvedValueOnce(observation())
|
||||
.mockResolvedValueOnce(iframeObservation)
|
||||
.mockResolvedValueOnce(iframeObservation);
|
||||
const recording = runtime();
|
||||
|
||||
await recording.captureInitial(7);
|
||||
await recording.refreshDocument(7, "producer-1");
|
||||
const drafts: RecordingDraftStep[] = [
|
||||
{
|
||||
op: "click",
|
||||
captureTarget: { tag: "button", role: "button", name: "表格视图" },
|
||||
targetHint: {
|
||||
geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" },
|
||||
},
|
||||
},
|
||||
];
|
||||
await recording.processDraft(7, drafts, 0, "producer-1");
|
||||
recording.cancel();
|
||||
|
||||
expect(captureRecordingObservation).toHaveBeenCalledTimes(3);
|
||||
expect(waitForDocumentSettled).toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
{ frameId: "child", target: { tabId: 7, sessionId: "oopif-session" } },
|
||||
{ signal: expect.any(AbortSignal) },
|
||||
);
|
||||
const draft = drafts[0];
|
||||
expect(draft?.op).toBe("click");
|
||||
if (!draft || draft.op !== "click") throw new Error("expected click draft");
|
||||
expect(draft.matchedTarget).toMatchObject({ ref: "e1", name: "表格视图" });
|
||||
});
|
||||
|
||||
it("does not poison final flush when a frame readiness refresh fails", async () => {
|
||||
captureRecordingObservation
|
||||
.mockResolvedValueOnce(observation())
|
||||
.mockRejectedValueOnce(new Error("child Document replaced"));
|
||||
const recording = runtime();
|
||||
|
||||
await recording.captureInitial(7);
|
||||
await expect(recording.refreshDocument(7, "producer-1")).rejects.toThrow(
|
||||
"child Document replaced",
|
||||
);
|
||||
await expect(recording.flush()).resolves.toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -1,80 +0,0 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { appendRecordedPayload, observeRecordedNavigation } from "../recording-step-buffer";
|
||||
|
||||
describe("recording-step-buffer", () => {
|
||||
it("stores semantic click without summary", () => {
|
||||
const buffer = { steps: [], pendingNavigation: false };
|
||||
appendRecordedPayload(buffer, {
|
||||
op: "click",
|
||||
target: { tag: "button", role: "button", name: "发布" },
|
||||
expects_navigation: true,
|
||||
});
|
||||
expect(buffer.steps).toEqual([
|
||||
{
|
||||
op: "click",
|
||||
target: { tag: "button", role: "button", name: "发布" },
|
||||
},
|
||||
]);
|
||||
expect(buffer.pendingNavigation).toBe(true);
|
||||
});
|
||||
|
||||
it("annotates navigated_to on action-caused navigation instead of wait_for_navigation", () => {
|
||||
const buffer = {
|
||||
steps: [
|
||||
{
|
||||
op: "click" as const,
|
||||
target: { tag: "button", role: "button", name: "发布" },
|
||||
},
|
||||
],
|
||||
currentUrl: "https://example.com/a",
|
||||
pendingNavigation: true,
|
||||
pendingNavigationDeadline: Date.now() + 5_000,
|
||||
};
|
||||
observeRecordedNavigation(buffer, "https://example.com/b", true);
|
||||
expect(buffer.steps).toEqual([
|
||||
{
|
||||
op: "click",
|
||||
target: { tag: "button", role: "button", name: "发布" },
|
||||
navigated_to: "https://example.com/b",
|
||||
},
|
||||
]);
|
||||
expect(JSON.stringify(buffer.steps)).not.toContain("wait_for_navigation");
|
||||
});
|
||||
|
||||
it("annotates navigated_to onto a select that auto-submits a navigation", () => {
|
||||
const buffer = {
|
||||
steps: [
|
||||
{
|
||||
op: "select" as const,
|
||||
target: { tag: "select", role: "combobox", name: "分类" },
|
||||
values: ["tech"],
|
||||
},
|
||||
],
|
||||
currentUrl: "https://example.com/list",
|
||||
pendingNavigation: true,
|
||||
pendingNavigationDeadline: Date.now() + 5_000,
|
||||
};
|
||||
observeRecordedNavigation(buffer, "https://example.com/list?cat=tech", true);
|
||||
expect(buffer.steps).toEqual([
|
||||
{
|
||||
op: "select",
|
||||
target: { tag: "select", role: "combobox", name: "分类" },
|
||||
values: ["tech"],
|
||||
navigated_to: "https://example.com/list?cat=tech",
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("emits navigate for uncaused URL changes", () => {
|
||||
const buffer = {
|
||||
steps: [],
|
||||
currentUrl: "https://example.com/a",
|
||||
pendingNavigation: false,
|
||||
};
|
||||
observeRecordedNavigation(buffer, "https://example.com/b", false);
|
||||
expect(buffer.steps[0]).toMatchObject({
|
||||
op: "navigate",
|
||||
url: "https://example.com/b",
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,136 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
|
||||
import { ObservationNodeIndex, type RegisteredObservation } from "../recording/observation-capture";
|
||||
import { RecordingObservationSession } from "../recording/observation-session";
|
||||
import { SettleController } from "../recording/settle-controller";
|
||||
import type { RecordingDraftStep } from "../recording/types";
|
||||
|
||||
const OBSERVATION: RegisteredObservation = {
|
||||
stateId: "s-next",
|
||||
rootFrameId: "root",
|
||||
index: new ObservationNodeIndex({ rootFrameId: "root", matchNodes: [], refs: [] }),
|
||||
url: "https://example.com/next",
|
||||
};
|
||||
|
||||
describe("SettleController", () => {
|
||||
afterEach(() => vi.useRealTimers());
|
||||
|
||||
it("aborts superseded capture work and lands the prior action on the next origin", async () => {
|
||||
vi.useFakeTimers();
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async () => ({
|
||||
result: { value: { idleMs: 1_000, readyState: "complete" } },
|
||||
})) as unknown as CdpRunner["send"],
|
||||
};
|
||||
const tabsApi: ChromeTabsApi = {
|
||||
get: vi.fn(async () => ({ id: 7, url: OBSERVATION.url }) as chrome.tabs.Tab),
|
||||
query: vi.fn(async () => []),
|
||||
};
|
||||
const session = new RecordingObservationSession();
|
||||
let firstSignal: AbortSignal | undefined;
|
||||
vi.spyOn(session, "capture")
|
||||
.mockImplementationOnce(async (_cdp, _tabs, _tabId, signal) => {
|
||||
firstSignal = signal;
|
||||
return new Promise<RegisteredObservation>((_resolve, reject) => {
|
||||
signal?.addEventListener(
|
||||
"abort",
|
||||
() => reject(new DOMException("observation aborted", "AbortError")),
|
||||
{ once: true },
|
||||
);
|
||||
});
|
||||
})
|
||||
.mockResolvedValueOnce(OBSERVATION);
|
||||
|
||||
const drafts: RecordingDraftStep[] = [
|
||||
{ op: "click", captureTarget: { tag: "button", name: "First" }, preStateId: "s1" },
|
||||
];
|
||||
const controller = new SettleController({ session, cdp, tabsApi, tabId: 7 });
|
||||
controller.schedule(drafts, 0);
|
||||
await vi.advanceTimersByTimeAsync(180);
|
||||
|
||||
drafts.push({
|
||||
op: "click",
|
||||
captureTarget: { tag: "button", name: "Second" },
|
||||
preStateId: "s-next",
|
||||
});
|
||||
controller.schedule(drafts, 1);
|
||||
expect(firstSignal?.aborted).toBe(true);
|
||||
expect(drafts[0]?.postStateId).toBe("s-next");
|
||||
|
||||
await vi.advanceTimersByTimeAsync(500);
|
||||
await controller.flush();
|
||||
expect(drafts[1]?.postStateId).toBe("s-next");
|
||||
});
|
||||
|
||||
it("does not consume a newer redirect while reading the prior landing URL", async () => {
|
||||
vi.useFakeTimers();
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async () => ({
|
||||
result: { value: { idleMs: 1_000, readyState: "complete" } },
|
||||
})) as unknown as CdpRunner["send"],
|
||||
};
|
||||
let resolveFirstTab!: (tab: chrome.tabs.Tab) => void;
|
||||
const firstTab = new Promise<chrome.tabs.Tab>((resolve) => {
|
||||
resolveFirstTab = resolve;
|
||||
});
|
||||
const tabsApi: ChromeTabsApi = {
|
||||
get: vi
|
||||
.fn()
|
||||
.mockImplementationOnce(() => firstTab)
|
||||
.mockResolvedValue({ id: 7, url: OBSERVATION.url }),
|
||||
query: vi.fn(async () => []),
|
||||
};
|
||||
const session = new RecordingObservationSession();
|
||||
session.cursor.lastSettled = OBSERVATION;
|
||||
const drafts: RecordingDraftStep[] = [];
|
||||
const controller = new SettleController({ session, cdp, tabsApi, tabId: 7 });
|
||||
|
||||
controller.scheduleRedirect(drafts, "https://example.com/intermediate");
|
||||
await vi.advanceTimersByTimeAsync(180);
|
||||
expect(tabsApi.get).toHaveBeenCalledTimes(1);
|
||||
|
||||
controller.scheduleRedirect(drafts, OBSERVATION.url);
|
||||
resolveFirstTab({ id: 7, url: "https://example.com/intermediate" } as chrome.tabs.Tab);
|
||||
await vi.advanceTimersByTimeAsync(500);
|
||||
await controller.flushRedirects();
|
||||
|
||||
expect(drafts).toEqual([]);
|
||||
expect(controller.hasPending).toBe(false);
|
||||
});
|
||||
|
||||
it("settles an action in its own OOPIF document scope", async () => {
|
||||
vi.useFakeTimers();
|
||||
const send = vi.fn();
|
||||
const sendToTarget = vi.fn(async (_target, method: string) => {
|
||||
if (method === "Page.createIsolatedWorld") return { executionContextId: 39 };
|
||||
return { result: { value: { idleMs: 1_000, readyState: "complete" } } };
|
||||
});
|
||||
const cdp: CdpRunner = {
|
||||
send: send as unknown as CdpRunner["send"],
|
||||
sendToTarget: sendToTarget as unknown as NonNullable<CdpRunner["sendToTarget"]>,
|
||||
};
|
||||
const tabsApi: ChromeTabsApi = {
|
||||
get: vi.fn(async () => ({ id: 7, url: OBSERVATION.url }) as chrome.tabs.Tab),
|
||||
query: vi.fn(async () => []),
|
||||
};
|
||||
const session = new RecordingObservationSession();
|
||||
vi.spyOn(session, "capture").mockResolvedValue(OBSERVATION);
|
||||
const drafts: RecordingDraftStep[] = [
|
||||
{ op: "click", captureTarget: { tag: "button", name: "Inside frame" } },
|
||||
];
|
||||
const controller = new SettleController({ session, cdp, tabsApi, tabId: 7 });
|
||||
const target = { tabId: 7, sessionId: "oopif-session" };
|
||||
|
||||
controller.schedule(drafts, 0, { target, frameId: "oopif-frame" });
|
||||
await vi.advanceTimersByTimeAsync(180);
|
||||
await controller.flush();
|
||||
|
||||
expect(send).not.toHaveBeenCalled();
|
||||
expect(sendToTarget).toHaveBeenCalledWith(
|
||||
target,
|
||||
"Runtime.evaluate",
|
||||
expect.objectContaining({ contextId: 39 }),
|
||||
);
|
||||
expect(drafts[0]?.postStateId).toBe("s-next");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,164 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { appendRecordedPayload, observeRecordedNavigation } from "../recording/step-buffer";
|
||||
|
||||
describe("recording-step-buffer", () => {
|
||||
it("stores semantic click without summary", () => {
|
||||
const buffer = { steps: [], navigation: { pendingNavigation: false } };
|
||||
appendRecordedPayload(buffer, {
|
||||
op: "click",
|
||||
target: { tag: "button", role: "button", name: "发布" },
|
||||
expects_navigation: true,
|
||||
});
|
||||
expect(buffer.steps).toEqual([
|
||||
{
|
||||
op: "click",
|
||||
captureTarget: { tag: "button", role: "button", name: "发布" },
|
||||
},
|
||||
]);
|
||||
expect(buffer.navigation.pendingNavigation).toBe(true);
|
||||
});
|
||||
|
||||
it("keeps the hovered element description as a capture fallback", () => {
|
||||
const buffer = { steps: [], navigation: { pendingNavigation: false } };
|
||||
appendRecordedPayload(
|
||||
buffer,
|
||||
{
|
||||
op: "hover",
|
||||
target: { tag: "button", role: "button", name: "新建" },
|
||||
geometry: {
|
||||
rect: { x: 900, y: 8, w: 60, h: 32 },
|
||||
tag: "button",
|
||||
},
|
||||
},
|
||||
{
|
||||
geometry: {
|
||||
rect: { x: 900, y: 8, w: 60, h: 32 },
|
||||
tag: "button",
|
||||
},
|
||||
},
|
||||
);
|
||||
expect(buffer.steps).toEqual([
|
||||
{
|
||||
op: "hover",
|
||||
captureTarget: { tag: "button", role: "button", name: "新建" },
|
||||
targetHint: {
|
||||
geometry: {
|
||||
rect: { x: 900, y: 8, w: 60, h: 32 },
|
||||
tag: "button",
|
||||
},
|
||||
},
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("annotates navigated_to on action-caused navigation instead of wait_for_navigation", () => {
|
||||
const buffer = {
|
||||
steps: [
|
||||
{
|
||||
op: "click" as const,
|
||||
captureTarget: { tag: "button", role: "button", name: "发布" },
|
||||
},
|
||||
],
|
||||
navigation: {
|
||||
currentUrl: "https://example.com/a",
|
||||
pendingNavigation: true,
|
||||
pendingNavigationDeadline: Date.now() + 5_000,
|
||||
},
|
||||
};
|
||||
observeRecordedNavigation(buffer, "https://example.com/b", true);
|
||||
expect(buffer.steps).toEqual([
|
||||
{
|
||||
op: "click",
|
||||
captureTarget: { tag: "button", role: "button", name: "发布" },
|
||||
navigatedTo: "https://example.com/b",
|
||||
},
|
||||
]);
|
||||
expect(JSON.stringify(buffer.steps)).not.toContain("wait_for_navigation");
|
||||
});
|
||||
|
||||
it("annotates navigated_to onto a select that auto-submits a navigation", () => {
|
||||
const buffer = {
|
||||
steps: [
|
||||
{
|
||||
op: "select" as const,
|
||||
captureTarget: { tag: "select", role: "combobox", name: "分类" },
|
||||
values: ["tech"],
|
||||
},
|
||||
],
|
||||
navigation: {
|
||||
currentUrl: "https://example.com/list",
|
||||
pendingNavigation: true,
|
||||
pendingNavigationDeadline: Date.now() + 5_000,
|
||||
},
|
||||
};
|
||||
observeRecordedNavigation(buffer, "https://example.com/list?cat=tech", true);
|
||||
expect(buffer.steps).toEqual([
|
||||
{
|
||||
op: "select",
|
||||
captureTarget: { tag: "select", role: "combobox", name: "分类" },
|
||||
values: ["tech"],
|
||||
navigatedTo: "https://example.com/list?cat=tech",
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("emits navigate for uncaused URL changes", () => {
|
||||
const buffer = {
|
||||
steps: [],
|
||||
navigation: { currentUrl: "https://example.com/a", pendingNavigation: false },
|
||||
};
|
||||
const result = observeRecordedNavigation(buffer, "https://example.com/b", false);
|
||||
expect(result).toEqual({ kind: "appended", index: 0 });
|
||||
expect(buffer.steps[0]).toMatchObject({
|
||||
op: "navigate",
|
||||
url: "https://example.com/b",
|
||||
});
|
||||
});
|
||||
|
||||
it("asks the recorder to coalesce redirect hops instead of emitting each one", () => {
|
||||
const buffer = {
|
||||
steps: [],
|
||||
navigation: {
|
||||
currentUrl: "https://passport.example/login",
|
||||
pendingNavigation: false,
|
||||
},
|
||||
};
|
||||
const hop1 = observeRecordedNavigation(
|
||||
buffer,
|
||||
"https://passport.example/callback",
|
||||
false,
|
||||
"link",
|
||||
["server_redirect"],
|
||||
);
|
||||
expect(hop1).toEqual({
|
||||
kind: "coalesce_redirect",
|
||||
url: "https://passport.example/callback",
|
||||
});
|
||||
expect(buffer.steps).toEqual([]);
|
||||
|
||||
const hop2 = observeRecordedNavigation(buffer, "https://app.example/dashboard", false, "link", [
|
||||
"client_redirect",
|
||||
]);
|
||||
expect(hop2).toEqual({
|
||||
kind: "coalesce_redirect",
|
||||
url: "https://app.example/dashboard",
|
||||
});
|
||||
expect(buffer.steps).toEqual([]);
|
||||
expect(buffer.navigation.currentUrl).toBe("https://app.example/dashboard");
|
||||
});
|
||||
|
||||
it("does not reuse redirect metadata for a later content-observed URL change", () => {
|
||||
const buffer = {
|
||||
steps: [],
|
||||
navigation: {
|
||||
currentUrl: "https://example.com/redirect",
|
||||
pendingNavigation: false,
|
||||
},
|
||||
};
|
||||
|
||||
const result = observeRecordedNavigation(buffer, "https://example.com/spa");
|
||||
|
||||
expect(result).toEqual({ kind: "appended", index: 0 });
|
||||
expect(buffer.steps[0]).toMatchObject({ op: "navigate", url: "https://example.com/spa" });
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,39 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { RecordingTabCoordinator } from "../recording/tab-coordinator";
|
||||
|
||||
describe("RecordingTabCoordinator", () => {
|
||||
it("keeps navigation state isolated per tab", () => {
|
||||
const tabs = new RecordingTabCoordinator(4, "https://example.com/first");
|
||||
const first = tabs.navigation(4);
|
||||
first.pendingNavigation = true;
|
||||
const second = tabs.navigation(5, "https://example.com/second");
|
||||
|
||||
expect(second).toEqual({
|
||||
currentUrl: "https://example.com/second",
|
||||
pendingNavigation: false,
|
||||
});
|
||||
expect(tabs.navigation(4)).toBe(first);
|
||||
expect(tabs.navigation(4).pendingNavigation).toBe(true);
|
||||
});
|
||||
|
||||
it("invalidates an earlier activation when a newer tab becomes active", () => {
|
||||
const tabs = new RecordingTabCoordinator(4);
|
||||
const second = tabs.noteActivation(5);
|
||||
const third = tabs.noteActivation(6);
|
||||
|
||||
expect(tabs.isLatest(second)).toBe(false);
|
||||
expect(tabs.isLatest(third)).toBe(true);
|
||||
tabs.commit(6);
|
||||
expect(tabs.currentTabId).toBe(6);
|
||||
});
|
||||
|
||||
it("commits the freshly observed URL when returning to an existing tab", () => {
|
||||
const tabs = new RecordingTabCoordinator(4);
|
||||
tabs.navigation(5, "https://example.com/old");
|
||||
|
||||
tabs.commit(5, "https://example.com/new");
|
||||
|
||||
expect(tabs.currentTabId).toBe(5);
|
||||
expect(tabs.navigation(5).currentUrl).toBe("https://example.com/new");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,159 @@
|
||||
import type { RenderedRef } from "@browser-skill/vom";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { CaptureVomMatchNode } from "@/tools/capture-vom-observation";
|
||||
import { ObservationNodeIndex, type RegisteredObservation } from "../recording/observation-capture";
|
||||
import { matchObservationTarget } from "../recording/target-matcher";
|
||||
|
||||
function node(backendNodeId: number, frameId: string, x = 10): CaptureVomMatchNode {
|
||||
return {
|
||||
backendNodeId,
|
||||
frameId,
|
||||
tag: "button",
|
||||
rect: { x, y: 20, w: 100, h: 30 },
|
||||
localRect: { x, y: 20, w: 100, h: 30 },
|
||||
};
|
||||
}
|
||||
|
||||
function observation(
|
||||
matchNodes: CaptureVomMatchNode[],
|
||||
refs: RenderedRef[],
|
||||
): RegisteredObservation {
|
||||
return {
|
||||
stateId: "s1",
|
||||
rootFrameId: "root",
|
||||
index: new ObservationNodeIndex({ rootFrameId: "root", matchNodes, refs }),
|
||||
url: "https://example.com",
|
||||
};
|
||||
}
|
||||
|
||||
describe("matchObservationTarget", () => {
|
||||
it("indexes the safe geometry DTO without discarding frame-local geometry", () => {
|
||||
const geometry = node(42, "child");
|
||||
geometry.localRect = { x: 5, y: 6, w: 100, h: 30 };
|
||||
const ref: RenderedRef = {
|
||||
ref: "e1",
|
||||
backendNodeId: 42,
|
||||
frameId: "child",
|
||||
line: 1,
|
||||
};
|
||||
const candidate = observation([geometry], [ref]).index.candidates("child", "button")[0];
|
||||
|
||||
expect(candidate).toEqual({ frameId: "child", geometry, ref });
|
||||
});
|
||||
|
||||
it("matches a unique node using canonical top-level viewport geometry", () => {
|
||||
const target = matchObservationTarget({
|
||||
observation: observation(
|
||||
[node(42, "root")],
|
||||
[{ ref: "e1", backendNodeId: 42, role: "button", name: "发布", line: 1 }],
|
||||
),
|
||||
hint: { geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" } },
|
||||
});
|
||||
expect(target).toEqual({ ref: "e1", role: "button", name: "发布" });
|
||||
});
|
||||
|
||||
it("uses frame id with backend node id so sibling frames cannot collide", () => {
|
||||
const target = matchObservationTarget({
|
||||
observation: observation(
|
||||
[node(42, "left"), node(42, "right")],
|
||||
[
|
||||
{ ref: "e1", backendNodeId: 42, frameId: "left", line: 1 },
|
||||
{ ref: "e2", backendNodeId: 42, frameId: "right", line: 2 },
|
||||
],
|
||||
),
|
||||
hint: {
|
||||
frameId: "right",
|
||||
geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" },
|
||||
},
|
||||
});
|
||||
expect(target.ref).toBe("e2");
|
||||
});
|
||||
|
||||
it("matches iframe capture geometry in the child viewport coordinate space", () => {
|
||||
const child = node(42, "child", 410);
|
||||
child.localRect = { x: 10, y: 20, w: 100, h: 30 };
|
||||
const target = matchObservationTarget({
|
||||
observation: observation(
|
||||
[child],
|
||||
[{ ref: "e1", backendNodeId: 42, frameId: "child", line: 1 }],
|
||||
),
|
||||
hint: {
|
||||
frameId: "child",
|
||||
geometrySpace: "local",
|
||||
geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" },
|
||||
},
|
||||
});
|
||||
expect(target.ref).toBe("e1");
|
||||
});
|
||||
|
||||
it("restricts a missing frame hint to the root frame", () => {
|
||||
const target = matchObservationTarget({
|
||||
observation: observation(
|
||||
[node(42, "child")],
|
||||
[{ ref: "e1", backendNodeId: 42, frameId: "child", line: 1 }],
|
||||
),
|
||||
hint: { geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" } },
|
||||
fallback: { tag: "button", name: "发布" },
|
||||
});
|
||||
expect(target).toEqual({ name: "发布", unmatched: true });
|
||||
});
|
||||
|
||||
it("fails closed when the source Document has no VOM frame mapping", () => {
|
||||
const target = matchObservationTarget({
|
||||
observation: observation(
|
||||
[node(42, "root")],
|
||||
[{ ref: "e1", backendNodeId: 42, role: "button", name: "发布", line: 1 }],
|
||||
),
|
||||
hint: {
|
||||
frameId: null,
|
||||
geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" },
|
||||
},
|
||||
fallback: { tag: "button", role: "button", name: "发布" },
|
||||
});
|
||||
expect(target).toEqual({ role: "button", name: "发布", unmatched: true });
|
||||
});
|
||||
|
||||
it("returns unmatched for ambiguous geometry", () => {
|
||||
const target = matchObservationTarget({
|
||||
observation: observation(
|
||||
[node(42, "root"), node(43, "root")],
|
||||
[
|
||||
{ ref: "e1", backendNodeId: 42, line: 1 },
|
||||
{ ref: "e2", backendNodeId: 43, line: 2 },
|
||||
],
|
||||
),
|
||||
hint: { geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" } },
|
||||
});
|
||||
expect(target.unmatched).toBe(true);
|
||||
});
|
||||
|
||||
it("uses semantics when geometry is unavailable without crossing frame boundaries", () => {
|
||||
const target = matchObservationTarget({
|
||||
observation: observation(
|
||||
[node(41, "root"), node(42, "child")],
|
||||
[
|
||||
{ ref: "e1", backendNodeId: 41, role: "button", name: "保存", line: 1 },
|
||||
{ ref: "e2", backendNodeId: 42, frameId: "child", role: "button", name: "保存", line: 2 },
|
||||
],
|
||||
),
|
||||
hint: { frameId: "child" },
|
||||
fallback: { tag: "button", role: "button", name: "保存" },
|
||||
});
|
||||
expect(target.ref).toBe("e2");
|
||||
});
|
||||
|
||||
it("uses semantics to disambiguate equal geometry in one frame", () => {
|
||||
const target = matchObservationTarget({
|
||||
observation: observation(
|
||||
[node(42, "root"), node(43, "root")],
|
||||
[
|
||||
{ ref: "e1", backendNodeId: 42, role: "button", name: "保存", line: 1 },
|
||||
{ ref: "e2", backendNodeId: 43, role: "button", name: "取消", line: 2 },
|
||||
],
|
||||
),
|
||||
hint: { geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" } },
|
||||
fallback: { tag: "button", role: "button", name: "取消" },
|
||||
});
|
||||
expect(target.ref).toBe("e2");
|
||||
});
|
||||
});
|
||||
+69
-32
@@ -1,6 +1,15 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { DraftTraceStep } from "@/transport/types";
|
||||
import { reduceTraceSteps, resolveTraceStartUrl, shouldRecordPress } from "../trace-reducer";
|
||||
import type { RecordingDraftStep } from "@/lib/recording/types";
|
||||
import { shouldRecordPress } from "../recording/draft-policy";
|
||||
import { buildTraceV2 } from "../recording/trace-reducer-v2";
|
||||
|
||||
function reduceTraceSteps(steps: RecordingDraftStep[], startUrl?: string) {
|
||||
return buildTraceV2({
|
||||
steps,
|
||||
startedAt: "2026-01-01T00:00:00.000Z",
|
||||
...(startUrl ? { startUrl } : {}),
|
||||
});
|
||||
}
|
||||
|
||||
describe("shouldRecordPress", () => {
|
||||
it("keeps Enter and Escape", () => {
|
||||
@@ -18,36 +27,47 @@ describe("shouldRecordPress", () => {
|
||||
});
|
||||
|
||||
describe("reduceTraceSteps", () => {
|
||||
it("does not expose internal tab transitions in trace v2", () => {
|
||||
const trace = reduceTraceSteps([
|
||||
{
|
||||
op: "switch_tab",
|
||||
preStateId: "s1",
|
||||
postStateId: "s2",
|
||||
},
|
||||
]);
|
||||
expect(trace.steps).toEqual([]);
|
||||
});
|
||||
|
||||
it("builds steps with pages dictionary and page id references", () => {
|
||||
const drafts: DraftTraceStep[] = [
|
||||
const drafts: RecordingDraftStep[] = [
|
||||
{
|
||||
op: "navigate",
|
||||
url: "https://example.com/search?q=hello&utm_source=x",
|
||||
page_url: "https://example.com/search?q=hello&utm_source=x",
|
||||
pageUrl: "https://example.com/search?q=hello&utm_source=x",
|
||||
},
|
||||
{
|
||||
op: "fill",
|
||||
target: { tag: "input", role: "textbox", name: "搜索", name_attr: "q" },
|
||||
captureTarget: { tag: "input", role: "textbox", name: "搜索", name_attr: "q" },
|
||||
value: "browser skill",
|
||||
page_url: "https://example.com/search?q=hello&utm_source=x",
|
||||
pageUrl: "https://example.com/search?q=hello&utm_source=x",
|
||||
},
|
||||
{
|
||||
op: "press",
|
||||
key: "Enter",
|
||||
target: { tag: "input", role: "textbox", name: "搜索", name_attr: "q" },
|
||||
navigated_to: "https://example.com/results/42",
|
||||
page_url: "https://example.com/search?q=hello&utm_source=x",
|
||||
captureTarget: { tag: "input", role: "textbox", name: "搜索", name_attr: "q" },
|
||||
navigatedTo: "https://example.com/results/42",
|
||||
pageUrl: "https://example.com/search?q=hello&utm_source=x",
|
||||
},
|
||||
{
|
||||
op: "click",
|
||||
target: { tag: "button", role: "button", name: "发布" },
|
||||
navigated_to: "https://example.com/p/99",
|
||||
page_url: "https://example.com/results/42",
|
||||
captureTarget: { tag: "button", role: "button", name: "发布" },
|
||||
navigatedTo: "https://example.com/p/99",
|
||||
pageUrl: "https://example.com/results/42",
|
||||
},
|
||||
{
|
||||
op: "press",
|
||||
key: "a",
|
||||
page_url: "https://example.com/p/99",
|
||||
pageUrl: "https://example.com/p/99",
|
||||
},
|
||||
];
|
||||
|
||||
@@ -104,16 +124,37 @@ describe("reduceTraceSteps", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("maps select navigated_to onto effect.navigated_to (page id)", () => {
|
||||
it("keeps committed empty fill steps", () => {
|
||||
const { steps } = reduceTraceSteps(
|
||||
[
|
||||
{
|
||||
op: "fill",
|
||||
captureTarget: { tag: "input", role: "textbox", name: "Search query" },
|
||||
value: "",
|
||||
pageUrl: "https://example.com/search",
|
||||
},
|
||||
],
|
||||
"https://example.com/search",
|
||||
);
|
||||
|
||||
expect(steps).toEqual([
|
||||
expect.objectContaining({
|
||||
op: "fill",
|
||||
value: "",
|
||||
}),
|
||||
]);
|
||||
});
|
||||
|
||||
it("maps select navigatedTo onto effect.navigatedTo (page id)", () => {
|
||||
const { pages, steps } = reduceTraceSteps(
|
||||
[
|
||||
{
|
||||
op: "select",
|
||||
target: { tag: "select", role: "combobox", name: "分类" },
|
||||
captureTarget: { tag: "select", role: "combobox", name: "分类" },
|
||||
values: ["tech"],
|
||||
labels: ["技术"],
|
||||
navigated_to: "https://example.com/list?cat=tech",
|
||||
page_url: "https://example.com/list",
|
||||
navigatedTo: "https://example.com/list?cat=tech",
|
||||
pageUrl: "https://example.com/list",
|
||||
},
|
||||
],
|
||||
"https://example.com/list",
|
||||
@@ -130,36 +171,32 @@ describe("reduceTraceSteps", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps hover steps before menu clicks", () => {
|
||||
it("drops hover steps, which only exist in trace v3", () => {
|
||||
const { steps } = reduceTraceSteps(
|
||||
[
|
||||
{
|
||||
op: "hover",
|
||||
target: { tag: "span", role: "button", name: "Account" },
|
||||
page_url: "https://example.com/app",
|
||||
captureTarget: { tag: "span", role: "button", name: "Account" },
|
||||
pageUrl: "https://example.com/app",
|
||||
},
|
||||
{
|
||||
op: "click",
|
||||
target: { tag: "a", role: "link", name: "Profile" },
|
||||
page_url: "https://example.com/app",
|
||||
captureTarget: { tag: "a", role: "link", name: "Profile" },
|
||||
pageUrl: "https://example.com/app",
|
||||
},
|
||||
],
|
||||
"https://example.com/app",
|
||||
);
|
||||
expect(steps.map((s) => s.op)).toEqual(["hover", "click"]);
|
||||
expect(steps[0]).toMatchObject({
|
||||
op: "hover",
|
||||
page: "p1",
|
||||
target: { name: "Account" },
|
||||
});
|
||||
expect(steps.map((s) => s.op)).toEqual(["click"]);
|
||||
});
|
||||
|
||||
it("resolveTraceStartUrl prefers explicit start URL", () => {
|
||||
expect(
|
||||
resolveTraceStartUrl(
|
||||
[{ op: "navigate", url: "https://example.com/other" }],
|
||||
"https://example.com/start",
|
||||
),
|
||||
buildTraceV2({
|
||||
steps: [{ op: "navigate", url: "https://example.com/other" }],
|
||||
startedAt: "2026-01-01T00:00:00.000Z",
|
||||
startUrl: "https://example.com/start",
|
||||
}).entry.start_url,
|
||||
).toBe("https://example.com/start");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,132 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { TRACE_VERSION_V3, VOM_FORMAT_VERSION } from "@/transport/types";
|
||||
import { RecordingStateRegistry } from "../recording/state-registry";
|
||||
import { buildTraceV3 } from "../recording/trace-builder-v3";
|
||||
import { reduceTraceStepsV3 } from "../recording/trace-reducer-v3";
|
||||
import type { RecordingDraftStep } from "../recording/types";
|
||||
|
||||
describe("trace reducer v3", () => {
|
||||
it("removes form literals when value redaction is requested", () => {
|
||||
const reduced = reduceTraceStepsV3(
|
||||
[
|
||||
{
|
||||
op: "fill",
|
||||
value: "private@example.com",
|
||||
preStateId: "s1",
|
||||
postStateId: "s1",
|
||||
},
|
||||
{
|
||||
op: "select",
|
||||
values: ["private-account-id"],
|
||||
labels: ["Private account"],
|
||||
preStateId: "s1",
|
||||
postStateId: "s1",
|
||||
},
|
||||
],
|
||||
{ redactValues: true },
|
||||
);
|
||||
|
||||
expect(reduced.steps[0]).toMatchObject({ op: "fill", value: "***", redacted: true });
|
||||
expect(reduced.steps[1]).toMatchObject({ op: "select" });
|
||||
expect(reduced.steps[1]).not.toHaveProperty("selection");
|
||||
expect(JSON.stringify(reduced.steps)).not.toContain("private@example.com");
|
||||
expect(JSON.stringify(reduced.steps)).not.toContain("private-account-id");
|
||||
});
|
||||
|
||||
it("emits tab transitions only for callers that advertised support", () => {
|
||||
const drafts: RecordingDraftStep[] = [
|
||||
{
|
||||
op: "switch_tab",
|
||||
preStateId: "s1",
|
||||
postStateId: "s2",
|
||||
},
|
||||
];
|
||||
|
||||
expect(reduceTraceStepsV3(drafts).steps).toEqual([]);
|
||||
expect(reduceTraceStepsV3(drafts, { includeTabSwitches: true }).steps).toEqual([
|
||||
{ op: "switch_tab", id: 1, state: "s1", result: { state: "s2" } },
|
||||
]);
|
||||
});
|
||||
|
||||
it("keeps hover steps before menu clicks", () => {
|
||||
const drafts: RecordingDraftStep[] = [
|
||||
{
|
||||
op: "hover",
|
||||
captureTarget: { tag: "span", role: "button", name: "Account" },
|
||||
preStateId: "s1",
|
||||
postStateId: "s1",
|
||||
},
|
||||
{
|
||||
op: "click",
|
||||
captureTarget: { tag: "a", role: "link", name: "Profile" },
|
||||
preStateId: "s1",
|
||||
postStateId: "s1",
|
||||
},
|
||||
];
|
||||
|
||||
expect(reduceTraceStepsV3(drafts).steps.map((s) => s.op)).toEqual(["hover", "click"]);
|
||||
});
|
||||
|
||||
it("does not export a tab transition without both observation endpoints", () => {
|
||||
const sourceOnly: RecordingDraftStep = { op: "switch_tab", preStateId: "s1" };
|
||||
const targetOnly: RecordingDraftStep = { op: "switch_tab", postStateId: "s2" };
|
||||
|
||||
expect(
|
||||
reduceTraceStepsV3([sourceOnly, targetOnly], { includeTabSwitches: true }).steps,
|
||||
).toEqual([]);
|
||||
});
|
||||
|
||||
it("collapses redirect hops while retaining draft-to-step identity", () => {
|
||||
const drafts: RecordingDraftStep[] = [
|
||||
{ op: "navigate", url: "https://example.com/start", preStateId: "s1", postStateId: "s2" },
|
||||
{
|
||||
op: "navigate",
|
||||
url: "https://example.com/final",
|
||||
transitionQualifiers: ["server_redirect"],
|
||||
preStateId: "s2",
|
||||
postStateId: "s3",
|
||||
},
|
||||
];
|
||||
const before = structuredClone(drafts);
|
||||
const reduced = reduceTraceStepsV3(drafts);
|
||||
|
||||
expect(drafts).toEqual(before);
|
||||
expect(reduced.steps).toEqual([
|
||||
expect.objectContaining({
|
||||
id: 1,
|
||||
op: "navigate",
|
||||
state: "s1",
|
||||
to: "https://example.com/final",
|
||||
result: { state: "s3" },
|
||||
}),
|
||||
]);
|
||||
expect(reduced.stepIdByDraftId.get(1)).toBe(1);
|
||||
expect(reduced.stepIdByDraftId.get(2)).toBe(1);
|
||||
});
|
||||
|
||||
it("builds the wire model from protocol constants", () => {
|
||||
const registry = new RecordingStateRegistry();
|
||||
const state = registry.register({ url: "https://example.com", vomText: "@vom 1" });
|
||||
const trace = buildTraceV3({
|
||||
registry,
|
||||
drafts: [
|
||||
{
|
||||
op: "click",
|
||||
captureTarget: { tag: "button", role: "button", name: "Save" },
|
||||
preStateId: state.id,
|
||||
postStateId: state.id,
|
||||
},
|
||||
],
|
||||
startedAt: "2026-08-12T00:00:00.000Z",
|
||||
stoppedBy: "user_finish",
|
||||
bskVersion: "test",
|
||||
});
|
||||
|
||||
expect(trace.version).toBe(TRACE_VERSION_V3);
|
||||
expect(trace.recorder.vom).toBe(VOM_FORMAT_VERSION);
|
||||
expect(trace.steps[0]).toMatchObject({
|
||||
op: "click",
|
||||
target: { role: "button", name: "Save", unmatched: true },
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,12 +1,13 @@
|
||||
/**
|
||||
* Build a semantic TargetDescriptor for an interacted element.
|
||||
* Build a semantic capture descriptor for an interacted element.
|
||||
*
|
||||
* Trace steps are an LLM *textbook*: each click must say what to look for
|
||||
* on screen (usually a short visible name). Tag-only noise like
|
||||
* `{ "tag": "div" }` / “点击div” is useless and must not be recorded.
|
||||
*/
|
||||
|
||||
export interface TargetDescriptor {
|
||||
/** Content-script capture descriptor before VOM geometric matching. */
|
||||
export interface CaptureTargetDescriptor {
|
||||
role?: string;
|
||||
name?: string;
|
||||
tag: string;
|
||||
@@ -247,7 +248,7 @@ export function resolveHoverElement(target: Element): Element | null {
|
||||
* (visible name), or at least a form `name_attr` for checkbox/radio.
|
||||
* Recording “点击div” with no name fails this bar.
|
||||
*/
|
||||
export function isMeaningfulClickTarget(target: TargetDescriptor): boolean {
|
||||
export function isMeaningfulClickTarget(target: CaptureTargetDescriptor): boolean {
|
||||
const name = target.name?.trim();
|
||||
if (name && isActionableLabel(name)) return true;
|
||||
if (
|
||||
@@ -323,7 +324,7 @@ function nearbyLabelText(el: Element): string | undefined {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
export function describeTarget(el: Element): TargetDescriptor {
|
||||
export function describeTarget(el: Element): CaptureTargetDescriptor {
|
||||
const tag = el.tagName.toLowerCase();
|
||||
const role = inferRole(el);
|
||||
const name = accessibleName(el);
|
||||
@@ -350,7 +351,7 @@ export function describeTarget(el: Element): TargetDescriptor {
|
||||
};
|
||||
}
|
||||
|
||||
export function describeEventTarget(target: EventTarget | null): TargetDescriptor | null {
|
||||
export function describeEventTarget(target: EventTarget | null): CaptureTargetDescriptor | null {
|
||||
if (!(target instanceof Element)) return null;
|
||||
const clickable = resolveClickableElement(target);
|
||||
if (!clickable) return null;
|
||||
|
||||
@@ -16,6 +16,13 @@ export const HELP_ACK = "bsk-help-ack";
|
||||
export const HELP_QUERY = "bsk-help-query";
|
||||
export const HELP_FINISH = "bsk-help-finish";
|
||||
|
||||
export interface HelpHighlightRect {
|
||||
top: number;
|
||||
left: number;
|
||||
width: number;
|
||||
height: number;
|
||||
}
|
||||
|
||||
export interface HelpRequestMessage {
|
||||
type: typeof HELP_REQUEST;
|
||||
requestId: string;
|
||||
@@ -26,6 +33,8 @@ export interface HelpRequestMessage {
|
||||
displayMode?: "full" | "compact";
|
||||
/** CSS selectors to scroll to + flash-highlight (may be empty). */
|
||||
selectors: string[];
|
||||
/** Top-viewport rectangles for targets that cannot be represented by a root selector. */
|
||||
rects?: HelpHighlightRect[];
|
||||
timeoutMs: number;
|
||||
}
|
||||
|
||||
@@ -74,6 +83,16 @@ export function isHelpRequestMessage(msg: unknown): msg is HelpRequestMessage {
|
||||
(m.displayMode === undefined || m.displayMode === "full" || m.displayMode === "compact") &&
|
||||
Array.isArray(m.selectors) &&
|
||||
m.selectors.every((selector) => typeof selector === "string") &&
|
||||
(m.rects === undefined ||
|
||||
(Array.isArray(m.rects) &&
|
||||
m.rects.every(
|
||||
(rect) =>
|
||||
typeof rect === "object" &&
|
||||
rect !== null &&
|
||||
["top", "left", "width", "height"].every((key) =>
|
||||
Number.isFinite((rect as Record<string, unknown>)[key]),
|
||||
),
|
||||
))) &&
|
||||
typeof m.timeoutMs === "number"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
* service worker and a tab's content script.
|
||||
*/
|
||||
|
||||
import type { TargetDescriptor } from "./describe-target";
|
||||
import type { CaptureTargetDescriptor } from "./describe-target";
|
||||
|
||||
export const RECORD_START = "bsk-record-start";
|
||||
export const RECORD_STEP = "bsk-record-step";
|
||||
@@ -16,6 +16,10 @@ export interface RecordStartAck {
|
||||
ok: true;
|
||||
}
|
||||
|
||||
export type RecordStepAck =
|
||||
| { ok: true; sequence: number }
|
||||
| { ok: false; expectedSequence: number; error: string };
|
||||
|
||||
export type RecordStopAck = { ok: true } | { ok: false; error: string };
|
||||
|
||||
export interface RecordQueryMessage {
|
||||
@@ -42,7 +46,7 @@ export interface RecordStartMessage {
|
||||
|
||||
export interface RecordStepPayload {
|
||||
op: "click" | "hover" | "fill" | "press" | "select" | "navigate";
|
||||
target?: TargetDescriptor;
|
||||
target?: CaptureTargetDescriptor;
|
||||
value?: string;
|
||||
key?: string;
|
||||
modifiers?: Array<"alt" | "ctrl" | "meta" | "shift">;
|
||||
@@ -50,17 +54,30 @@ export interface RecordStepPayload {
|
||||
labels?: string[];
|
||||
url?: string;
|
||||
redacted?: boolean;
|
||||
commit?: "enter" | "suggestion" | "blur";
|
||||
/** Page URL when the step was captured. */
|
||||
page_url?: string;
|
||||
/** Event-target bounds in the source frame's viewport coordinate space. */
|
||||
geometry?: {
|
||||
rect: { x: number; y: number; w: number; h: number };
|
||||
tag: string;
|
||||
};
|
||||
/** Capture-only hint; never persisted unless converted to navigated_to. */
|
||||
expects_navigation?: boolean;
|
||||
/** Whether an observed URL change was synchronously caused by the action. */
|
||||
navigation_caused_by_action?: boolean;
|
||||
/** Raw webNavigation transition metadata for cause mapping. */
|
||||
transitionType?: string;
|
||||
transitionQualifiers?: string[];
|
||||
}
|
||||
|
||||
export interface RecordStepMessage {
|
||||
type: typeof RECORD_STEP;
|
||||
requestId: string;
|
||||
/** Stable for the lifetime of one content-script document. */
|
||||
producerId: string;
|
||||
/** Monotonic within one content-script producer and recording request. */
|
||||
sequence: number;
|
||||
step: RecordStepPayload;
|
||||
}
|
||||
|
||||
@@ -112,8 +129,27 @@ export function isRecordQueryMessage(msg: unknown): msg is RecordQueryMessage {
|
||||
export function isRecordStepMessage(msg: unknown): msg is RecordStepMessage {
|
||||
if (typeof msg !== "object" || msg === null) return false;
|
||||
const m = msg as Record<string, unknown>;
|
||||
if (m.type !== RECORD_STEP || typeof m.requestId !== "string") return false;
|
||||
if (
|
||||
m.type !== RECORD_STEP ||
|
||||
typeof m.requestId !== "string" ||
|
||||
typeof m.producerId !== "string" ||
|
||||
m.producerId.length === 0 ||
|
||||
typeof m.sequence !== "number" ||
|
||||
!Number.isSafeInteger(m.sequence) ||
|
||||
m.sequence < 1
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
const step = m.step;
|
||||
if (typeof step !== "object" || step === null) return false;
|
||||
return typeof (step as RecordStepPayload).op === "string";
|
||||
}
|
||||
|
||||
export function isAcceptedRecordStepAck(
|
||||
value: unknown,
|
||||
sequence: number,
|
||||
): value is Extract<RecordStepAck, { ok: true }> {
|
||||
if (typeof value !== "object" || value === null) return false;
|
||||
const ack = value as Partial<RecordStepAck>;
|
||||
return ack.ok === true && ack.sequence === sequence;
|
||||
}
|
||||
|
||||
@@ -1,136 +0,0 @@
|
||||
import type { DraftTraceStep } from "@/transport/types";
|
||||
import type { RecordStepPayload } from "./record-bridge";
|
||||
|
||||
export interface RecordingStepBuffer {
|
||||
steps: DraftTraceStep[];
|
||||
currentUrl?: string;
|
||||
pendingNavigation: boolean;
|
||||
pendingNavigationDeadline?: number;
|
||||
}
|
||||
|
||||
const NAVIGATION_TRIGGER_WINDOW_MS = 3_000;
|
||||
|
||||
function toDraftStep(payload: RecordStepPayload): DraftTraceStep | null {
|
||||
const pageUrl = payload.page_url;
|
||||
switch (payload.op) {
|
||||
case "click":
|
||||
return payload.target
|
||||
? {
|
||||
op: "click",
|
||||
target: payload.target,
|
||||
...(pageUrl ? { page_url: pageUrl } : {}),
|
||||
}
|
||||
: null;
|
||||
case "hover":
|
||||
return payload.target
|
||||
? {
|
||||
op: "hover",
|
||||
target: payload.target,
|
||||
...(pageUrl ? { page_url: pageUrl } : {}),
|
||||
}
|
||||
: null;
|
||||
case "fill":
|
||||
return payload.target
|
||||
? {
|
||||
op: "fill",
|
||||
target: payload.target,
|
||||
value: payload.value ?? "",
|
||||
...(payload.redacted ? { redacted: true } : {}),
|
||||
...(pageUrl ? { page_url: pageUrl } : {}),
|
||||
}
|
||||
: null;
|
||||
case "press":
|
||||
return payload.key
|
||||
? {
|
||||
op: "press",
|
||||
key: payload.key,
|
||||
...(payload.target ? { target: payload.target } : {}),
|
||||
...(payload.modifiers?.length ? { modifiers: payload.modifiers } : {}),
|
||||
...(pageUrl ? { page_url: pageUrl } : {}),
|
||||
}
|
||||
: null;
|
||||
case "select":
|
||||
return payload.target && payload.values
|
||||
? {
|
||||
op: "select",
|
||||
target: payload.target,
|
||||
values: payload.values,
|
||||
...(payload.labels?.length ? { labels: payload.labels } : {}),
|
||||
...(pageUrl ? { page_url: pageUrl } : {}),
|
||||
}
|
||||
: null;
|
||||
case "navigate":
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function annotateLastStepNavigation(buffer: RecordingStepBuffer, url: string): boolean {
|
||||
for (let i = buffer.steps.length - 1; i >= 0; i -= 1) {
|
||||
const step = buffer.steps[i];
|
||||
if (!step) continue;
|
||||
if (step.op === "click" || step.op === "press" || step.op === "select") {
|
||||
buffer.steps[i] = { ...step, navigated_to: url };
|
||||
return true;
|
||||
}
|
||||
break;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
export function observeRecordedNavigation(
|
||||
buffer: RecordingStepBuffer,
|
||||
url: string,
|
||||
causedByAction?: boolean,
|
||||
): void {
|
||||
if (!url || url === buffer.currentUrl) return;
|
||||
buffer.currentUrl = url;
|
||||
const pendingIsCurrent =
|
||||
buffer.pendingNavigation &&
|
||||
(buffer.pendingNavigationDeadline === undefined ||
|
||||
buffer.pendingNavigationDeadline >= Date.now());
|
||||
|
||||
if (causedByAction === true || (causedByAction === undefined && pendingIsCurrent)) {
|
||||
buffer.pendingNavigation = false;
|
||||
buffer.pendingNavigationDeadline = undefined;
|
||||
if (!annotateLastStepNavigation(buffer, url)) {
|
||||
buffer.steps.push({
|
||||
op: "navigate",
|
||||
url,
|
||||
page_url: url,
|
||||
});
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
buffer.pendingNavigation = false;
|
||||
buffer.pendingNavigationDeadline = undefined;
|
||||
buffer.steps.push({
|
||||
op: "navigate",
|
||||
url,
|
||||
page_url: url,
|
||||
});
|
||||
}
|
||||
|
||||
export function appendRecordedPayload(
|
||||
buffer: RecordingStepBuffer,
|
||||
payload: RecordStepPayload,
|
||||
): void {
|
||||
if (payload.op === "navigate") {
|
||||
if (payload.url) {
|
||||
observeRecordedNavigation(buffer, payload.url, payload.navigation_caused_by_action);
|
||||
}
|
||||
return;
|
||||
}
|
||||
const step = toDraftStep({
|
||||
...payload,
|
||||
page_url: payload.page_url ?? buffer.currentUrl,
|
||||
});
|
||||
if (!step) return;
|
||||
buffer.steps.push(step);
|
||||
if (step.op === "click" || step.op === "press" || step.op === "select" || step.op === "fill") {
|
||||
buffer.pendingNavigation = payload.expects_navigation === true;
|
||||
buffer.pendingNavigationDeadline = buffer.pendingNavigation
|
||||
? Date.now() + NAVIGATION_TRIGGER_WINDOW_MS
|
||||
: undefined;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
import {
|
||||
RECORD_DOCUMENT_ATTRIBUTE,
|
||||
recordingDocumentMarkerValue,
|
||||
} from "@/shared/recording-document-identity";
|
||||
|
||||
export interface RecordingDocumentMarker {
|
||||
restore(): void;
|
||||
ensure(): void;
|
||||
}
|
||||
|
||||
export function waitForDocumentElement(): Promise<HTMLElement> {
|
||||
if (document.documentElement) return Promise.resolve(document.documentElement);
|
||||
return new Promise((resolve) => {
|
||||
const observer = new MutationObserver(() => {
|
||||
if (!document.documentElement) return;
|
||||
observer.disconnect();
|
||||
resolve(document.documentElement);
|
||||
});
|
||||
observer.observe(document, { childList: true });
|
||||
});
|
||||
}
|
||||
|
||||
export function markRecordingDocument(
|
||||
producerId: string,
|
||||
root: HTMLElement = document.documentElement,
|
||||
): RecordingDocumentMarker {
|
||||
const previous = root.getAttribute(RECORD_DOCUMENT_ATTRIBUTE);
|
||||
const markerValue = recordingDocumentMarkerValue(producerId);
|
||||
const ensure = () => root.setAttribute(RECORD_DOCUMENT_ATTRIBUTE, markerValue);
|
||||
ensure();
|
||||
return {
|
||||
ensure,
|
||||
restore() {
|
||||
if (root.getAttribute(RECORD_DOCUMENT_ATTRIBUTE) !== markerValue) return;
|
||||
if (previous === null) root.removeAttribute(RECORD_DOCUMENT_ATTRIBUTE);
|
||||
else root.setAttribute(RECORD_DOCUMENT_ATTRIBUTE, previous);
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
import type { CdpTarget } from "@/browser-driver/frame-graph";
|
||||
import { type CdpRunner, sendToCdpTarget } from "@/tools/shared";
|
||||
|
||||
const SETTLE_MIN_MS = 150;
|
||||
const SETTLE_QUIET_MS = 250;
|
||||
const SETTLE_MAX_MS = 2_000;
|
||||
const SETTLE_POLL_MS = 60;
|
||||
const SETTLE_WORLD_NAME = "__bsk_record_settle__";
|
||||
|
||||
const QUIET_PROBE = `(() => {
|
||||
const scope = window;
|
||||
let probe = scope.__bskRecordQuiet;
|
||||
if (!probe) {
|
||||
probe = { changedAt: Date.now() };
|
||||
const observer = new MutationObserver(() => {
|
||||
probe.changedAt = Date.now();
|
||||
});
|
||||
observer.observe(document.documentElement, {
|
||||
subtree: true,
|
||||
childList: true,
|
||||
attributes: true,
|
||||
characterData: true,
|
||||
});
|
||||
scope.__bskRecordQuiet = probe;
|
||||
}
|
||||
return { idleMs: Date.now() - probe.changedAt, readyState: document.readyState };
|
||||
})()`;
|
||||
|
||||
export interface DocumentSettleScope {
|
||||
target: CdpTarget;
|
||||
/** Omit only for the target's root document. */
|
||||
frameId?: string;
|
||||
}
|
||||
|
||||
type SettleOutcome = "quiet" | "timeout" | "cancelled";
|
||||
|
||||
interface QuietProbe {
|
||||
idleMs: number;
|
||||
readyState: string;
|
||||
}
|
||||
|
||||
interface ProbeContext {
|
||||
executionContextId?: number;
|
||||
}
|
||||
|
||||
function sleep(ms: number, signal?: AbortSignal): Promise<void> {
|
||||
return new Promise((resolve) => {
|
||||
if (signal?.aborted) return resolve();
|
||||
const timer = setTimeout(resolve, ms);
|
||||
signal?.addEventListener(
|
||||
"abort",
|
||||
() => {
|
||||
clearTimeout(timer);
|
||||
resolve();
|
||||
},
|
||||
{ once: true },
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
async function executionContextForScope(
|
||||
cdp: CdpRunner,
|
||||
scope: DocumentSettleScope,
|
||||
): Promise<number | undefined> {
|
||||
if (!scope.frameId) return undefined;
|
||||
const result = await sendToCdpTarget<{ executionContextId?: number }>(
|
||||
cdp,
|
||||
scope.target,
|
||||
"Page.createIsolatedWorld",
|
||||
{
|
||||
frameId: scope.frameId,
|
||||
worldName: SETTLE_WORLD_NAME,
|
||||
grantUniveralAccess: false,
|
||||
},
|
||||
);
|
||||
return result.executionContextId;
|
||||
}
|
||||
|
||||
async function readQuietProbe(
|
||||
cdp: CdpRunner,
|
||||
scope: DocumentSettleScope,
|
||||
context: ProbeContext,
|
||||
): Promise<QuietProbe | null> {
|
||||
try {
|
||||
if (scope.frameId && context.executionContextId === undefined) {
|
||||
context.executionContextId = await executionContextForScope(cdp, scope);
|
||||
if (context.executionContextId === undefined) return null;
|
||||
}
|
||||
const reply = await sendToCdpTarget<{ result?: { value?: unknown } }>(
|
||||
cdp,
|
||||
scope.target,
|
||||
"Runtime.evaluate",
|
||||
{
|
||||
expression: QUIET_PROBE,
|
||||
returnByValue: true,
|
||||
...(context.executionContextId !== undefined
|
||||
? { contextId: context.executionContextId }
|
||||
: {}),
|
||||
},
|
||||
);
|
||||
const value = reply.result?.value;
|
||||
if (!value || typeof value !== "object") return null;
|
||||
const { idleMs, readyState } = value as { idleMs?: unknown; readyState?: unknown };
|
||||
if (typeof idleMs !== "number" || typeof readyState !== "string") return null;
|
||||
return { idleMs, readyState };
|
||||
} catch {
|
||||
context.executionContextId = undefined;
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export async function waitForDocumentSettled(
|
||||
cdp: CdpRunner,
|
||||
scope: DocumentSettleScope,
|
||||
options: { signal?: AbortSignal } = {},
|
||||
): Promise<SettleOutcome> {
|
||||
const startedAt = Date.now();
|
||||
const floor = startedAt + SETTLE_MIN_MS;
|
||||
const deadline = startedAt + SETTLE_MAX_MS;
|
||||
const context: ProbeContext = {};
|
||||
|
||||
for (;;) {
|
||||
if (options.signal?.aborted) return "cancelled";
|
||||
await sleep(SETTLE_POLL_MS, options.signal);
|
||||
if (options.signal?.aborted) return "cancelled";
|
||||
|
||||
const probe = await readQuietProbe(cdp, scope, context);
|
||||
const now = Date.now();
|
||||
if (now < floor) continue;
|
||||
if (now >= deadline) return "timeout";
|
||||
if (!probe || probe.readyState === "loading") continue;
|
||||
if (probe.idleMs >= SETTLE_QUIET_MS) return "quiet";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
import type { RecordingDraftStep } from "./types";
|
||||
|
||||
const CLIPBOARD_KEYS = new Set(["a", "c", "v", "x", "A", "C", "V", "X"]);
|
||||
const MODIFIER_ONLY_KEYS = new Set(["Meta", "Control", "Alt", "Shift", "OS", "Hyper", "Super"]);
|
||||
|
||||
export function shouldRecordPress(
|
||||
key: string,
|
||||
modifiers?: Array<"alt" | "ctrl" | "meta" | "shift">,
|
||||
): boolean {
|
||||
if (MODIFIER_ONLY_KEYS.has(key)) return false;
|
||||
const mods = modifiers ?? [];
|
||||
const hasCtrlOrMeta = mods.includes("ctrl") || mods.includes("meta");
|
||||
if (hasCtrlOrMeta && CLIPBOARD_KEYS.has(key)) return false;
|
||||
if (key === "Enter" || key === "Escape") return true;
|
||||
if (key.length === 1 && !hasCtrlOrMeta && !mods.includes("alt")) return false;
|
||||
return false;
|
||||
}
|
||||
|
||||
export function shouldIncludeDraft(step: RecordingDraftStep): boolean {
|
||||
return step.op !== "press" || shouldRecordPress(step.key, step.modifiers);
|
||||
}
|
||||
|
||||
export function resolveDraftStartUrl(
|
||||
drafts: RecordingDraftStep[],
|
||||
explicitStartUrl?: string,
|
||||
fallbackUrl?: string,
|
||||
): string {
|
||||
if (explicitStartUrl) return explicitStartUrl;
|
||||
const navigation = drafts.find(
|
||||
(step): step is Extract<RecordingDraftStep, { op: "navigate" }> => step.op === "navigate",
|
||||
);
|
||||
if (navigation) return navigation.url;
|
||||
return drafts.find((step) => step.pageUrl)?.pageUrl ?? fallbackUrl ?? "about:blank";
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
export const RECORD_FRAME_PORT = "bsk-record-frame";
|
||||
export const RECORD_FRAME_QUERY = "bsk-record-frame-query";
|
||||
export const RECORD_FRAME_START = "bsk-record-frame-start";
|
||||
|
||||
export interface RecordFrameQueryMessage {
|
||||
type: typeof RECORD_FRAME_QUERY;
|
||||
}
|
||||
|
||||
export interface RecordFrameQueryResponse {
|
||||
active: boolean;
|
||||
requestId?: string;
|
||||
startedAtMs?: number;
|
||||
}
|
||||
|
||||
export interface RecordFrameStartMessage {
|
||||
type: typeof RECORD_FRAME_START;
|
||||
requestId: string;
|
||||
startedAtMs: number;
|
||||
}
|
||||
|
||||
export type RecordFramePortMessage =
|
||||
| {
|
||||
type: "ready";
|
||||
requestId: string;
|
||||
producerId: string;
|
||||
}
|
||||
| {
|
||||
type: "ready_ack";
|
||||
requestId: string;
|
||||
producerId: string;
|
||||
}
|
||||
| {
|
||||
type: "stop";
|
||||
requestId: string;
|
||||
commandId: string;
|
||||
}
|
||||
| {
|
||||
type: "cancel";
|
||||
requestId: string;
|
||||
}
|
||||
| {
|
||||
type: "stopped";
|
||||
requestId: string;
|
||||
commandId: string;
|
||||
ok: boolean;
|
||||
error?: string;
|
||||
};
|
||||
|
||||
export function isRecordFrameQueryMessage(value: unknown): value is RecordFrameQueryMessage {
|
||||
return (
|
||||
typeof value === "object" &&
|
||||
value !== null &&
|
||||
(value as { type?: unknown }).type === RECORD_FRAME_QUERY
|
||||
);
|
||||
}
|
||||
|
||||
export function isRecordFrameStartMessage(value: unknown): value is RecordFrameStartMessage {
|
||||
if (typeof value !== "object" || value === null) return false;
|
||||
const message = value as Partial<RecordFrameStartMessage>;
|
||||
return (
|
||||
message.type === RECORD_FRAME_START &&
|
||||
typeof message.requestId === "string" &&
|
||||
typeof message.startedAtMs === "number"
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,399 @@
|
||||
import {
|
||||
isRecordFrameQueryMessage,
|
||||
RECORD_FRAME_PORT,
|
||||
RECORD_FRAME_START,
|
||||
type RecordFramePortMessage,
|
||||
type RecordFrameQueryResponse,
|
||||
type RecordFrameStartMessage,
|
||||
} from "./frame-bridge";
|
||||
|
||||
export interface RecordingCaptureScope {
|
||||
tabId: number;
|
||||
documentId: string;
|
||||
browserFrameId: number;
|
||||
producerId: string;
|
||||
}
|
||||
|
||||
interface ArmedRecording {
|
||||
requestId: string;
|
||||
startedAtMs: number;
|
||||
tabIds: Set<number>;
|
||||
agents: Map<string, FrameAgent>;
|
||||
onDocumentReady?: (scope: RecordingCaptureScope) => void | Promise<void>;
|
||||
finishing: boolean;
|
||||
}
|
||||
|
||||
interface FrameAgent extends RecordingCaptureScope {
|
||||
requestId: string;
|
||||
port: chrome.runtime.Port;
|
||||
stopWaiters: Map<string, (ok: boolean) => void>;
|
||||
}
|
||||
|
||||
interface BrowserFrame {
|
||||
frameId: number;
|
||||
documentId?: string;
|
||||
}
|
||||
|
||||
interface BrowserFrameNavigation extends BrowserFrame {
|
||||
tabId: number;
|
||||
lifecycle: "committed" | "completed";
|
||||
}
|
||||
|
||||
const RECORD_FRAME_STOP_TIMEOUT_MS = 5_000;
|
||||
|
||||
export interface RecordFrameCoordinatorDeps {
|
||||
getAllFrames(tabId: number): Promise<BrowserFrame[]>;
|
||||
sendToDocument(
|
||||
tabId: number,
|
||||
message: RecordFrameStartMessage,
|
||||
target: { documentId?: string; frameId?: number },
|
||||
): Promise<unknown>;
|
||||
subscribeFrameNavigation?(listener: (frame: BrowserFrameNavigation) => void): () => void;
|
||||
}
|
||||
|
||||
function documentKey(tabId: number, documentId: string): string {
|
||||
return `${tabId}:${documentId}`;
|
||||
}
|
||||
|
||||
function senderAddress(
|
||||
sender: chrome.runtime.MessageSender,
|
||||
): { tabId: number; documentId: string; browserFrameId: number } | null {
|
||||
const tabId = sender.tab?.id;
|
||||
const documentId = sender.documentId;
|
||||
const browserFrameId = sender.frameId;
|
||||
if (
|
||||
typeof tabId !== "number" ||
|
||||
typeof documentId !== "string" ||
|
||||
typeof browserFrameId !== "number"
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
return { tabId, documentId, browserFrameId };
|
||||
}
|
||||
|
||||
function defaultDeps(): RecordFrameCoordinatorDeps {
|
||||
return {
|
||||
async getAllFrames(tabId) {
|
||||
return (await chrome.webNavigation.getAllFrames({ tabId })) ?? [];
|
||||
},
|
||||
sendToDocument(tabId, message, target) {
|
||||
return chrome.tabs.sendMessage(tabId, message, target);
|
||||
},
|
||||
subscribeFrameNavigation(listener) {
|
||||
const onCommitted = (details: chrome.webNavigation.WebNavigationTransitionCallbackDetails) =>
|
||||
listener({ ...details, lifecycle: "committed" });
|
||||
const onCompleted = (details: chrome.webNavigation.WebNavigationFramedCallbackDetails) =>
|
||||
listener({ ...details, lifecycle: "completed" });
|
||||
chrome.webNavigation.onCommitted.addListener(onCommitted);
|
||||
chrome.webNavigation.onCompleted.addListener(onCompleted);
|
||||
return () => {
|
||||
chrome.webNavigation.onCommitted.removeListener(onCommitted);
|
||||
chrome.webNavigation.onCompleted.removeListener(onCompleted);
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export class RecordFrameCoordinator {
|
||||
readonly #deps: RecordFrameCoordinatorDeps;
|
||||
readonly #recordings = new Map<string, ArmedRecording>();
|
||||
#attached = false;
|
||||
#detachFrameNavigation: (() => void) | null = null;
|
||||
|
||||
constructor(deps: RecordFrameCoordinatorDeps = defaultDeps()) {
|
||||
this.#deps = deps;
|
||||
}
|
||||
|
||||
attach(): () => void {
|
||||
if (this.#attached) return () => {};
|
||||
this.#attached = true;
|
||||
chrome.runtime.onConnect.addListener(this.#onConnect);
|
||||
chrome.runtime.onMessage.addListener(this.#onMessage);
|
||||
this.#detachFrameNavigation =
|
||||
this.#deps.subscribeFrameNavigation?.(this.#onFrameNavigation) ?? null;
|
||||
return () => {
|
||||
if (!this.#attached) return;
|
||||
this.#attached = false;
|
||||
chrome.runtime.onConnect.removeListener(this.#onConnect);
|
||||
chrome.runtime.onMessage.removeListener(this.#onMessage);
|
||||
this.#detachFrameNavigation?.();
|
||||
this.#detachFrameNavigation = null;
|
||||
for (const recording of this.#recordings.values()) this.#cancelAgents(recording);
|
||||
this.#recordings.clear();
|
||||
};
|
||||
}
|
||||
|
||||
begin(
|
||||
requestId: string,
|
||||
startedAtMs: number,
|
||||
initialTabId: number,
|
||||
onDocumentReady?: (scope: RecordingCaptureScope) => void | Promise<void>,
|
||||
): void {
|
||||
const previous = this.#recordings.get(requestId);
|
||||
if (previous) this.#cancelAgents(previous);
|
||||
this.#recordings.set(requestId, {
|
||||
requestId,
|
||||
startedAtMs,
|
||||
// Register the initial tab before navigation begins. A newly-created
|
||||
// child document can query at document_start before armTab's frame
|
||||
// snapshot sees it, especially when Chromium promotes it to an OOPIF.
|
||||
tabIds: new Set([initialTabId]),
|
||||
agents: new Map(),
|
||||
...(onDocumentReady ? { onDocumentReady } : {}),
|
||||
finishing: false,
|
||||
});
|
||||
}
|
||||
|
||||
async armTab(requestId: string, tabId: number): Promise<boolean> {
|
||||
const recording = this.#recordings.get(requestId);
|
||||
if (!recording || recording.finishing) return false;
|
||||
recording.tabIds.add(tabId);
|
||||
|
||||
let frames: BrowserFrame[];
|
||||
try {
|
||||
frames = await this.#deps.getAllFrames(tabId);
|
||||
} catch {
|
||||
frames = [{ frameId: 0 }];
|
||||
}
|
||||
if (!frames.some((frame) => frame.frameId === 0)) frames.unshift({ frameId: 0 });
|
||||
|
||||
const results = await Promise.all(
|
||||
frames.map(async (frame) => {
|
||||
const started = await this.#startFrame(recording, tabId, frame);
|
||||
return { frameId: frame.frameId, started };
|
||||
}),
|
||||
);
|
||||
return results.some((result) => result.frameId === 0 && result.started);
|
||||
}
|
||||
|
||||
sourceFor(
|
||||
requestId: string,
|
||||
producerId: string,
|
||||
sender: chrome.runtime.MessageSender,
|
||||
): RecordingCaptureScope | null {
|
||||
const address = senderAddress(sender);
|
||||
const recording = this.#recordings.get(requestId);
|
||||
if (!address || !recording) return null;
|
||||
const agent = recording.agents.get(documentKey(address.tabId, address.documentId));
|
||||
if (!agent || agent.producerId !== producerId) return null;
|
||||
return {
|
||||
tabId: agent.tabId,
|
||||
documentId: agent.documentId,
|
||||
browserFrameId: agent.browserFrameId,
|
||||
producerId: agent.producerId,
|
||||
};
|
||||
}
|
||||
|
||||
async stop(requestId: string): Promise<boolean> {
|
||||
const recording = this.#recordings.get(requestId);
|
||||
if (!recording) return true;
|
||||
recording.finishing = true;
|
||||
const commandId = crypto.randomUUID();
|
||||
const results = await Promise.all(
|
||||
[...recording.agents.values()].map((agent) => this.#stopAgent(agent, requestId, commandId)),
|
||||
);
|
||||
const succeeded = results.every(Boolean);
|
||||
if (succeeded) this.#recordings.delete(requestId);
|
||||
else recording.finishing = false;
|
||||
return succeeded;
|
||||
}
|
||||
|
||||
cancel(requestId: string): void {
|
||||
const recording = this.#recordings.get(requestId);
|
||||
if (!recording) return;
|
||||
this.#cancelAgents(recording);
|
||||
this.#recordings.delete(requestId);
|
||||
}
|
||||
|
||||
readonly #onMessage = (
|
||||
message: unknown,
|
||||
sender: chrome.runtime.MessageSender,
|
||||
sendResponse: (response: RecordFrameQueryResponse) => void,
|
||||
): boolean => {
|
||||
if (!isRecordFrameQueryMessage(message)) return false;
|
||||
const tabId = sender.tab?.id;
|
||||
const recording =
|
||||
typeof tabId === "number"
|
||||
? [...this.#recordings.values()].find(
|
||||
(candidate) => !candidate.finishing && candidate.tabIds.has(tabId),
|
||||
)
|
||||
: undefined;
|
||||
sendResponse(
|
||||
recording
|
||||
? {
|
||||
active: true,
|
||||
requestId: recording.requestId,
|
||||
startedAtMs: recording.startedAtMs,
|
||||
}
|
||||
: { active: false },
|
||||
);
|
||||
return false;
|
||||
};
|
||||
|
||||
readonly #onFrameNavigation = (frame: BrowserFrameNavigation): void => {
|
||||
if (frame.frameId === 0) return;
|
||||
for (const recording of this.#recordings.values()) {
|
||||
if (recording.finishing || !recording.tabIds.has(frame.tabId)) continue;
|
||||
// onCommitted can race document_start, while onCompleted runs after the
|
||||
// content script is available. Starting twice is safe and closes both
|
||||
// the early-query and late-SPA-frame gaps.
|
||||
void (async () => {
|
||||
const started = await this.#startFrame(recording, frame.tabId, frame);
|
||||
if (!started || frame.lifecycle !== "completed" || recording.finishing) return;
|
||||
const agent = frame.documentId
|
||||
? recording.agents.get(documentKey(frame.tabId, frame.documentId))
|
||||
: [...recording.agents.values()].find(
|
||||
(candidate) =>
|
||||
candidate.tabId === frame.tabId && candidate.browserFrameId === frame.frameId,
|
||||
);
|
||||
if (!agent) return;
|
||||
await recording.onDocumentReady?.({
|
||||
tabId: agent.tabId,
|
||||
documentId: agent.documentId,
|
||||
browserFrameId: agent.browserFrameId,
|
||||
producerId: agent.producerId,
|
||||
});
|
||||
})();
|
||||
}
|
||||
};
|
||||
|
||||
readonly #onConnect = (port: chrome.runtime.Port): void => {
|
||||
if (port.name !== RECORD_FRAME_PORT) return;
|
||||
const address = senderAddress(port.sender ?? {});
|
||||
if (!address) {
|
||||
port.disconnect();
|
||||
return;
|
||||
}
|
||||
let registered: FrameAgent | null = null;
|
||||
port.onMessage.addListener((raw: unknown) => {
|
||||
const message = raw as Partial<RecordFramePortMessage>;
|
||||
if (message.type === "ready") {
|
||||
if (typeof message.requestId !== "string" || typeof message.producerId !== "string") {
|
||||
port.disconnect();
|
||||
return;
|
||||
}
|
||||
const requestId = message.requestId;
|
||||
const producerId = message.producerId;
|
||||
if (registered) {
|
||||
if (registered.requestId === requestId && registered.producerId === producerId) {
|
||||
port.postMessage({
|
||||
type: "ready_ack",
|
||||
requestId,
|
||||
producerId,
|
||||
} satisfies RecordFramePortMessage);
|
||||
} else {
|
||||
port.disconnect();
|
||||
}
|
||||
return;
|
||||
}
|
||||
const recording = this.#recordings.get(requestId);
|
||||
if (
|
||||
!recording ||
|
||||
recording.finishing ||
|
||||
!recording.tabIds.has(address.tabId) ||
|
||||
producerId.length === 0
|
||||
) {
|
||||
port.disconnect();
|
||||
return;
|
||||
}
|
||||
const agent: FrameAgent = {
|
||||
...address,
|
||||
requestId,
|
||||
producerId,
|
||||
port,
|
||||
stopWaiters: new Map(),
|
||||
};
|
||||
registered = agent;
|
||||
const key = documentKey(address.tabId, address.documentId);
|
||||
const previous = recording.agents.get(key);
|
||||
if (previous && previous !== agent) previous.port.disconnect();
|
||||
recording.agents.set(key, agent);
|
||||
port.postMessage({
|
||||
type: "ready_ack",
|
||||
requestId,
|
||||
producerId,
|
||||
} satisfies RecordFramePortMessage);
|
||||
return;
|
||||
}
|
||||
if (
|
||||
message.type === "stopped" &&
|
||||
registered &&
|
||||
message.requestId === registered.requestId &&
|
||||
typeof message.commandId === "string"
|
||||
) {
|
||||
registered.stopWaiters.get(message.commandId)?.(message.ok === true);
|
||||
registered.stopWaiters.delete(message.commandId);
|
||||
}
|
||||
});
|
||||
port.onDisconnect.addListener(() => {
|
||||
if (!registered) return;
|
||||
for (const resolve of registered.stopWaiters.values()) resolve(true);
|
||||
const recording = this.#recordings.get(registered.requestId);
|
||||
const key = documentKey(registered.tabId, registered.documentId);
|
||||
if (recording?.agents.get(key) === registered) recording.agents.delete(key);
|
||||
});
|
||||
};
|
||||
|
||||
#stopAgent(agent: FrameAgent, requestId: string, commandId: string): Promise<boolean> {
|
||||
return new Promise((resolve) => {
|
||||
const timer = setTimeout(() => {
|
||||
agent.stopWaiters.delete(commandId);
|
||||
resolve(false);
|
||||
}, RECORD_FRAME_STOP_TIMEOUT_MS);
|
||||
agent.stopWaiters.set(commandId, (ok) => {
|
||||
clearTimeout(timer);
|
||||
resolve(ok);
|
||||
});
|
||||
try {
|
||||
agent.port.postMessage({
|
||||
type: "stop",
|
||||
requestId,
|
||||
commandId,
|
||||
} satisfies RecordFramePortMessage);
|
||||
} catch {
|
||||
clearTimeout(timer);
|
||||
agent.stopWaiters.delete(commandId);
|
||||
resolve(true);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async #startFrame(
|
||||
recording: ArmedRecording,
|
||||
tabId: number,
|
||||
frame: BrowserFrame,
|
||||
): Promise<boolean> {
|
||||
const message: RecordFrameStartMessage = {
|
||||
type: RECORD_FRAME_START,
|
||||
requestId: recording.requestId,
|
||||
startedAtMs: recording.startedAtMs,
|
||||
};
|
||||
try {
|
||||
const response = await this.#deps.sendToDocument(tabId, message, {
|
||||
...(frame.documentId ? { documentId: frame.documentId } : { frameId: frame.frameId }),
|
||||
});
|
||||
return isStarted(response);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
#cancelAgents(recording: ArmedRecording): void {
|
||||
for (const agent of recording.agents.values()) {
|
||||
try {
|
||||
agent.port.postMessage({
|
||||
type: "cancel",
|
||||
requestId: recording.requestId,
|
||||
} satisfies RecordFramePortMessage);
|
||||
} catch {
|
||||
// A destroyed Document has no remaining capture state to cancel.
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function isStarted(value: unknown): boolean {
|
||||
return typeof value === "object" && value !== null && (value as { ok?: unknown }).ok === true;
|
||||
}
|
||||
|
||||
export const recordFrameCoordinator = new RecordFrameCoordinator();
|
||||
@@ -0,0 +1,5 @@
|
||||
const REDIRECT_QUALIFIERS = new Set(["client_redirect", "server_redirect"]);
|
||||
|
||||
export function hasRedirectQualifier(qualifiers?: readonly string[]): boolean {
|
||||
return (qualifiers ?? []).some((qualifier) => REDIRECT_QUALIFIERS.has(qualifier));
|
||||
}
|
||||
@@ -0,0 +1,140 @@
|
||||
import type { RenderedRef } from "@browser-skill/vom";
|
||||
import type { CdpTarget } from "@/browser-driver/frame-graph";
|
||||
import {
|
||||
type CaptureVomMatchNode,
|
||||
type CaptureVomObservationResult,
|
||||
captureVomObservation,
|
||||
} from "@/tools/capture-vom-observation";
|
||||
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
|
||||
|
||||
export interface IndexedObservationNode {
|
||||
frameId: string;
|
||||
geometry: CaptureVomMatchNode;
|
||||
ref?: RenderedRef;
|
||||
}
|
||||
|
||||
export interface CapturedRecordingObservation {
|
||||
rootFrameId: string;
|
||||
index: ObservationNodeIndex;
|
||||
url: string;
|
||||
title?: string;
|
||||
vomText: string;
|
||||
truncated: boolean;
|
||||
}
|
||||
|
||||
export interface RegisteredObservation {
|
||||
stateId: string;
|
||||
rootFrameId: string;
|
||||
index: ObservationNodeIndex;
|
||||
url: string;
|
||||
}
|
||||
|
||||
export interface RecordingDocumentScope {
|
||||
frameId: string;
|
||||
target: CdpTarget;
|
||||
}
|
||||
|
||||
function nodeKey(frameId: string, backendNodeId: number): string {
|
||||
return `${frameId}:${backendNodeId}`;
|
||||
}
|
||||
|
||||
function frameTagKey(frameId: string, tag: string): string {
|
||||
return `${frameId}:${tag.toLowerCase()}`;
|
||||
}
|
||||
|
||||
export class ObservationNodeIndex {
|
||||
readonly #nodesByFrameTag = new Map<string, IndexedObservationNode[]>();
|
||||
readonly #refById = new Map<string, RenderedRef>();
|
||||
readonly #refsByFrame = new Map<string, RenderedRef[]>();
|
||||
readonly #scopeByProducer = new Map<string, RecordingDocumentScope | null>();
|
||||
|
||||
constructor(
|
||||
input: Pick<CaptureVomObservationResult, "rootFrameId" | "matchNodes" | "refs"> &
|
||||
Partial<Pick<CaptureVomObservationResult, "frames">>,
|
||||
) {
|
||||
const refByNode = new Map<string, RenderedRef>();
|
||||
for (const ref of input.refs) {
|
||||
const frameId = ref.frameId ?? input.rootFrameId;
|
||||
refByNode.set(nodeKey(frameId, ref.backendNodeId), ref);
|
||||
this.#refById.set(ref.ref, ref);
|
||||
const frameRefs = this.#refsByFrame.get(frameId) ?? [];
|
||||
frameRefs.push(ref);
|
||||
this.#refsByFrame.set(frameId, frameRefs);
|
||||
}
|
||||
for (const frame of input.frames ?? []) {
|
||||
const documentId = frame.recordingDocumentId;
|
||||
if (!documentId) continue;
|
||||
this.#scopeByProducer.set(
|
||||
documentId,
|
||||
this.#scopeByProducer.has(documentId)
|
||||
? null
|
||||
: { frameId: frame.frameId, target: frame.target },
|
||||
);
|
||||
}
|
||||
for (const geometry of input.matchNodes) {
|
||||
const { frameId } = geometry;
|
||||
const entry = {
|
||||
frameId,
|
||||
geometry,
|
||||
ref: refByNode.get(nodeKey(frameId, geometry.backendNodeId)),
|
||||
};
|
||||
const key = frameTagKey(frameId, geometry.tag);
|
||||
const bucket = this.#nodesByFrameTag.get(key) ?? [];
|
||||
bucket.push(entry);
|
||||
this.#nodesByFrameTag.set(key, bucket);
|
||||
}
|
||||
}
|
||||
|
||||
candidates(frameId: string, tag: string): readonly IndexedObservationNode[] {
|
||||
return this.#nodesByFrameTag.get(frameTagKey(frameId, tag)) ?? [];
|
||||
}
|
||||
|
||||
ref(refId: string): RenderedRef | undefined {
|
||||
return this.#refById.get(refId);
|
||||
}
|
||||
|
||||
refs(frameId: string): readonly RenderedRef[] {
|
||||
return this.#refsByFrame.get(frameId) ?? [];
|
||||
}
|
||||
|
||||
documentScope(producerId: string): RecordingDocumentScope | undefined {
|
||||
return this.#scopeByProducer.get(producerId) ?? undefined;
|
||||
}
|
||||
}
|
||||
|
||||
async function readTabMeta(
|
||||
tabsApi: ChromeTabsApi,
|
||||
tabId: number,
|
||||
): Promise<{ url: string; title?: string }> {
|
||||
try {
|
||||
const tab = await tabsApi.get(tabId);
|
||||
return { url: tab.url ?? "about:blank", title: tab.title };
|
||||
} catch {
|
||||
return { url: "about:blank" };
|
||||
}
|
||||
}
|
||||
|
||||
export async function captureRecordingObservation(input: {
|
||||
cdp: CdpRunner;
|
||||
tabsApi: ChromeTabsApi;
|
||||
tabId: number;
|
||||
maxTokens: number;
|
||||
redactValues: boolean;
|
||||
signal?: AbortSignal;
|
||||
}): Promise<CapturedRecordingObservation> {
|
||||
const { url, title } = await readTabMeta(input.tabsApi, input.tabId);
|
||||
const captured = await captureVomObservation(input.cdp, input.tabId, url, {
|
||||
maxTokens: input.maxTokens,
|
||||
redactValues: input.redactValues,
|
||||
conditionalSurfaceProbe: false,
|
||||
signal: input.signal,
|
||||
});
|
||||
return {
|
||||
rootFrameId: captured.rootFrameId,
|
||||
index: new ObservationNodeIndex(captured),
|
||||
url,
|
||||
title,
|
||||
vomText: captured.text,
|
||||
truncated: captured.truncated,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
|
||||
import { captureRecordingObservation, type RegisteredObservation } from "./observation-capture";
|
||||
import { RecordingStateRegistry } from "./state-registry";
|
||||
import { matchObservationTarget, unmatchedTarget } from "./target-matcher";
|
||||
import type { RecordingDraftStep, StepAnnotation, TargetedRecordingDraft } from "./types";
|
||||
|
||||
const DEFAULT_MAX_PAGE_TOKENS = 3_000;
|
||||
const MIN_CAPTURE_INTERVAL_MS = 200;
|
||||
|
||||
export interface TabObservationCursor {
|
||||
lastSettled: RegisteredObservation | null;
|
||||
lastCaptureAt: number;
|
||||
}
|
||||
|
||||
function abortableDelay(ms: number, signal?: AbortSignal): Promise<void> {
|
||||
return new Promise((resolve) => {
|
||||
if (signal?.aborted) return resolve();
|
||||
const timer = setTimeout(resolve, ms);
|
||||
signal?.addEventListener(
|
||||
"abort",
|
||||
() => {
|
||||
clearTimeout(timer);
|
||||
resolve();
|
||||
},
|
||||
{ once: true },
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
function isTargeted(draft: RecordingDraftStep): draft is TargetedRecordingDraft {
|
||||
return (
|
||||
draft.op === "click" ||
|
||||
draft.op === "hover" ||
|
||||
draft.op === "fill" ||
|
||||
draft.op === "press" ||
|
||||
draft.op === "select"
|
||||
);
|
||||
}
|
||||
|
||||
export class RecordingObservationSession {
|
||||
readonly registry: RecordingStateRegistry;
|
||||
readonly cursor: TabObservationCursor;
|
||||
readonly annotations: StepAnnotation[];
|
||||
readonly #maxTokens: number;
|
||||
readonly #redactValues: boolean;
|
||||
|
||||
constructor(
|
||||
options: {
|
||||
registry?: RecordingStateRegistry;
|
||||
cursor?: TabObservationCursor;
|
||||
annotations?: StepAnnotation[];
|
||||
maxTokens?: number;
|
||||
redactValues?: boolean;
|
||||
} = {},
|
||||
) {
|
||||
this.registry = options.registry ?? new RecordingStateRegistry();
|
||||
this.cursor = options.cursor ?? { lastSettled: null, lastCaptureAt: 0 };
|
||||
this.annotations = options.annotations ?? [];
|
||||
this.#maxTokens = options.maxTokens ?? DEFAULT_MAX_PAGE_TOKENS;
|
||||
this.#redactValues = options.redactValues ?? false;
|
||||
}
|
||||
|
||||
async capture(
|
||||
cdp: CdpRunner,
|
||||
tabsApi: ChromeTabsApi,
|
||||
tabId: number,
|
||||
signal?: AbortSignal,
|
||||
): Promise<RegisteredObservation> {
|
||||
const waitMs = Math.max(0, MIN_CAPTURE_INTERVAL_MS - (Date.now() - this.cursor.lastCaptureAt));
|
||||
if (waitMs > 0) await abortableDelay(waitMs, signal);
|
||||
if (signal?.aborted) throw new DOMException("observation aborted", "AbortError");
|
||||
const captured = await captureRecordingObservation({
|
||||
cdp,
|
||||
tabsApi,
|
||||
tabId,
|
||||
maxTokens: this.#maxTokens,
|
||||
redactValues: this.#redactValues,
|
||||
signal,
|
||||
});
|
||||
const state = this.registry.register({
|
||||
url: captured.url,
|
||||
title: captured.title,
|
||||
vomText: captured.vomText,
|
||||
truncated: captured.truncated,
|
||||
});
|
||||
const observation: RegisteredObservation = {
|
||||
stateId: state.id,
|
||||
rootFrameId: captured.rootFrameId,
|
||||
index: captured.index,
|
||||
url: captured.url,
|
||||
};
|
||||
this.cursor.lastSettled = observation;
|
||||
this.cursor.lastCaptureAt = Date.now();
|
||||
return observation;
|
||||
}
|
||||
|
||||
bindDraft(draft: RecordingDraftStep, draftId: number, previousActionPending = false): void {
|
||||
const observation = this.cursor.lastSettled;
|
||||
if (isTargeted(draft)) {
|
||||
draft.matchedTarget = observation
|
||||
? matchObservationTarget({
|
||||
observation,
|
||||
hint: draft.targetHint,
|
||||
fallback: draft.captureTarget,
|
||||
})
|
||||
: unmatchedTarget(draft.captureTarget);
|
||||
}
|
||||
if (!observation) return;
|
||||
|
||||
const unmatched = isTargeted(draft) && draft.matchedTarget?.unmatched === true;
|
||||
if (previousActionPending && unmatched) return;
|
||||
draft.preStateId = observation.stateId;
|
||||
this.registry.markStep(observation.stateId, draftId);
|
||||
|
||||
if (!isTargeted(draft) || !draft.matchedTarget?.ref) return;
|
||||
const ref = observation.index.ref(draft.matchedTarget.ref);
|
||||
if (!ref) return;
|
||||
this.annotations.push({
|
||||
draftId,
|
||||
op: draft.op,
|
||||
line: ref.line,
|
||||
stateId: observation.stateId,
|
||||
...(draft.op === "fill" && !this.#redactValues
|
||||
? { detail: JSON.stringify(draft.value) }
|
||||
: {}),
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,277 @@
|
||||
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
|
||||
import type { StopReason, TraceV3 } from "@/transport/types";
|
||||
import { type DocumentSettleScope, waitForDocumentSettled } from "./document-settle";
|
||||
import type { RegisteredObservation } from "./observation-capture";
|
||||
import { RecordingObservationSession } from "./observation-session";
|
||||
import { inferMissingPostStates, SettleController } from "./settle-controller";
|
||||
import { RecordingStateRegistry } from "./state-registry";
|
||||
import { buildTraceV3 } from "./trace-builder-v3";
|
||||
import type { RecordingDraftStep, StepAnnotation } from "./types";
|
||||
|
||||
interface TabRecordingContext {
|
||||
session: RecordingObservationSession;
|
||||
settle: SettleController;
|
||||
pendingCapture: PendingCapture | null;
|
||||
frameRefresh: Promise<void>;
|
||||
frameRefreshAbort: AbortController | null;
|
||||
}
|
||||
|
||||
interface PendingCapture {
|
||||
promise: Promise<RegisteredObservation>;
|
||||
abort: AbortController;
|
||||
}
|
||||
|
||||
export class RecordingObservationRuntime {
|
||||
readonly #cdp: CdpRunner;
|
||||
readonly #tabsApi: ChromeTabsApi;
|
||||
readonly #registry = new RecordingStateRegistry();
|
||||
readonly #annotations: StepAnnotation[] = [];
|
||||
readonly #contexts = new Map<number, TabRecordingContext>();
|
||||
readonly #maxTokens?: number;
|
||||
readonly #redactValues: boolean;
|
||||
|
||||
constructor(input: {
|
||||
cdp: CdpRunner;
|
||||
tabsApi: ChromeTabsApi;
|
||||
maxTokens?: number;
|
||||
redactValues?: boolean;
|
||||
}) {
|
||||
this.#cdp = input.cdp;
|
||||
this.#tabsApi = input.tabsApi;
|
||||
this.#maxTokens = input.maxTokens;
|
||||
this.#redactValues = input.redactValues ?? false;
|
||||
}
|
||||
|
||||
#context(tabId: number): TabRecordingContext {
|
||||
const existing = this.#contexts.get(tabId);
|
||||
if (existing) return existing;
|
||||
const session = new RecordingObservationSession({
|
||||
registry: this.#registry,
|
||||
annotations: this.#annotations,
|
||||
maxTokens: this.#maxTokens,
|
||||
redactValues: this.#redactValues,
|
||||
});
|
||||
const context: TabRecordingContext = {
|
||||
session,
|
||||
settle: new SettleController({
|
||||
session,
|
||||
cdp: this.#cdp,
|
||||
tabsApi: this.#tabsApi,
|
||||
tabId,
|
||||
}),
|
||||
pendingCapture: null,
|
||||
frameRefresh: Promise.resolve(),
|
||||
frameRefreshAbort: null,
|
||||
};
|
||||
this.#contexts.set(tabId, context);
|
||||
return context;
|
||||
}
|
||||
|
||||
async #capture(tabId: number): Promise<RegisteredObservation> {
|
||||
const context = this.#context(tabId);
|
||||
if (context.pendingCapture) return context.pendingCapture.promise;
|
||||
|
||||
const abort = new AbortController();
|
||||
let pending!: PendingCapture;
|
||||
const promise = context.session.capture(this.#cdp, this.#tabsApi, tabId, abort.signal);
|
||||
pending = {
|
||||
abort,
|
||||
promise: promise.finally(() => {
|
||||
if (context.pendingCapture === pending) context.pendingCapture = null;
|
||||
}),
|
||||
};
|
||||
context.pendingCapture = pending;
|
||||
return pending.promise;
|
||||
}
|
||||
|
||||
async captureInitial(tabId: number): Promise<void> {
|
||||
const context = this.#context(tabId);
|
||||
if (context.session.cursor.lastSettled) return;
|
||||
await this.#capture(tabId);
|
||||
}
|
||||
|
||||
/** Refresh the safe observation after a late child Document becomes usable. */
|
||||
async refreshDocument(tabId: number, producerId: string): Promise<void> {
|
||||
const context = this.#context(tabId);
|
||||
context.frameRefreshAbort?.abort();
|
||||
const abort = new AbortController();
|
||||
context.frameRefreshAbort = abort;
|
||||
const previous = context.frameRefresh;
|
||||
const refresh = (async () => {
|
||||
await previous;
|
||||
if (abort.signal.aborted) return;
|
||||
if (context.pendingCapture) await Promise.allSettled([context.pendingCapture.promise]);
|
||||
if (abort.signal.aborted) return;
|
||||
|
||||
// The first capture discovers the safe producer -> CDP Document scope.
|
||||
await this.#capture(tabId);
|
||||
if (abort.signal.aborted) return;
|
||||
const scope = context.session.cursor.lastSettled?.index.documentScope(producerId);
|
||||
if (scope) {
|
||||
await waitForDocumentSettled(this.#cdp, scope, { signal: abort.signal });
|
||||
}
|
||||
if (abort.signal.aborted) return;
|
||||
|
||||
// Capture again after the child SPA is quiet so the next action can be
|
||||
// matched against refs that actually exist in its pre-action state.
|
||||
await this.#capture(tabId);
|
||||
})();
|
||||
const tracked = refresh.catch(() => {});
|
||||
context.frameRefresh = tracked;
|
||||
try {
|
||||
await refresh;
|
||||
} finally {
|
||||
if (context.frameRefresh === tracked) context.frameRefreshAbort = null;
|
||||
}
|
||||
}
|
||||
|
||||
async captureTabTransition(
|
||||
fromTabId: number,
|
||||
toTabId: number,
|
||||
): Promise<{ preStateId?: string; postStateId?: string; targetUrl?: string }> {
|
||||
const from = this.#context(fromTabId);
|
||||
await this.#flushContext(from);
|
||||
if (!from.session.cursor.lastSettled) {
|
||||
try {
|
||||
await this.captureInitial(fromTabId);
|
||||
} catch {
|
||||
// A target-tab observation can still preserve the forward transition.
|
||||
}
|
||||
}
|
||||
|
||||
const to = this.#context(toTabId);
|
||||
let target = to.session.cursor.lastSettled;
|
||||
try {
|
||||
target = await this.#capture(toTabId);
|
||||
} catch {
|
||||
// Reuse the last settled target state when a refresh races navigation.
|
||||
}
|
||||
|
||||
const source = from.session.cursor.lastSettled;
|
||||
return {
|
||||
...(source ? { preStateId: source.stateId } : {}),
|
||||
...(target ? { postStateId: target.stateId, targetUrl: target.url } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
bindTabTransition(
|
||||
draft: Extract<RecordingDraftStep, { op: "switch_tab" }>,
|
||||
draftId: number,
|
||||
): void {
|
||||
if (draft.preStateId) this.#registry.markStep(draft.preStateId, draftId);
|
||||
}
|
||||
|
||||
async processDraft(
|
||||
tabId: number,
|
||||
drafts: RecordingDraftStep[],
|
||||
draftIndex: number,
|
||||
producerId?: string,
|
||||
): Promise<void> {
|
||||
const draft = drafts[draftIndex];
|
||||
if (!draft) return;
|
||||
const context = this.#context(tabId);
|
||||
if (!context.session.cursor.lastSettled) {
|
||||
try {
|
||||
await this.captureInitial(tabId);
|
||||
} catch {
|
||||
// Post-action settle can still provide a usable state.
|
||||
}
|
||||
}
|
||||
const scope = producerId
|
||||
? context.session.cursor.lastSettled?.index.documentScope(producerId)
|
||||
: undefined;
|
||||
if (
|
||||
producerId &&
|
||||
(draft.op === "click" ||
|
||||
draft.op === "hover" ||
|
||||
draft.op === "fill" ||
|
||||
draft.op === "press" ||
|
||||
draft.op === "select")
|
||||
) {
|
||||
draft.targetHint = {
|
||||
...(draft.targetHint ?? {}),
|
||||
frameId: scope?.frameId ?? null,
|
||||
geometrySpace: "local",
|
||||
};
|
||||
}
|
||||
context.session.bindDraft(draft, draftIndex + 1, context.settle.hasPending);
|
||||
context.settle.schedule(drafts, draftIndex, scope);
|
||||
}
|
||||
|
||||
scheduleSettle(
|
||||
tabId: number,
|
||||
drafts: RecordingDraftStep[],
|
||||
draftIndex: number,
|
||||
scope?: DocumentSettleScope,
|
||||
): void {
|
||||
this.#context(tabId).settle.schedule(drafts, draftIndex, scope);
|
||||
}
|
||||
|
||||
clearRedirect(tabId: number): void {
|
||||
this.#contexts.get(tabId)?.settle.clearRedirect();
|
||||
}
|
||||
|
||||
scheduleRedirect(tabId: number, drafts: RecordingDraftStep[], url: string): void {
|
||||
this.#context(tabId).settle.scheduleRedirect(drafts, url);
|
||||
}
|
||||
|
||||
async #flushContext(context: TabRecordingContext): Promise<void> {
|
||||
await context.frameRefresh;
|
||||
if (context.pendingCapture) await Promise.allSettled([context.pendingCapture.promise]);
|
||||
await context.settle.flushRedirects();
|
||||
await context.settle.flush();
|
||||
}
|
||||
|
||||
async flushRedirects(tabId: number): Promise<void> {
|
||||
await this.#contexts.get(tabId)?.settle.flushRedirects();
|
||||
}
|
||||
|
||||
async flush(): Promise<void> {
|
||||
for (const context of this.#contexts.values()) await this.#flushContext(context);
|
||||
}
|
||||
|
||||
async settleTrailing(tabId: number, drafts: RecordingDraftStep[]): Promise<void> {
|
||||
const context = this.#context(tabId);
|
||||
if (context.pendingCapture) await Promise.allSettled([context.pendingCapture.promise]);
|
||||
if (!context.session.cursor.lastSettled) {
|
||||
try {
|
||||
await this.captureInitial(tabId);
|
||||
} catch {
|
||||
// The trace may remain observation-free when CDP is unavailable.
|
||||
}
|
||||
}
|
||||
await context.settle.settleTrailing(drafts);
|
||||
inferMissingPostStates(drafts);
|
||||
}
|
||||
|
||||
cancel(): void {
|
||||
for (const context of this.#contexts.values()) {
|
||||
context.frameRefreshAbort?.abort();
|
||||
context.pendingCapture?.abort.abort();
|
||||
context.settle.cancel();
|
||||
}
|
||||
}
|
||||
|
||||
buildTrace(input: {
|
||||
drafts: RecordingDraftStep[];
|
||||
startedAt: string;
|
||||
purpose?: string;
|
||||
startUrl?: string;
|
||||
stoppedBy: StopReason;
|
||||
bskVersion: string;
|
||||
includeTabSwitches?: boolean;
|
||||
}): TraceV3 {
|
||||
return buildTraceV3({
|
||||
registry: this.#registry,
|
||||
drafts: input.drafts,
|
||||
annotations: this.#annotations,
|
||||
startedAt: input.startedAt,
|
||||
purpose: input.purpose,
|
||||
startUrl: input.startUrl,
|
||||
stoppedBy: input.stoppedBy,
|
||||
bskVersion: input.bskVersion,
|
||||
redactValues: this.#redactValues,
|
||||
includeTabSwitches: input.includeTabSwitches,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,233 @@
|
||||
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
|
||||
import { type DocumentSettleScope, waitForDocumentSettled } from "./document-settle";
|
||||
import { RecordingObservationSession } from "./observation-session";
|
||||
import type { RecordingDraftStep } from "./types";
|
||||
|
||||
interface PendingSettle {
|
||||
abort: AbortController;
|
||||
scope: DocumentSettleScope;
|
||||
}
|
||||
|
||||
interface PendingRedirect {
|
||||
url: string;
|
||||
abort: AbortController;
|
||||
}
|
||||
|
||||
const CAPTURE_RETRY_DELAY_MS = 250;
|
||||
|
||||
function delay(ms: number, signal?: AbortSignal): Promise<void> {
|
||||
return new Promise((resolve) => {
|
||||
if (signal?.aborted) return resolve();
|
||||
const timer = setTimeout(resolve, ms);
|
||||
signal?.addEventListener(
|
||||
"abort",
|
||||
() => {
|
||||
clearTimeout(timer);
|
||||
resolve();
|
||||
},
|
||||
{ once: true },
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
function isAbortError(error: unknown): boolean {
|
||||
return (
|
||||
typeof error === "object" &&
|
||||
error !== null &&
|
||||
(error as { name?: string }).name === "AbortError"
|
||||
);
|
||||
}
|
||||
|
||||
export function inferMissingPostStates(drafts: RecordingDraftStep[]): void {
|
||||
for (let index = 0; index < drafts.length - 1; index += 1) {
|
||||
const draft = drafts[index];
|
||||
const next = drafts[index + 1];
|
||||
if (draft && next && !draft.postStateId && next.preStateId) {
|
||||
draft.postStateId = next.preStateId;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export class SettleController {
|
||||
readonly #session: RecordingObservationSession;
|
||||
readonly #cdp: CdpRunner;
|
||||
readonly #tabsApi: ChromeTabsApi;
|
||||
readonly #tabId: number;
|
||||
readonly #rootScope: DocumentSettleScope;
|
||||
readonly #pending = new Map<number, PendingSettle>();
|
||||
#queue = Promise.resolve();
|
||||
#pendingRedirect: PendingRedirect | null = null;
|
||||
#redirectQueue = Promise.resolve();
|
||||
|
||||
constructor(input: {
|
||||
session: RecordingObservationSession;
|
||||
cdp: CdpRunner;
|
||||
tabsApi: ChromeTabsApi;
|
||||
tabId: number;
|
||||
}) {
|
||||
this.#session = input.session;
|
||||
this.#cdp = input.cdp;
|
||||
this.#tabsApi = input.tabsApi;
|
||||
this.#tabId = input.tabId;
|
||||
this.#rootScope = { target: { tabId: input.tabId } };
|
||||
}
|
||||
|
||||
get hasPending(): boolean {
|
||||
return this.#pending.size > 0 || this.#pendingRedirect !== null;
|
||||
}
|
||||
|
||||
async #captureWithRetry(signal?: AbortSignal) {
|
||||
try {
|
||||
return await this.#session.capture(this.#cdp, this.#tabsApi, this.#tabId, signal);
|
||||
} catch (error) {
|
||||
if (signal?.aborted) throw error;
|
||||
await delay(CAPTURE_RETRY_DELAY_MS, signal);
|
||||
return this.#session.capture(this.#cdp, this.#tabsApi, this.#tabId, signal);
|
||||
}
|
||||
}
|
||||
|
||||
schedule(
|
||||
drafts: RecordingDraftStep[],
|
||||
draftIndex: number,
|
||||
scope: DocumentSettleScope = this.#rootScope,
|
||||
): void {
|
||||
if (scope.target.tabId !== this.#tabId) {
|
||||
throw new Error(
|
||||
`settle scope tab ${scope.target.tabId} does not belong to tab ${this.#tabId}`,
|
||||
);
|
||||
}
|
||||
const landing = drafts[draftIndex]?.preStateId;
|
||||
for (const [index, pending] of this.#pending) {
|
||||
if (index >= draftIndex) continue;
|
||||
pending.abort.abort();
|
||||
this.#pending.delete(index);
|
||||
if (landing && drafts[index] && !drafts[index].postStateId) {
|
||||
drafts[index].postStateId = landing;
|
||||
}
|
||||
}
|
||||
|
||||
this.#pending.get(draftIndex)?.abort.abort();
|
||||
const pending = { abort: new AbortController(), scope };
|
||||
this.#pending.set(draftIndex, pending);
|
||||
const task = async () => {
|
||||
if (pending.abort.signal.aborted) return;
|
||||
try {
|
||||
const outcome = await waitForDocumentSettled(this.#cdp, pending.scope, {
|
||||
signal: pending.abort.signal,
|
||||
});
|
||||
if (outcome === "cancelled") return;
|
||||
const observation = await this.#captureWithRetry(pending.abort.signal);
|
||||
if (!pending.abort.signal.aborted && drafts[draftIndex]) {
|
||||
drafts[draftIndex].postStateId = observation.stateId;
|
||||
}
|
||||
} catch (error) {
|
||||
if (!isAbortError(error)) {
|
||||
console.warn(
|
||||
`[bsk record] post-action observation failed for step ${draftIndex + 1}`,
|
||||
error,
|
||||
);
|
||||
}
|
||||
} finally {
|
||||
if (this.#pending.get(draftIndex) === pending) this.#pending.delete(draftIndex);
|
||||
}
|
||||
};
|
||||
this.#queue = this.#queue.then(task, task).catch(() => {});
|
||||
}
|
||||
|
||||
cancel(): void {
|
||||
for (const pending of this.#pending.values()) pending.abort.abort();
|
||||
this.#pending.clear();
|
||||
this.clearRedirect();
|
||||
}
|
||||
|
||||
async flush(): Promise<void> {
|
||||
for (let round = 0; round < 10; round += 1) {
|
||||
const current = this.#queue;
|
||||
await current;
|
||||
if (current === this.#queue) return;
|
||||
}
|
||||
console.warn("[bsk record] settle queue kept growing; using completed observations");
|
||||
}
|
||||
|
||||
async settleTrailing(drafts: RecordingDraftStep[]): Promise<void> {
|
||||
const trailing: RecordingDraftStep[] = [];
|
||||
for (let index = drafts.length - 1; index >= 0; index -= 1) {
|
||||
const draft = drafts[index];
|
||||
if (!draft || draft.postStateId) break;
|
||||
trailing.push(draft);
|
||||
}
|
||||
if (trailing.length === 0) return;
|
||||
try {
|
||||
const observation = await this.#captureWithRetry();
|
||||
for (const draft of trailing) draft.postStateId = observation.stateId;
|
||||
} catch (error) {
|
||||
console.warn("[bsk record] final observation at stop failed", error);
|
||||
}
|
||||
}
|
||||
|
||||
clearRedirect(): void {
|
||||
this.#pendingRedirect?.abort.abort();
|
||||
this.#pendingRedirect = null;
|
||||
}
|
||||
|
||||
scheduleRedirect(drafts: RecordingDraftStep[], url: string): void {
|
||||
this.#pendingRedirect?.abort.abort();
|
||||
const pending: PendingRedirect = { url, abort: new AbortController() };
|
||||
this.#pendingRedirect = pending;
|
||||
const task = () => this.#settleRedirect(drafts, pending);
|
||||
this.#redirectQueue = this.#redirectQueue.then(task, task).catch(() => {});
|
||||
}
|
||||
|
||||
async #settleRedirect(drafts: RecordingDraftStep[], pending: PendingRedirect): Promise<void> {
|
||||
const outcome = await waitForDocumentSettled(this.#cdp, this.#rootScope, {
|
||||
signal: pending.abort.signal,
|
||||
});
|
||||
if (outcome === "cancelled" || this.#pendingRedirect !== pending) return;
|
||||
|
||||
let finalUrl = pending.url;
|
||||
try {
|
||||
finalUrl = (await this.#tabsApi.get(this.#tabId)).url || finalUrl;
|
||||
} catch {
|
||||
// The navigation event URL remains the best available destination.
|
||||
}
|
||||
if (this.#pendingRedirect !== pending) return;
|
||||
this.#pendingRedirect = null;
|
||||
if (
|
||||
!finalUrl ||
|
||||
finalUrl === "about:blank" ||
|
||||
this.#session.cursor.lastSettled?.url === finalUrl
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
const last = drafts[drafts.length - 1];
|
||||
if (last?.op === "navigate" && last.url === finalUrl) {
|
||||
if (!last.postStateId) this.schedule(drafts, drafts.length - 1);
|
||||
await this.flush();
|
||||
return;
|
||||
}
|
||||
|
||||
const draft: RecordingDraftStep = {
|
||||
op: "navigate",
|
||||
url: finalUrl,
|
||||
pageUrl: finalUrl,
|
||||
cause: "browser",
|
||||
transitionQualifiers: ["server_redirect"],
|
||||
preStateId: this.#session.cursor.lastSettled?.stateId,
|
||||
};
|
||||
drafts.push(draft);
|
||||
const draftIndex = drafts.length - 1;
|
||||
if (draft.preStateId) this.#session.registry.markStep(draft.preStateId, draftIndex + 1);
|
||||
this.schedule(drafts, draftIndex);
|
||||
await this.flush();
|
||||
}
|
||||
|
||||
async flushRedirects(): Promise<void> {
|
||||
for (let round = 0; round < 10; round += 1) {
|
||||
const current = this.#redirectQueue;
|
||||
await current;
|
||||
if (current === this.#redirectQueue) return;
|
||||
}
|
||||
console.warn("[bsk record] redirect queue kept growing; using the final completed landing");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
export interface RecordedStateEntry {
|
||||
id: string;
|
||||
url: string;
|
||||
title?: string;
|
||||
/** Rendered VOM text; capture indexes and source DOM/AX data never enter the registry. */
|
||||
vomText: string;
|
||||
truncated: boolean;
|
||||
stepsHere: number[];
|
||||
}
|
||||
|
||||
function stateIdentity(url: string, body: string): string {
|
||||
return `${url}\0${body}`;
|
||||
}
|
||||
|
||||
export class RecordingStateRegistry {
|
||||
readonly #entriesById = new Map<string, RecordedStateEntry>();
|
||||
readonly #idByIdentity = new Map<string, string>();
|
||||
#nextId = 1;
|
||||
|
||||
register(input: {
|
||||
url: string;
|
||||
title?: string;
|
||||
vomText: string;
|
||||
truncated?: boolean;
|
||||
}): RecordedStateEntry {
|
||||
const identity = stateIdentity(input.url, input.vomText);
|
||||
const existingId = this.#idByIdentity.get(identity);
|
||||
if (existingId) {
|
||||
const existing = this.#entriesById.get(existingId)!;
|
||||
if (!existing.title && input.title) existing.title = input.title;
|
||||
if (input.truncated) existing.truncated = true;
|
||||
return existing;
|
||||
}
|
||||
|
||||
const entry: RecordedStateEntry = {
|
||||
id: `s${this.#nextId}`,
|
||||
url: input.url,
|
||||
...(input.title ? { title: input.title } : {}),
|
||||
vomText: input.vomText,
|
||||
truncated: input.truncated ?? false,
|
||||
stepsHere: [],
|
||||
};
|
||||
this.#nextId += 1;
|
||||
this.#entriesById.set(entry.id, entry);
|
||||
this.#idByIdentity.set(identity, entry.id);
|
||||
return entry;
|
||||
}
|
||||
|
||||
values(): RecordedStateEntry[] {
|
||||
return [...this.#entriesById.values()];
|
||||
}
|
||||
|
||||
markStep(stateId: string, draftId: number): void {
|
||||
const entry = this.#entriesById.get(stateId);
|
||||
if (entry && !entry.stepsHere.includes(draftId)) entry.stepsHere.push(draftId);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
import type { RecordStepPayload } from "../record-bridge";
|
||||
import { hasRedirectQualifier } from "./navigation-policy";
|
||||
import type { RecordingDraftStep, TargetMatchHint } from "./types";
|
||||
|
||||
export interface RecordingStepBuffer {
|
||||
steps: RecordingDraftStep[];
|
||||
navigation: RecordingNavigationCursor;
|
||||
}
|
||||
|
||||
export interface RecordingNavigationCursor {
|
||||
currentUrl?: string;
|
||||
pendingNavigation: boolean;
|
||||
pendingNavigationDeadline?: number;
|
||||
}
|
||||
|
||||
const NAVIGATION_TRIGGER_WINDOW_MS = 3_000;
|
||||
|
||||
function toDraftStep(
|
||||
payload: RecordStepPayload,
|
||||
targetHint?: TargetMatchHint,
|
||||
): RecordingDraftStep | null {
|
||||
const pageUrl = payload.page_url;
|
||||
const common = {
|
||||
...(pageUrl ? { pageUrl } : {}),
|
||||
...(targetHint ? { targetHint } : {}),
|
||||
};
|
||||
switch (payload.op) {
|
||||
case "click":
|
||||
return payload.target ? { op: "click", captureTarget: payload.target, ...common } : null;
|
||||
case "hover":
|
||||
return payload.target ? { op: "hover", captureTarget: payload.target, ...common } : null;
|
||||
case "fill":
|
||||
return payload.target
|
||||
? {
|
||||
op: "fill",
|
||||
captureTarget: payload.target,
|
||||
value: payload.value ?? "",
|
||||
...(payload.commit ? { commit: payload.commit } : {}),
|
||||
...(payload.redacted ? { redacted: true } : {}),
|
||||
...common,
|
||||
}
|
||||
: null;
|
||||
case "press":
|
||||
return payload.key
|
||||
? {
|
||||
op: "press",
|
||||
key: payload.key,
|
||||
...(payload.target ? { captureTarget: payload.target } : {}),
|
||||
...(payload.modifiers?.length ? { modifiers: payload.modifiers } : {}),
|
||||
...common,
|
||||
}
|
||||
: null;
|
||||
case "select":
|
||||
return payload.target && payload.values
|
||||
? {
|
||||
op: "select",
|
||||
captureTarget: payload.target,
|
||||
values: payload.values,
|
||||
...(payload.labels?.length ? { labels: payload.labels } : {}),
|
||||
...common,
|
||||
}
|
||||
: null;
|
||||
case "navigate":
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function annotateLastStepNavigation(buffer: RecordingStepBuffer, url: string): number {
|
||||
for (let index = buffer.steps.length - 1; index >= 0; index -= 1) {
|
||||
const step = buffer.steps[index];
|
||||
if (!step) continue;
|
||||
if (step.op === "click" || step.op === "press" || step.op === "select" || step.op === "fill") {
|
||||
step.navigatedTo = url;
|
||||
return index;
|
||||
}
|
||||
break;
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
export type NavigationObserveResult =
|
||||
| { kind: "noop" }
|
||||
| { kind: "annotated"; index: number }
|
||||
| { kind: "appended"; index: number }
|
||||
| { kind: "coalesce_redirect"; url: string };
|
||||
|
||||
export function observeRecordedNavigation(
|
||||
buffer: RecordingStepBuffer,
|
||||
url: string,
|
||||
causedByAction?: boolean,
|
||||
transitionType?: string,
|
||||
transitionQualifiers?: string[],
|
||||
): NavigationObserveResult {
|
||||
const navigation = buffer.navigation;
|
||||
if (!url || url === navigation.currentUrl) return { kind: "noop" };
|
||||
navigation.currentUrl = url;
|
||||
|
||||
const pendingIsCurrent =
|
||||
navigation.pendingNavigation &&
|
||||
(navigation.pendingNavigationDeadline === undefined ||
|
||||
navigation.pendingNavigationDeadline >= Date.now());
|
||||
|
||||
if (causedByAction === true || (causedByAction === undefined && pendingIsCurrent)) {
|
||||
navigation.pendingNavigation = false;
|
||||
navigation.pendingNavigationDeadline = undefined;
|
||||
const annotatedIndex = annotateLastStepNavigation(buffer, url);
|
||||
if (annotatedIndex >= 0) return { kind: "annotated", index: annotatedIndex };
|
||||
} else {
|
||||
navigation.pendingNavigation = false;
|
||||
navigation.pendingNavigationDeadline = undefined;
|
||||
}
|
||||
|
||||
if (hasRedirectQualifier(transitionQualifiers)) {
|
||||
return { kind: "coalesce_redirect", url };
|
||||
}
|
||||
buffer.steps.push({
|
||||
op: "navigate",
|
||||
url,
|
||||
pageUrl: url,
|
||||
transitionType,
|
||||
transitionQualifiers,
|
||||
});
|
||||
return { kind: "appended", index: buffer.steps.length - 1 };
|
||||
}
|
||||
|
||||
export function appendRecordedPayload(
|
||||
buffer: RecordingStepBuffer,
|
||||
payload: RecordStepPayload,
|
||||
targetHint?: TargetMatchHint,
|
||||
): number | null {
|
||||
if (payload.op === "navigate") {
|
||||
if (!payload.url) return null;
|
||||
const result = observeRecordedNavigation(
|
||||
buffer,
|
||||
payload.url,
|
||||
payload.navigation_caused_by_action,
|
||||
payload.transitionType,
|
||||
payload.transitionQualifiers,
|
||||
);
|
||||
return result.kind === "appended" ? result.index : null;
|
||||
}
|
||||
const step = toDraftStep(
|
||||
{ ...payload, page_url: payload.page_url ?? buffer.navigation.currentUrl },
|
||||
targetHint,
|
||||
);
|
||||
if (!step) return null;
|
||||
buffer.steps.push(step);
|
||||
if (step.op === "click" || step.op === "press" || step.op === "select" || step.op === "fill") {
|
||||
buffer.navigation.pendingNavigation = payload.expects_navigation === true;
|
||||
buffer.navigation.pendingNavigationDeadline = buffer.navigation.pendingNavigation
|
||||
? Date.now() + NAVIGATION_TRIGGER_WINDOW_MS
|
||||
: undefined;
|
||||
}
|
||||
return buffer.steps.length - 1;
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
import type { RecordingNavigationCursor } from "./step-buffer";
|
||||
|
||||
export interface TabActivation {
|
||||
tabId: number;
|
||||
revision: number;
|
||||
}
|
||||
|
||||
export class RecordingTabCoordinator {
|
||||
readonly #navigationByTab = new Map<number, RecordingNavigationCursor>();
|
||||
#activeTabId: number;
|
||||
#currentTabId: number;
|
||||
#activationRevision = 0;
|
||||
|
||||
constructor(initialTabId: number, initialUrl?: string) {
|
||||
this.#activeTabId = initialTabId;
|
||||
this.#currentTabId = initialTabId;
|
||||
this.#navigationByTab.set(initialTabId, {
|
||||
currentUrl: initialUrl,
|
||||
pendingNavigation: false,
|
||||
});
|
||||
}
|
||||
|
||||
get activeTabId(): number {
|
||||
return this.#activeTabId;
|
||||
}
|
||||
|
||||
get currentTabId(): number {
|
||||
return this.#currentTabId;
|
||||
}
|
||||
|
||||
noteActivation(tabId: number): TabActivation {
|
||||
this.#activeTabId = tabId;
|
||||
this.#activationRevision += 1;
|
||||
return { tabId, revision: this.#activationRevision };
|
||||
}
|
||||
|
||||
isLatest(activation: TabActivation): boolean {
|
||||
return (
|
||||
activation.tabId === this.#activeTabId && activation.revision === this.#activationRevision
|
||||
);
|
||||
}
|
||||
|
||||
commit(tabId: number, currentUrl?: string): void {
|
||||
if (currentUrl !== undefined) this.navigation(tabId).currentUrl = currentUrl;
|
||||
this.#currentTabId = tabId;
|
||||
}
|
||||
|
||||
navigation(tabId: number, fallbackUrl?: string): RecordingNavigationCursor {
|
||||
const existing = this.#navigationByTab.get(tabId);
|
||||
if (existing) {
|
||||
if (!existing.currentUrl && fallbackUrl) existing.currentUrl = fallbackUrl;
|
||||
return existing;
|
||||
}
|
||||
const created: RecordingNavigationCursor = {
|
||||
currentUrl: fallbackUrl,
|
||||
pendingNavigation: false,
|
||||
};
|
||||
this.#navigationByTab.set(tabId, created);
|
||||
return created;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
import type { RenderedRef } from "@browser-skill/vom";
|
||||
import type { TargetDescriptorV3 } from "@/transport/types";
|
||||
import type { CaptureTargetDescriptor } from "../describe-target";
|
||||
import type { IndexedObservationNode, RegisteredObservation } from "./observation-capture";
|
||||
import type { TargetGeometry, TargetMatchHint } from "./types";
|
||||
|
||||
const MATCH_TOLERANCE_PX = 2;
|
||||
|
||||
function close(a: number, b: number): boolean {
|
||||
return Math.abs(a - b) <= MATCH_TOLERANCE_PX;
|
||||
}
|
||||
|
||||
function rectMatches(a: TargetGeometry["rect"], b: TargetGeometry["rect"]): boolean {
|
||||
return close(a.x, b.x) && close(a.y, b.y) && close(a.w, b.w) && close(a.h, b.h);
|
||||
}
|
||||
|
||||
function candidateMatches(
|
||||
candidate: IndexedObservationNode,
|
||||
geometry: TargetGeometry,
|
||||
geometrySpace: "top" | "local",
|
||||
): boolean {
|
||||
const rect = geometrySpace === "local" ? candidate.geometry.localRect : candidate.geometry.rect;
|
||||
return rect != null && rectMatches(geometry.rect, rect);
|
||||
}
|
||||
|
||||
export function unmatchedTarget(fallback?: CaptureTargetDescriptor): TargetDescriptorV3 {
|
||||
return {
|
||||
...(fallback?.role ? { role: fallback.role } : {}),
|
||||
...(fallback?.name ? { name: fallback.name } : {}),
|
||||
unmatched: true,
|
||||
};
|
||||
}
|
||||
|
||||
function normalized(value: string | undefined): string {
|
||||
return value?.replace(/\s+/g, " ").trim().toLowerCase() ?? "";
|
||||
}
|
||||
|
||||
function matchesSemantics(ref: RenderedRef, fallback?: CaptureTargetDescriptor): boolean {
|
||||
const role = normalized(fallback?.role);
|
||||
const name = normalized(fallback?.name);
|
||||
if (!role && !name) return false;
|
||||
if (role && normalized(ref.role) !== role) return false;
|
||||
if (name && normalized(ref.name) !== name) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
function descriptor(ref: RenderedRef): TargetDescriptorV3 {
|
||||
return {
|
||||
ref: ref.ref,
|
||||
...(ref.role ? { role: ref.role } : {}),
|
||||
...(ref.name ? { name: ref.name } : {}),
|
||||
...(ref.ctx ? { ctx: ref.ctx } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
export function matchObservationTarget(input: {
|
||||
observation: RegisteredObservation;
|
||||
hint?: TargetMatchHint;
|
||||
fallback?: CaptureTargetDescriptor;
|
||||
}): TargetDescriptorV3 {
|
||||
if (input.hint?.frameId === null) return unmatchedTarget(input.fallback);
|
||||
const frameId = input.hint?.frameId ?? input.observation.rootFrameId;
|
||||
const geometry = input.hint?.geometry;
|
||||
if (geometry) {
|
||||
const matches = input.observation.index
|
||||
.candidates(frameId, geometry.tag)
|
||||
.filter(
|
||||
(candidate) =>
|
||||
candidate.ref &&
|
||||
candidateMatches(candidate, geometry, input.hint?.geometrySpace ?? "top"),
|
||||
);
|
||||
if (matches.length === 1) return descriptor(matches[0]!.ref!);
|
||||
const semanticMatches = matches.filter(
|
||||
(candidate) => candidate.ref && matchesSemantics(candidate.ref, input.fallback),
|
||||
);
|
||||
if (semanticMatches.length === 1) return descriptor(semanticMatches[0]!.ref!);
|
||||
return unmatchedTarget(input.fallback);
|
||||
}
|
||||
|
||||
const semanticMatches = input.observation.index
|
||||
.refs(frameId)
|
||||
.filter((ref) => matchesSemantics(ref, input.fallback));
|
||||
return semanticMatches.length === 1
|
||||
? descriptor(semanticMatches[0]!)
|
||||
: unmatchedTarget(input.fallback);
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
import {
|
||||
type StepV3,
|
||||
type StopReason,
|
||||
TRACE_VERSION_V3,
|
||||
type TraceStateV3,
|
||||
type TraceV3,
|
||||
VOM_FORMAT_VERSION,
|
||||
} from "@/transport/types";
|
||||
import { resolveDraftStartUrl } from "./draft-policy";
|
||||
import type { RecordedStateEntry, RecordingStateRegistry } from "./state-registry";
|
||||
import { reduceTraceStepsV3 } from "./trace-reducer-v3";
|
||||
import { formatTraceStateBody } from "./trace-state-body";
|
||||
import type { RecordingDraftStep, StepAnnotation } from "./types";
|
||||
|
||||
function publishedEntries(registry: RecordingStateRegistry, steps: StepV3[]): RecordedStateEntry[] {
|
||||
const entries = registry.values();
|
||||
if (steps.length === 0) return entries.slice(0, 1);
|
||||
const referenced = new Set(steps.flatMap((step) => [step.state, step.result.state]));
|
||||
return entries.filter((entry) => referenced.has(entry.id));
|
||||
}
|
||||
|
||||
function remapDraftIds(draftIds: number[], stepIdByDraftId: Map<number, number>): number[] {
|
||||
return [
|
||||
...new Set(
|
||||
draftIds.flatMap((id) => {
|
||||
const stepId = stepIdByDraftId.get(id);
|
||||
return stepId === undefined ? [] : [stepId];
|
||||
}),
|
||||
),
|
||||
].sort((a, b) => a - b);
|
||||
}
|
||||
|
||||
export function buildTraceV3(input: {
|
||||
registry: RecordingStateRegistry;
|
||||
drafts: RecordingDraftStep[];
|
||||
annotations?: StepAnnotation[];
|
||||
startedAt: string;
|
||||
purpose?: string;
|
||||
startUrl?: string;
|
||||
stoppedBy: StopReason;
|
||||
bskVersion: string;
|
||||
redactValues?: boolean;
|
||||
includeTabSwitches?: boolean;
|
||||
}): TraceV3 {
|
||||
const reduced = reduceTraceStepsV3(input.drafts, {
|
||||
includeTabSwitches: input.includeTabSwitches,
|
||||
redactValues: input.redactValues,
|
||||
});
|
||||
const entries = publishedEntries(input.registry, reduced.steps);
|
||||
const publishedId = new Map(entries.map((entry, index) => [entry.id, `s${index + 1}`]));
|
||||
const annotationsByState = new Map<string, StepAnnotation[]>();
|
||||
for (const annotation of input.annotations ?? []) {
|
||||
const bucket = annotationsByState.get(annotation.stateId) ?? [];
|
||||
bucket.push(annotation);
|
||||
annotationsByState.set(annotation.stateId, bucket);
|
||||
}
|
||||
const steps = reduced.steps.map((step) => ({
|
||||
...step,
|
||||
state: publishedId.get(step.state) ?? step.state,
|
||||
result: { state: publishedId.get(step.result.state) ?? step.result.state },
|
||||
}));
|
||||
const states: TraceStateV3[] = entries.map((entry) => {
|
||||
const id = publishedId.get(entry.id) ?? entry.id;
|
||||
return {
|
||||
id,
|
||||
url: entry.url,
|
||||
...(entry.title ? { title: entry.title } : {}),
|
||||
body: formatTraceStateBody({
|
||||
stateId: id,
|
||||
url: entry.url,
|
||||
title: entry.title,
|
||||
stepIds: remapDraftIds(entry.stepsHere, reduced.stepIdByDraftId),
|
||||
vomText: entry.vomText,
|
||||
annotations: annotationsByState.get(entry.id) ?? [],
|
||||
stepIdByDraftId: reduced.stepIdByDraftId,
|
||||
}),
|
||||
...(entry.truncated ? { truncated: true } : {}),
|
||||
};
|
||||
});
|
||||
|
||||
return {
|
||||
version: TRACE_VERSION_V3,
|
||||
...(input.purpose ? { purpose: input.purpose } : {}),
|
||||
recorded_at: new Date().toISOString(),
|
||||
started_at: input.startedAt,
|
||||
stopped_by: input.stoppedBy,
|
||||
entry: { start_url: resolveDraftStartUrl(input.drafts, input.startUrl, states[0]?.url) },
|
||||
recorder: { bsk: input.bskVersion, vom: VOM_FORMAT_VERSION },
|
||||
states,
|
||||
steps,
|
||||
};
|
||||
}
|
||||
+62
-72
@@ -1,31 +1,10 @@
|
||||
import type { DraftTraceStep, PageRef, SelectedOption, Step } from "@/transport/types";
|
||||
|
||||
const CLIPBOARD_KEYS = new Set(["a", "c", "v", "x", "A", "C", "V", "X"]);
|
||||
const MODIFIER_ONLY_KEYS = new Set(["Meta", "Control", "Alt", "Shift", "OS", "Hyper", "Super"]);
|
||||
|
||||
export function shouldRecordPress(
|
||||
key: string,
|
||||
modifiers?: Array<"alt" | "ctrl" | "meta" | "shift">,
|
||||
): boolean {
|
||||
if (MODIFIER_ONLY_KEYS.has(key)) return false;
|
||||
const mods = modifiers ?? [];
|
||||
const hasCtrlOrMeta = mods.includes("ctrl") || mods.includes("meta");
|
||||
if (hasCtrlOrMeta && CLIPBOARD_KEYS.has(key)) return false;
|
||||
if (key === "Enter" || key === "Escape") return true;
|
||||
// Drop bare character typing — FillSession already records the value.
|
||||
if (key.length === 1 && !hasCtrlOrMeta && !mods.includes("alt")) return false;
|
||||
return false;
|
||||
}
|
||||
|
||||
function shouldIncludeDraft(step: DraftTraceStep): boolean {
|
||||
if (step.op === "fill" && !(step.value ?? "").trim() && !step.redacted) return false;
|
||||
if (step.op === "press" && !shouldRecordPress(step.key, step.modifiers)) return false;
|
||||
return true;
|
||||
}
|
||||
import type { PageRefV2, SelectedOptionV2, StepV2, TraceV2 } from "@/transport/types";
|
||||
import { resolveDraftStartUrl, shouldIncludeDraft } from "./draft-policy";
|
||||
import type { RecordingDraftStep } from "./types";
|
||||
|
||||
/** Collapse consecutive navigations to the last hop. */
|
||||
function collapseNavigations(steps: DraftTraceStep[]): DraftTraceStep[] {
|
||||
const out: DraftTraceStep[] = [];
|
||||
function collapseNavigations(steps: RecordingDraftStep[]): RecordingDraftStep[] {
|
||||
const out: RecordingDraftStep[] = [];
|
||||
for (const step of steps) {
|
||||
const prev = out[out.length - 1];
|
||||
if (step.op === "navigate" && prev?.op === "navigate") {
|
||||
@@ -37,7 +16,7 @@ function collapseNavigations(steps: DraftTraceStep[]): DraftTraceStep[] {
|
||||
return out;
|
||||
}
|
||||
|
||||
function collectUrls(steps: DraftTraceStep[], startUrl?: string): string[] {
|
||||
function collectUrls(steps: RecordingDraftStep[], startUrl?: string): string[] {
|
||||
const urls: string[] = [];
|
||||
if (startUrl) urls.push(startUrl);
|
||||
for (const step of steps) {
|
||||
@@ -45,8 +24,8 @@ function collectUrls(steps: DraftTraceStep[], startUrl?: string): string[] {
|
||||
urls.push(step.url);
|
||||
continue;
|
||||
}
|
||||
if ("page_url" in step && step.page_url) urls.push(step.page_url);
|
||||
if ("navigated_to" in step && step.navigated_to) urls.push(step.navigated_to);
|
||||
if (step.pageUrl) urls.push(step.pageUrl);
|
||||
if ("navigatedTo" in step && step.navigatedTo) urls.push(step.navigatedTo);
|
||||
}
|
||||
const seen = new Set<string>();
|
||||
const unique: string[] = [];
|
||||
@@ -59,9 +38,9 @@ function collectUrls(steps: DraftTraceStep[], startUrl?: string): string[] {
|
||||
}
|
||||
|
||||
function buildPageRegistry(
|
||||
steps: DraftTraceStep[],
|
||||
steps: RecordingDraftStep[],
|
||||
startUrl?: string,
|
||||
): { pages: PageRef[]; urlToId: Map<string, string> } {
|
||||
): { pages: PageRefV2[]; urlToId: Map<string, string> } {
|
||||
const urls = collectUrls(steps, startUrl);
|
||||
const urlToId = new Map<string, string>();
|
||||
const pages = urls.map((url, index) => {
|
||||
@@ -82,28 +61,28 @@ function pageIdFor(
|
||||
return urlToId.values().next().value ?? "p1";
|
||||
}
|
||||
|
||||
function pageUrlForDraft(step: DraftTraceStep, fallbackUrl?: string): string | undefined {
|
||||
if (step.op === "navigate") return step.page_url ?? step.url;
|
||||
if ("page_url" in step && step.page_url) return step.page_url;
|
||||
function pageUrlForDraft(step: RecordingDraftStep, fallbackUrl?: string): string | undefined {
|
||||
if (step.op === "navigate") return step.pageUrl ?? step.url;
|
||||
if (step.pageUrl) return step.pageUrl;
|
||||
return fallbackUrl;
|
||||
}
|
||||
|
||||
function effectForNavigation(
|
||||
navigatedTo: string | undefined,
|
||||
urlToId: Map<string, string>,
|
||||
): Step["effect"] {
|
||||
): StepV2["effect"] {
|
||||
if (!navigatedTo) return undefined;
|
||||
const pageId = urlToId.get(navigatedTo);
|
||||
if (!pageId) return undefined;
|
||||
return { navigated_to: pageId };
|
||||
}
|
||||
|
||||
function withEffect(step: Step, effect: Step["effect"]): Step {
|
||||
function withEffect(step: StepV2, effect: StepV2["effect"]): StepV2 {
|
||||
if (!effect) return step;
|
||||
return { ...step, effect };
|
||||
}
|
||||
|
||||
function toSelection(values: string[], labels?: string[]): SelectedOption[] {
|
||||
function toSelection(values: string[], labels?: string[]): SelectedOptionV2[] {
|
||||
return values.map((value, index) => ({
|
||||
value,
|
||||
...(labels?.[index] ? { label: labels[index] } : {}),
|
||||
@@ -111,11 +90,11 @@ function toSelection(values: string[], labels?: string[]): SelectedOption[] {
|
||||
}
|
||||
|
||||
function toV2Step(
|
||||
step: DraftTraceStep,
|
||||
step: RecordingDraftStep,
|
||||
id: number,
|
||||
urlToId: Map<string, string>,
|
||||
fallbackUrl?: string,
|
||||
): Step | null {
|
||||
): StepV2 | null {
|
||||
if (!shouldIncludeDraft(step)) return null;
|
||||
|
||||
const pageUrl = pageUrlForDraft(step, fallbackUrl);
|
||||
@@ -130,28 +109,23 @@ function toV2Step(
|
||||
to: step.url,
|
||||
};
|
||||
case "click":
|
||||
if (!step.captureTarget) return null;
|
||||
return withEffect(
|
||||
{
|
||||
op: "click",
|
||||
id,
|
||||
page,
|
||||
target: step.target,
|
||||
target: step.captureTarget,
|
||||
},
|
||||
effectForNavigation(step.navigated_to, urlToId),
|
||||
effectForNavigation(step.navigatedTo, urlToId),
|
||||
);
|
||||
case "hover":
|
||||
return {
|
||||
op: "hover",
|
||||
id,
|
||||
page,
|
||||
target: step.target,
|
||||
};
|
||||
case "fill":
|
||||
if (!step.captureTarget) return null;
|
||||
return {
|
||||
op: "fill",
|
||||
id,
|
||||
page,
|
||||
target: step.target,
|
||||
target: step.captureTarget,
|
||||
value: step.value,
|
||||
...(step.redacted ? { redacted: true } : {}),
|
||||
};
|
||||
@@ -162,44 +136,51 @@ function toV2Step(
|
||||
id,
|
||||
page,
|
||||
key: step.key,
|
||||
...(step.target ? { target: step.target } : {}),
|
||||
...(step.captureTarget ? { target: step.captureTarget } : {}),
|
||||
...(step.modifiers?.length ? { modifiers: step.modifiers } : {}),
|
||||
},
|
||||
effectForNavigation(step.navigated_to, urlToId),
|
||||
effectForNavigation(step.navigatedTo, urlToId),
|
||||
);
|
||||
case "select":
|
||||
if (!step.captureTarget) return null;
|
||||
return withEffect(
|
||||
{
|
||||
op: "select",
|
||||
id,
|
||||
page,
|
||||
target: step.target,
|
||||
target: step.captureTarget,
|
||||
selection: toSelection(step.values, step.labels),
|
||||
},
|
||||
effectForNavigation(step.navigated_to, urlToId),
|
||||
effectForNavigation(step.navigatedTo, urlToId),
|
||||
);
|
||||
// `hover` only exists in Trace v3. Peers that negotiate v2 predate the
|
||||
// step variant and fail to decode the whole result if we emit it.
|
||||
case "hover":
|
||||
case "scroll":
|
||||
case "switch_tab":
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export interface ReducedTrace {
|
||||
pages: PageRef[];
|
||||
steps: Step[];
|
||||
interface ReducedTrace {
|
||||
pages: PageRefV2[];
|
||||
steps: StepV2[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Compile capture drafts into record-only trace v2 steps.
|
||||
* Variable inputs are NOT classified here — executing agents infer that at run time.
|
||||
*/
|
||||
export function reduceTraceSteps(steps: DraftTraceStep[], startUrl?: string): ReducedTrace {
|
||||
function reduceTraceSteps(steps: RecordingDraftStep[], startUrl?: string): ReducedTrace {
|
||||
const collapsed = collapseNavigations(steps);
|
||||
const { pages, urlToId } = buildPageRegistry(collapsed, startUrl);
|
||||
const out: Step[] = [];
|
||||
const out: StepV2[] = [];
|
||||
let id = 1;
|
||||
let lastUrl = startUrl;
|
||||
for (const draft of collapsed) {
|
||||
if (draft.op === "navigate") lastUrl = draft.url;
|
||||
else if ("navigated_to" in draft && draft.navigated_to) lastUrl = draft.navigated_to;
|
||||
else if ("page_url" in draft && draft.page_url) lastUrl = draft.page_url;
|
||||
else if ("navigatedTo" in draft && draft.navigatedTo) lastUrl = draft.navigatedTo;
|
||||
else if (draft.pageUrl) lastUrl = draft.pageUrl;
|
||||
const step = toV2Step(draft, id, urlToId, lastUrl);
|
||||
if (!step) continue;
|
||||
out.push(step);
|
||||
@@ -208,18 +189,27 @@ export function reduceTraceSteps(steps: DraftTraceStep[], startUrl?: string): Re
|
||||
return { pages, steps: out };
|
||||
}
|
||||
|
||||
export function resolveTraceStartUrl(
|
||||
drafts: DraftTraceStep[],
|
||||
function resolveTraceStartUrl(
|
||||
drafts: RecordingDraftStep[],
|
||||
startUrl?: string,
|
||||
pages?: PageRef[],
|
||||
pages?: PageRefV2[],
|
||||
): string {
|
||||
if (startUrl) return startUrl;
|
||||
const navigate = drafts.find((step): step is Extract<DraftTraceStep, { op: "navigate" }> => {
|
||||
return step.op === "navigate";
|
||||
});
|
||||
if (navigate) return navigate.url;
|
||||
for (const draft of drafts) {
|
||||
if ("page_url" in draft && draft.page_url) return draft.page_url;
|
||||
}
|
||||
return pages?.[0]?.url ?? "about:blank";
|
||||
return resolveDraftStartUrl(drafts, startUrl, pages?.[0]?.url);
|
||||
}
|
||||
|
||||
export function buildTraceV2(input: {
|
||||
steps: RecordingDraftStep[];
|
||||
startedAt: string;
|
||||
startUrl?: string;
|
||||
purpose?: string;
|
||||
}): TraceV2 {
|
||||
const { pages, steps } = reduceTraceSteps(input.steps, input.startUrl);
|
||||
return {
|
||||
recorded_at: new Date().toISOString(),
|
||||
started_at: input.startedAt,
|
||||
...(input.purpose ? { purpose: input.purpose } : {}),
|
||||
entry: { start_url: resolveTraceStartUrl(input.steps, input.startUrl, pages) },
|
||||
pages,
|
||||
steps,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,154 @@
|
||||
import type { NavigationCause, StepV3 } from "@/transport/types";
|
||||
import { shouldIncludeDraft } from "./draft-policy";
|
||||
import { hasRedirectQualifier } from "./navigation-policy";
|
||||
import { unmatchedTarget } from "./target-matcher";
|
||||
import type { RecordingDraftStep } from "./types";
|
||||
|
||||
interface CollapsedDraft {
|
||||
draft: RecordingDraftStep;
|
||||
draftIds: number[];
|
||||
}
|
||||
|
||||
const TRANSITION_CAUSES: Record<string, NavigationCause> = {
|
||||
typed: "user_typed",
|
||||
generated: "user_typed",
|
||||
keyword: "user_typed",
|
||||
keyword_generated: "user_typed",
|
||||
link: "link",
|
||||
form_submit: "form_submit",
|
||||
reload: "reload",
|
||||
auto_bookmark: "browser",
|
||||
start_page: "browser",
|
||||
};
|
||||
|
||||
function isRedirect(step: Extract<RecordingDraftStep, { op: "navigate" }>): boolean {
|
||||
return hasRedirectQualifier(step.transitionQualifiers);
|
||||
}
|
||||
|
||||
function collapseRedirects(steps: RecordingDraftStep[]): CollapsedDraft[] {
|
||||
const output: CollapsedDraft[] = [];
|
||||
steps.forEach((step, index) => {
|
||||
const previous = output[output.length - 1];
|
||||
if (step.op === "navigate" && previous?.draft.op === "navigate" && isRedirect(step)) {
|
||||
previous.draft = {
|
||||
...previous.draft,
|
||||
url: step.url,
|
||||
postStateId: step.postStateId ?? previous.draft.postStateId,
|
||||
};
|
||||
previous.draftIds.push(index + 1);
|
||||
return;
|
||||
}
|
||||
output.push({ draft: { ...step }, draftIds: [index + 1] });
|
||||
});
|
||||
return output;
|
||||
}
|
||||
|
||||
function navigationCause(step: Extract<RecordingDraftStep, { op: "navigate" }>): NavigationCause {
|
||||
if (step.cause) return step.cause;
|
||||
const qualifiers = step.transitionQualifiers ?? [];
|
||||
if (qualifiers.includes("forward_back")) return "history";
|
||||
if (qualifiers.includes("from_address_bar")) return "user_typed";
|
||||
return TRANSITION_CAUSES[step.transitionType ?? ""] ?? "browser";
|
||||
}
|
||||
|
||||
function selection(values: string[], labels?: string[]): Array<{ value: string; label?: string }> {
|
||||
return values.map((value, index) => ({
|
||||
value,
|
||||
...(labels?.[index] ? { label: labels[index] } : {}),
|
||||
}));
|
||||
}
|
||||
|
||||
interface ReduceDraftOptions {
|
||||
includeTabSwitches: boolean;
|
||||
redactValues: boolean;
|
||||
}
|
||||
|
||||
function reduceDraft(
|
||||
draft: RecordingDraftStep,
|
||||
id: number,
|
||||
options: ReduceDraftOptions,
|
||||
): StepV3 | null {
|
||||
if (!shouldIncludeDraft(draft)) return null;
|
||||
if (draft.op === "switch_tab") {
|
||||
if (!options.includeTabSwitches || !draft.preStateId || !draft.postStateId) return null;
|
||||
return {
|
||||
op: "switch_tab",
|
||||
id,
|
||||
state: draft.preStateId,
|
||||
result: { state: draft.postStateId },
|
||||
};
|
||||
}
|
||||
const state = draft.preStateId ?? draft.postStateId;
|
||||
const resultState = draft.postStateId ?? draft.preStateId;
|
||||
if (!state || !resultState) return null;
|
||||
const common = { id, state, result: { state: resultState } };
|
||||
|
||||
switch (draft.op) {
|
||||
case "navigate":
|
||||
return { op: "navigate", ...common, to: draft.url, cause: navigationCause(draft) };
|
||||
case "click":
|
||||
return {
|
||||
op: "click",
|
||||
...common,
|
||||
target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget),
|
||||
};
|
||||
case "hover":
|
||||
return {
|
||||
op: "hover",
|
||||
...common,
|
||||
target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget),
|
||||
};
|
||||
case "fill":
|
||||
const fillIsRedacted = options.redactValues || draft.redacted === true;
|
||||
return {
|
||||
op: "fill",
|
||||
...common,
|
||||
target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget),
|
||||
value: fillIsRedacted ? "***" : draft.value,
|
||||
commit: draft.commit ?? "blur",
|
||||
...(fillIsRedacted ? { redacted: true } : {}),
|
||||
};
|
||||
case "press":
|
||||
return {
|
||||
op: "press",
|
||||
...common,
|
||||
key: draft.key,
|
||||
...(draft.captureTarget || draft.matchedTarget
|
||||
? { target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget) }
|
||||
: {}),
|
||||
...(draft.modifiers?.length ? { modifiers: draft.modifiers } : {}),
|
||||
};
|
||||
case "select":
|
||||
return {
|
||||
op: "select",
|
||||
...common,
|
||||
target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget),
|
||||
...(!options.redactValues ? { selection: selection(draft.values, draft.labels) } : {}),
|
||||
};
|
||||
case "scroll":
|
||||
return { op: "scroll", ...common };
|
||||
}
|
||||
}
|
||||
|
||||
export interface ReducedTraceV3 {
|
||||
steps: StepV3[];
|
||||
stepIdByDraftId: Map<number, number>;
|
||||
}
|
||||
|
||||
export function reduceTraceStepsV3(
|
||||
steps: RecordingDraftStep[],
|
||||
options: { includeTabSwitches?: boolean; redactValues?: boolean } = {},
|
||||
): ReducedTraceV3 {
|
||||
const output: StepV3[] = [];
|
||||
const stepIdByDraftId = new Map<number, number>();
|
||||
for (const { draft, draftIds } of collapseRedirects(steps)) {
|
||||
const step = reduceDraft(draft, output.length + 1, {
|
||||
includeTabSwitches: options.includeTabSwitches === true,
|
||||
redactValues: options.redactValues === true,
|
||||
});
|
||||
if (!step) continue;
|
||||
output.push(step);
|
||||
for (const draftId of draftIds) stepIdByDraftId.set(draftId, step.id);
|
||||
}
|
||||
return { steps: output, stepIdByDraftId };
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
import type { StepAnnotation } from "./types";
|
||||
|
||||
function annotationText(annotation: StepAnnotation, stepId: number): string {
|
||||
return ` ⟵ step ${stepId}: ${annotation.op}${annotation.detail ? `: ${annotation.detail}` : ""}`;
|
||||
}
|
||||
|
||||
export function formatTraceStateBody(input: {
|
||||
stateId: string;
|
||||
url: string;
|
||||
title?: string;
|
||||
stepIds: number[];
|
||||
vomText: string;
|
||||
annotations: StepAnnotation[];
|
||||
stepIdByDraftId: Map<number, number>;
|
||||
}): string {
|
||||
const lines = ["# bsk-observation 1", `state: ${JSON.stringify(input.stateId)}`];
|
||||
lines.push(`url: ${JSON.stringify(input.url)}`);
|
||||
if (input.title) lines.push(`title: ${JSON.stringify(input.title)}`);
|
||||
if (input.stepIds.length > 0) lines.push(`steps_here: [${input.stepIds.join(", ")}]`);
|
||||
lines.push("---");
|
||||
|
||||
const byLine = new Map<number, Array<{ annotation: StepAnnotation; stepId: number }>>();
|
||||
for (const annotation of input.annotations) {
|
||||
const stepId = input.stepIdByDraftId.get(annotation.draftId);
|
||||
if (stepId === undefined) continue;
|
||||
const bucket = byLine.get(annotation.line) ?? [];
|
||||
bucket.push({ annotation, stepId });
|
||||
byLine.set(annotation.line, bucket);
|
||||
}
|
||||
|
||||
input.vomText.split("\n").forEach((bodyLine, lineIndex) => {
|
||||
let line = bodyLine;
|
||||
for (const item of byLine.get(lineIndex) ?? []) {
|
||||
line += annotationText(item.annotation, item.stepId);
|
||||
}
|
||||
lines.push(line);
|
||||
});
|
||||
return `${lines.join("\n")}\n`;
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
import type { CaptureTargetDescriptor } from "@/lib/describe-target";
|
||||
import type {
|
||||
FillCommit,
|
||||
KeyModifier,
|
||||
NavigationCause,
|
||||
StepV3,
|
||||
TargetDescriptorV3,
|
||||
} from "@/transport/types";
|
||||
|
||||
export interface TargetGeometry {
|
||||
/** Top-level viewport-relative CSS pixels, as defined by the geometry module. */
|
||||
rect: { x: number; y: number; w: number; h: number };
|
||||
tag: string;
|
||||
}
|
||||
|
||||
export interface TargetMatchHint {
|
||||
geometry?: TargetGeometry;
|
||||
/** Missing means top frame; null means the source Document could not be resolved. */
|
||||
frameId?: string | null;
|
||||
geometrySpace?: "top" | "local";
|
||||
}
|
||||
|
||||
export interface StepAnnotation {
|
||||
draftId: number;
|
||||
op: StepV3["op"];
|
||||
line: number;
|
||||
stateId: string;
|
||||
detail?: string;
|
||||
}
|
||||
|
||||
interface DraftStateLink {
|
||||
pageUrl?: string;
|
||||
preStateId?: string;
|
||||
postStateId?: string;
|
||||
}
|
||||
|
||||
interface DraftTarget {
|
||||
captureTarget?: CaptureTargetDescriptor;
|
||||
targetHint?: TargetMatchHint;
|
||||
matchedTarget?: TargetDescriptorV3;
|
||||
}
|
||||
|
||||
interface DraftNavigationEffect {
|
||||
navigatedTo?: string;
|
||||
}
|
||||
|
||||
export type RecordingDraftStep =
|
||||
| ({ op: "click" } & DraftStateLink & DraftTarget & DraftNavigationEffect)
|
||||
| ({ op: "hover" } & DraftStateLink & DraftTarget)
|
||||
| ({
|
||||
op: "fill";
|
||||
value: string;
|
||||
commit?: FillCommit;
|
||||
redacted?: boolean;
|
||||
} & DraftStateLink &
|
||||
DraftTarget &
|
||||
DraftNavigationEffect)
|
||||
| ({
|
||||
op: "press";
|
||||
key: string;
|
||||
modifiers?: KeyModifier[];
|
||||
} & DraftStateLink &
|
||||
DraftTarget &
|
||||
DraftNavigationEffect)
|
||||
| ({
|
||||
op: "select";
|
||||
values: string[];
|
||||
labels?: string[];
|
||||
} & DraftStateLink &
|
||||
DraftTarget &
|
||||
DraftNavigationEffect)
|
||||
| ({ op: "scroll" } & DraftStateLink)
|
||||
| ({ op: "switch_tab" } & DraftStateLink)
|
||||
| ({
|
||||
op: "navigate";
|
||||
url: string;
|
||||
cause?: NavigationCause;
|
||||
transitionType?: string;
|
||||
transitionQualifiers?: string[];
|
||||
} & DraftStateLink);
|
||||
|
||||
export type TargetedRecordingDraft = Extract<
|
||||
RecordingDraftStep,
|
||||
{ op: "click" | "hover" | "fill" | "press" | "select" }
|
||||
>;
|
||||
@@ -33,4 +33,19 @@ describe("RefStore", () => {
|
||||
expect(s.resolve("@e11")).toBe(11);
|
||||
expect(s.size()).toBe(2);
|
||||
});
|
||||
|
||||
it("preserves the child CDP session as part of ref identity", () => {
|
||||
const s = new RefStore();
|
||||
s.set("e1", 42, {
|
||||
tabId: 7,
|
||||
frameId: "child-frame",
|
||||
cdpSessionId: "child-session",
|
||||
});
|
||||
expect(s.resolveEntry("e1")).toMatchObject({
|
||||
backendNodeId: 42,
|
||||
tabId: 7,
|
||||
frameId: "child-frame",
|
||||
cdpSessionId: "child-session",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
/**
|
||||
* Per-session map from `@e<N>` snapshot refs to CDP `backendNodeId`.
|
||||
* Per-session map from `@e<N>` snapshot refs to a CDP node address.
|
||||
*
|
||||
* Each fresh `tool.snapshot` resets the store: M6 will call
|
||||
* `replace(...)` with the new ref → backendNodeId pairs. Tools like
|
||||
* `tool.click` consume the store via `resolve("@e1")`.
|
||||
* `replace(...)` with the new ref → node address pairs. A node address
|
||||
* includes the owning flat CDP session/frame when the element lives in
|
||||
* an OOPIF; tools resolve live geometry from that identity at call time.
|
||||
*
|
||||
* Refs are session-scoped (§7): looking up a ref in the wrong session
|
||||
* returns `null`, never silently leaks. Storing values in different
|
||||
@@ -15,10 +16,19 @@ export type BackendNodeId = number;
|
||||
export interface RefEntry {
|
||||
backendNodeId: BackendNodeId;
|
||||
tabId: number | null;
|
||||
frameId?: string;
|
||||
cdpSessionId?: string;
|
||||
generation: number;
|
||||
}
|
||||
|
||||
type RefInput = BackendNodeId | { backendNodeId: BackendNodeId; tabId: number };
|
||||
export type RefInput =
|
||||
| BackendNodeId
|
||||
| {
|
||||
backendNodeId: BackendNodeId;
|
||||
tabId: number;
|
||||
frameId?: string;
|
||||
cdpSessionId?: string;
|
||||
};
|
||||
|
||||
export class RefStore {
|
||||
private readonly map = new Map<string, RefEntry>();
|
||||
@@ -44,7 +54,7 @@ export class RefStore {
|
||||
}
|
||||
|
||||
/**
|
||||
* Replace the entire store with a new ref → backendNodeId mapping.
|
||||
* Replace the entire store with a new ref → CDP node identity mapping.
|
||||
* Used after every fresh `tool.snapshot`.
|
||||
*/
|
||||
replace(entries: Iterable<readonly [string, RefInput]>): void {
|
||||
@@ -53,10 +63,20 @@ export class RefStore {
|
||||
for (const [ref, input] of entries) this.map.set(normaliseRef(ref), this.entry(input));
|
||||
}
|
||||
|
||||
set(ref: string, id: BackendNodeId, opts: { tabId?: number } = {}): void {
|
||||
set(
|
||||
ref: string,
|
||||
id: BackendNodeId,
|
||||
opts: {
|
||||
tabId?: number;
|
||||
frameId?: string;
|
||||
cdpSessionId?: string;
|
||||
} = {},
|
||||
): void {
|
||||
this.map.set(normaliseRef(ref), {
|
||||
backendNodeId: id,
|
||||
tabId: opts.tabId ?? null,
|
||||
...(opts.frameId ? { frameId: opts.frameId } : {}),
|
||||
...(opts.cdpSessionId ? { cdpSessionId: opts.cdpSessionId } : {}),
|
||||
generation: this.generation,
|
||||
});
|
||||
}
|
||||
@@ -80,6 +100,8 @@ export class RefStore {
|
||||
return {
|
||||
backendNodeId: input.backendNodeId,
|
||||
tabId: input.tabId,
|
||||
...(input.frameId ? { frameId: input.frameId } : {}),
|
||||
...(input.cdpSessionId ? { cdpSessionId: input.cdpSessionId } : {}),
|
||||
generation: this.generation,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
export const RECORD_DOCUMENT_ATTRIBUTE = "data-bsk-record-document";
|
||||
|
||||
const RECORDING_DOCUMENT_ID_PATTERN =
|
||||
/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
|
||||
const RECORDING_DOCUMENT_MARKER_PREFIX = "bsk:";
|
||||
|
||||
export function recordingDocumentMarkerValue(identity: string): string {
|
||||
if (!RECORDING_DOCUMENT_ID_PATTERN.test(identity)) {
|
||||
throw new TypeError("recording Document identity must be a random UUID");
|
||||
}
|
||||
return `${RECORDING_DOCUMENT_MARKER_PREFIX}${identity}`;
|
||||
}
|
||||
|
||||
/** Read BrowserSkill's opaque per-Document recording identity from captured attributes. */
|
||||
export function readRecordingDocumentIdentity(
|
||||
attrs: Readonly<Record<string, string>>,
|
||||
): string | undefined {
|
||||
const marker = attrs[RECORD_DOCUMENT_ATTRIBUTE];
|
||||
if (!marker?.startsWith(RECORDING_DOCUMENT_MARKER_PREFIX)) return undefined;
|
||||
const identity = marker.slice(RECORDING_DOCUMENT_MARKER_PREFIX.length);
|
||||
return identity && RECORDING_DOCUMENT_ID_PATTERN.test(identity) ? identity : undefined;
|
||||
}
|
||||
@@ -612,4 +612,14 @@ describe("isInjectableContentScriptUrl", () => {
|
||||
).toBe(false);
|
||||
expect(isInjectableContentScriptUrl("https://chromewebstore.google.com/")).toBe(false);
|
||||
});
|
||||
|
||||
it("rejects the Edge Add-ons store", () => {
|
||||
expect(
|
||||
isInjectableContentScriptUrl(
|
||||
"https://microsoftedge.microsoft.com/addons/detail/browserskill/emacgiaaaiojkkpkddmmdfhmokgmnikg",
|
||||
),
|
||||
).toBe(false);
|
||||
// Non-store paths on the same host stay injectable.
|
||||
expect(isInjectableContentScriptUrl("https://microsoftedge.microsoft.com/")).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -8,6 +8,7 @@ import type {
|
||||
RequestFrame,
|
||||
} from "@/transport/types";
|
||||
import { ToolDispatcher } from "../dispatcher";
|
||||
import { resetBrowserObservationForTests, setBrowserObservationAttachForTests } from "../record";
|
||||
|
||||
type TestDispatcherCdp = NonNullable<ConstructorParameters<typeof ToolDispatcher>[0]["cdp"]>;
|
||||
|
||||
@@ -44,9 +45,72 @@ function makeRequest(method: string, params: unknown): RequestFrame {
|
||||
|
||||
describe("ToolDispatcher", () => {
|
||||
afterEach(() => {
|
||||
resetBrowserObservationForTests();
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
it("uses the configured recording runtime for start and stop", async () => {
|
||||
const { transport, sent, deliver } = fakeTransport();
|
||||
const sessions = new SessionManager({
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 4242),
|
||||
remove: vi.fn(),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
},
|
||||
});
|
||||
await sessions.start("aa11");
|
||||
const tab = {
|
||||
id: 7,
|
||||
windowId: 4242,
|
||||
active: true,
|
||||
status: "complete",
|
||||
url: "https://example.com/start",
|
||||
} as chrome.tabs.Tab;
|
||||
const frameCoordinator = {
|
||||
begin: vi.fn(),
|
||||
armTab: vi.fn(async () => true),
|
||||
sourceFor: vi.fn(() => null),
|
||||
stop: vi.fn(async () => true),
|
||||
cancel: vi.fn(),
|
||||
};
|
||||
const sendToTab = vi.fn(async () => ({ ok: true }));
|
||||
setBrowserObservationAttachForTests(
|
||||
() => () => {},
|
||||
() => () => {},
|
||||
);
|
||||
const dispatcher = new ToolDispatcher({
|
||||
transport,
|
||||
sessions,
|
||||
recording: {
|
||||
tabsApi: {
|
||||
get: vi.fn(async () => tab),
|
||||
query: vi.fn(async () => [tab]),
|
||||
},
|
||||
frameCoordinator,
|
||||
sendToTab,
|
||||
},
|
||||
});
|
||||
dispatcher.start();
|
||||
|
||||
deliver(
|
||||
makeRequest("tool.record_start", {
|
||||
session_id: "aa11",
|
||||
url: "https://example.com/start",
|
||||
}),
|
||||
);
|
||||
await vi.waitFor(() => expect(sent).toHaveLength(1));
|
||||
|
||||
expect(sent[0]).toMatchObject({ result: { tab_id: 7, recording: true } });
|
||||
expect(frameCoordinator.begin).toHaveBeenCalledOnce();
|
||||
expect(frameCoordinator.armTab).toHaveBeenCalledWith(expect.any(String), 7);
|
||||
|
||||
deliver({ id: "r-2", method: "tool.record_stop", params: { session_id: "aa11" } });
|
||||
await vi.waitFor(() => expect(sent).toHaveLength(2));
|
||||
|
||||
expect(frameCoordinator.stop).toHaveBeenCalledOnce();
|
||||
expect(sent[1]).toMatchObject({ id: "r-2", result: { trace: { steps: [] } } });
|
||||
});
|
||||
|
||||
it("routes tool.session_start to the SessionManager and replies with the window id", async () => {
|
||||
const { transport, sent, deliver } = fakeTransport();
|
||||
const sessions = new SessionManager({
|
||||
@@ -203,6 +267,55 @@ describe("ToolDispatcher", () => {
|
||||
expect(sent[0]).toEqual({ id: "r-1", result: {} });
|
||||
});
|
||||
|
||||
it("forwards daemon cancellation into an in-flight session stop", async () => {
|
||||
const { transport, sent, deliver } = fakeTransport();
|
||||
const remove = vi.fn(async () => {});
|
||||
const sessions = new SessionManager({
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 4242),
|
||||
remove,
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
},
|
||||
});
|
||||
await sessions.start("aa11");
|
||||
let finishDetach: () => void = () => {};
|
||||
const detach = new Promise<void>((resolve) => {
|
||||
finishDetach = resolve;
|
||||
});
|
||||
const cdp = {
|
||||
send: vi.fn(),
|
||||
detachSession: vi.fn(() => detach),
|
||||
ensureNetworkCapture: vi.fn(async () => {}),
|
||||
networkEntriesSince: vi.fn(() => ({
|
||||
tab_id: 7,
|
||||
entries: [],
|
||||
next_since: 0,
|
||||
truncated: false,
|
||||
})),
|
||||
setDeviceMetricsOverride: vi.fn(async () => {}),
|
||||
clearDeviceMetricsOverride: vi.fn(async () => {}),
|
||||
setUserAgentOverride: vi.fn(async () => {}),
|
||||
setTouchEmulationEnabled: vi.fn(async () => {}),
|
||||
};
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions, cdp: cdp as TestDispatcherCdp });
|
||||
dispatcher.start();
|
||||
|
||||
deliver(makeRequest("tool.session_stop", { session_id: "aa11" }));
|
||||
await vi.waitFor(() => expect(cdp.detachSession).toHaveBeenCalledWith("aa11"));
|
||||
deliver({ id: "cancel-stop", method: "cancel", params: { rpc_id: "r-1" } });
|
||||
await flushMicrotasks();
|
||||
finishDetach();
|
||||
await flushMicrotasks();
|
||||
|
||||
expect(sent).toContainEqual({ id: "cancel-stop", result: { cancelled: true } });
|
||||
expect(sent).toContainEqual({
|
||||
id: "r-1",
|
||||
error: expect.objectContaining({ code: "cancelled" }),
|
||||
});
|
||||
expect(sessions.has("aa11")).toBe(true);
|
||||
expect(remove).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("returns not_found when stopping an unknown session", async () => {
|
||||
const { transport, sent, deliver } = fakeTransport();
|
||||
const sessions = new SessionManager({
|
||||
|
||||
@@ -1,24 +1,54 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { nodeBoundingRect, quadBoundingRect } from "../element-geometry";
|
||||
import { nodeContentRegion } from "../element-geometry";
|
||||
import type { CdpRunner } from "../shared";
|
||||
|
||||
describe("quadBoundingRect", () => {
|
||||
it("computes axis-aligned bounds from an 8-double quad", () => {
|
||||
expect(quadBoundingRect([10, 20, 110, 20, 110, 60, 10, 60])).toEqual({
|
||||
x: 10,
|
||||
y: 20,
|
||||
width: 100,
|
||||
height: 40,
|
||||
describe("nodeContentRegion", () => {
|
||||
it("rejects invalid content quads before using the box model fallback", async () => {
|
||||
const send = async (_tabId: number, method: string) => {
|
||||
if (method === "DOM.getContentQuads") return { quads: [[0, 0, 0, 0, 0, 0, 0, 0]] };
|
||||
if (method === "DOM.getBoxModel") {
|
||||
return { model: { content: [5, 10, 25, 10, 25, 30, 5, 30] } };
|
||||
}
|
||||
throw new Error(`unexpected CDP call ${method}`);
|
||||
};
|
||||
|
||||
await expect(nodeContentRegion({ send: send as CdpRunner["send"] }, 7, 555)).resolves.toEqual([
|
||||
[
|
||||
{ x: 5, y: 10 },
|
||||
{ x: 25, y: 10 },
|
||||
{ x: 25, y: 30 },
|
||||
{ x: 5, y: 30 },
|
||||
],
|
||||
]);
|
||||
});
|
||||
|
||||
it("preserves every visible content fragment", async () => {
|
||||
const send = async () => ({
|
||||
quads: [
|
||||
[10, 20, 60, 20, 60, 40, 10, 40],
|
||||
[15, 50, 90, 50, 90, 80, 15, 80],
|
||||
],
|
||||
});
|
||||
const cdp: CdpRunner = {
|
||||
send: send as CdpRunner["send"],
|
||||
};
|
||||
|
||||
await expect(nodeContentRegion(cdp, 7, 555)).resolves.toEqual([
|
||||
[
|
||||
{ x: 10, y: 20 },
|
||||
{ x: 60, y: 20 },
|
||||
{ x: 60, y: 40 },
|
||||
{ x: 10, y: 40 },
|
||||
],
|
||||
[
|
||||
{ x: 15, y: 50 },
|
||||
{ x: 90, y: 50 },
|
||||
{ x: 90, y: 80 },
|
||||
{ x: 15, y: 80 },
|
||||
],
|
||||
]);
|
||||
});
|
||||
|
||||
it("returns null for degenerate quads", () => {
|
||||
expect(quadBoundingRect([0, 0, 0, 0, 0, 0, 0, 0])).toBeNull();
|
||||
expect(quadBoundingRect([1, 2, 3])).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("nodeBoundingRect", () => {
|
||||
it("falls back to visible descendant bounds for zero-size containers", async () => {
|
||||
const calls: Array<{ method: string; params?: object }> = [];
|
||||
const send = async (_tabId: number, method: string, params?: object) => {
|
||||
@@ -38,12 +68,14 @@ describe("nodeBoundingRect", () => {
|
||||
};
|
||||
const cdp = { send: send as CdpRunner["send"] };
|
||||
|
||||
await expect(nodeBoundingRect(cdp, 7, 555)).resolves.toEqual({
|
||||
x: 242,
|
||||
y: 468,
|
||||
width: 769,
|
||||
height: 180,
|
||||
});
|
||||
await expect(nodeContentRegion(cdp, 7, 555)).resolves.toEqual([
|
||||
[
|
||||
{ x: 242, y: 468 },
|
||||
{ x: 1011, y: 468 },
|
||||
{ x: 1011, y: 648 },
|
||||
{ x: 242, y: 648 },
|
||||
],
|
||||
]);
|
||||
expect(calls.map((c) => c.method)).toEqual([
|
||||
"DOM.getContentQuads",
|
||||
"DOM.getBoxModel",
|
||||
|
||||
@@ -0,0 +1,286 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { resolveFrameProjection, resolveNodeGeometry } from "../frame-geometry";
|
||||
import {
|
||||
clipPolygon,
|
||||
polygonArea,
|
||||
polygonBounds,
|
||||
projectAndClipRegion,
|
||||
projectUnitPoint,
|
||||
type Quad,
|
||||
rectPolygon,
|
||||
regionBounds,
|
||||
} from "../geometry";
|
||||
import type { CdpRunner } from "../shared";
|
||||
|
||||
describe("frame geometry projection", () => {
|
||||
it("keeps region bounds separate from polygon area", () => {
|
||||
const region = [
|
||||
rectPolygon({ x: 0, y: 0, w: 10, h: 10 }),
|
||||
rectPolygon({ x: 20, y: 5, w: 5, h: 5 }),
|
||||
];
|
||||
expect(regionBounds(region)).toEqual({ x: 0, y: 0, width: 25, height: 10 });
|
||||
expect(polygonArea(region[0])).toBe(100);
|
||||
});
|
||||
|
||||
it("maps through the iframe content box instead of its border box", () => {
|
||||
const contentQuad: Quad = [
|
||||
{ x: 204, y: 306 },
|
||||
{ x: 604, y: 306 },
|
||||
{ x: 604, y: 506 },
|
||||
{ x: 204, y: 506 },
|
||||
];
|
||||
const projected = projectAndClipRegion(
|
||||
[rectPolygon({ x: 10, y: 20, w: 100, h: 40 })],
|
||||
[{ sourceViewport: { width: 200, height: 100 }, destinationQuad: contentQuad }],
|
||||
{ width: 1000, height: 800 },
|
||||
);
|
||||
expect(polygonBounds(projected.flat())).toEqual({ x: 224, y: 346, width: 200, height: 80 });
|
||||
});
|
||||
|
||||
it("uses a projective mapping for perspective-transformed iframe quads", () => {
|
||||
const quad: Quad = [
|
||||
{ x: 0, y: 0 },
|
||||
{ x: 200, y: 20 },
|
||||
{ x: 180, y: 140 },
|
||||
{ x: 20, y: 100 },
|
||||
];
|
||||
expect(projectUnitPoint({ x: 0, y: 0 }, quad)).toEqual(quad[0]);
|
||||
expect(projectUnitPoint({ x: 1, y: 0 }, quad)).toEqual(quad[1]);
|
||||
expect(projectUnitPoint({ x: 1, y: 1 }, quad)).toEqual(quad[2]);
|
||||
expect(projectUnitPoint({ x: 0, y: 1 }, quad)).toEqual(quad[3]);
|
||||
});
|
||||
|
||||
it("clips against the iframe polygon rather than its bounding box", () => {
|
||||
const frameQuad: Quad = [
|
||||
{ x: 50, y: 0 },
|
||||
{ x: 100, y: 50 },
|
||||
{ x: 50, y: 100 },
|
||||
{ x: 0, y: 50 },
|
||||
];
|
||||
expect(clipPolygon(rectPolygon({ x: 0, y: 0, w: 10, h: 10 }), frameQuad)).toEqual([]);
|
||||
});
|
||||
|
||||
it("does not reapply same-target frame transforms around an OOPIF", async () => {
|
||||
const send = vi.fn(async (_tabId, method, params) => {
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 800, clientHeight: 600 } };
|
||||
}
|
||||
if (method === "DOM.getBoxModel") {
|
||||
const backendNodeId = (params as { backendNodeId?: number }).backendNodeId;
|
||||
return backendNodeId === 20
|
||||
? { model: { content: [110, 70, 310, 70, 310, 170, 110, 170] } }
|
||||
: { model: { content: [100, 50, 400, 50, 400, 250, 100, 250] } };
|
||||
}
|
||||
throw new Error(`unexpected root command ${method}`);
|
||||
});
|
||||
const cdp: CdpRunner = {
|
||||
send: send as CdpRunner["send"],
|
||||
sendToTarget: vi.fn(async (_target, method) => {
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
|
||||
}
|
||||
throw new Error(`unexpected child command ${method}`);
|
||||
}) as CdpRunner["sendToTarget"],
|
||||
};
|
||||
const projection = await resolveFrameProjection(
|
||||
cdp,
|
||||
{
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "same-process-parent",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 10,
|
||||
target: { tabId: 4 },
|
||||
},
|
||||
{
|
||||
frameId: "oopif",
|
||||
parentFrameId: "same-process-parent",
|
||||
ownerBackendNodeId: 20,
|
||||
target: { tabId: 4, sessionId: "oopif-session" },
|
||||
},
|
||||
],
|
||||
},
|
||||
"oopif",
|
||||
);
|
||||
|
||||
expect(projection?.edges).toEqual([
|
||||
{
|
||||
sourceViewport: { width: 200, height: 100 },
|
||||
destinationQuad: [
|
||||
{ x: 110, y: 70 },
|
||||
{ x: 310, y: 70 },
|
||||
{ x: 310, y: 170 },
|
||||
{ x: 110, y: 170 },
|
||||
],
|
||||
destinationClips: [
|
||||
[
|
||||
{ x: 100, y: 50 },
|
||||
{ x: 400, y: 50 },
|
||||
{ x: 400, y: 250 },
|
||||
{ x: 100, y: 250 },
|
||||
],
|
||||
],
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("clips live OOPIF geometry at every frame boundary", async () => {
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async (_tabId, method) => {
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 800, clientHeight: 600 } };
|
||||
}
|
||||
if (method === "DOM.getBoxModel") {
|
||||
return { model: { content: [100, 100, 300, 100, 300, 200, 100, 200] } };
|
||||
}
|
||||
throw new Error(`unexpected root command ${method}`);
|
||||
}) as CdpRunner["send"],
|
||||
sendToTarget: vi.fn(async (_target, method) => {
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
|
||||
}
|
||||
if (method === "DOM.getContentQuads") {
|
||||
return { quads: [[-50, -20, 250, -20, 250, 120, -50, 120]] };
|
||||
}
|
||||
throw new Error(`unexpected child command ${method}`);
|
||||
}) as CdpRunner["sendToTarget"],
|
||||
getFrameGraph: vi.fn(async () => ({
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "child",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 10,
|
||||
target: { tabId: 4, sessionId: "child-session" },
|
||||
},
|
||||
],
|
||||
})),
|
||||
};
|
||||
|
||||
const geometry = await resolveNodeGeometry(cdp, 4, {
|
||||
target: { tabId: 4, sessionId: "child-session" },
|
||||
frameId: "child",
|
||||
backendNodeId: 101,
|
||||
});
|
||||
|
||||
expect(geometry).toMatchObject({
|
||||
topBounds: { x: 100, y: 100, width: 200, height: 100 },
|
||||
actionPoint: { x: 200, y: 150 },
|
||||
});
|
||||
});
|
||||
|
||||
it("reuses one frame graph while scrolling and projecting live geometry", async () => {
|
||||
const getFrameGraph = vi.fn(async () => ({
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "child",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 10,
|
||||
target: { tabId: 4, sessionId: "child-session" },
|
||||
},
|
||||
],
|
||||
}));
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async (_tabId, method) => {
|
||||
if (method === "DOM.scrollIntoViewIfNeeded") return {};
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 800, clientHeight: 600 } };
|
||||
}
|
||||
if (method === "DOM.getBoxModel") {
|
||||
return { model: { content: [100, 100, 300, 100, 300, 200, 100, 200] } };
|
||||
}
|
||||
throw new Error(`unexpected root command ${method}`);
|
||||
}) as CdpRunner["send"],
|
||||
sendToTarget: vi.fn(async (_target, method) => {
|
||||
if (method === "DOM.scrollIntoViewIfNeeded") return {};
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
|
||||
}
|
||||
if (method === "DOM.getContentQuads") {
|
||||
return { quads: [[0, 0, 20, 0, 20, 20, 0, 20]] };
|
||||
}
|
||||
throw new Error(`unexpected child command ${method}`);
|
||||
}) as CdpRunner["sendToTarget"],
|
||||
getFrameGraph,
|
||||
};
|
||||
|
||||
await expect(
|
||||
resolveNodeGeometry(
|
||||
cdp,
|
||||
4,
|
||||
{
|
||||
target: { tabId: 4, sessionId: "child-session" },
|
||||
frameId: "child",
|
||||
backendNodeId: 101,
|
||||
},
|
||||
{ scrollIntoView: true },
|
||||
),
|
||||
).resolves.not.toMatchObject({ code: expect.any(String) });
|
||||
expect(getFrameGraph).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("clips live geometry to same-target iframe ancestors", async () => {
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async (_tabId, method) => {
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 800, clientHeight: 600 } };
|
||||
}
|
||||
if (method === "DOM.getBoxModel") {
|
||||
return { model: { content: [100, 100, 300, 100, 300, 200, 100, 200] } };
|
||||
}
|
||||
if (method === "DOM.getContentQuads") {
|
||||
return { quads: [[50, 50, 350, 50, 350, 250, 50, 250]] };
|
||||
}
|
||||
throw new Error(`unexpected ${method}`);
|
||||
}) as CdpRunner["send"],
|
||||
getFrameGraph: vi.fn(async () => ({
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "child",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 10,
|
||||
target: { tabId: 4 },
|
||||
},
|
||||
],
|
||||
})),
|
||||
};
|
||||
|
||||
const geometry = await resolveNodeGeometry(cdp, 4, {
|
||||
target: { tabId: 4 },
|
||||
frameId: "child",
|
||||
backendNodeId: 101,
|
||||
});
|
||||
|
||||
expect(geometry).toMatchObject({
|
||||
topBounds: { x: 100, y: 100, width: 200, height: 100 },
|
||||
actionPoint: { x: 200, y: 150 },
|
||||
});
|
||||
});
|
||||
|
||||
it("fails closed when an OOPIF frame graph is unavailable", async () => {
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(),
|
||||
sendToTarget: vi.fn(async (_target, method) => {
|
||||
if (method === "DOM.getContentQuads") {
|
||||
return { quads: [[0, 0, 20, 0, 20, 20, 0, 20]] };
|
||||
}
|
||||
throw new Error(`unexpected ${method}`);
|
||||
}) as CdpRunner["sendToTarget"],
|
||||
};
|
||||
|
||||
await expect(
|
||||
resolveNodeGeometry(cdp, 4, {
|
||||
target: { tabId: 4, sessionId: "child-session" },
|
||||
frameId: "child",
|
||||
backendNodeId: 101,
|
||||
}),
|
||||
).resolves.toMatchObject({ code: "cdp_failed" });
|
||||
});
|
||||
});
|
||||
@@ -1,5 +1,6 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { SessionManager } from "@/session-manager/manager";
|
||||
import { RefStore } from "@/session-manager/ref-store";
|
||||
import type { RequestHelpParams } from "@/transport/types";
|
||||
import { handleRequestHelp, type RequestHelpDeps, resetHelpLifecycleForTests } from "../human-loop";
|
||||
|
||||
@@ -50,11 +51,10 @@ function installHelpLifecycleChrome() {
|
||||
}
|
||||
|
||||
function fakeManager(sessionId: string, agentWindowId: number, tabId: number) {
|
||||
const refStore = new RefStore();
|
||||
const mgr = {
|
||||
get: (id: string) =>
|
||||
id === sessionId
|
||||
? { sessionId, agentWindowId, refStore: { resolve: () => null }, borrowedTabs: new Map() }
|
||||
: null,
|
||||
id === sessionId ? { sessionId, agentWindowId, refStore, borrowedTabs: new Map() } : null,
|
||||
findByWindowId: (wid: number) => (wid === agentWindowId ? { sessionId } : null),
|
||||
} as unknown as SessionManager;
|
||||
return mgr;
|
||||
@@ -339,26 +339,23 @@ describe("handleRequestHelp", () => {
|
||||
|
||||
ac.abort();
|
||||
await expect(pending).resolves.toMatchObject({ code: "cancelled" });
|
||||
expect(cdpSend).toHaveBeenCalledWith(
|
||||
5,
|
||||
"DOM.querySelectorAll",
|
||||
expect.objectContaining({ selector: "[data-bsk-help]" }),
|
||||
);
|
||||
expect(cdpSend).toHaveBeenCalledWith(
|
||||
6,
|
||||
"DOM.querySelectorAll",
|
||||
expect.objectContaining({ selector: "[data-bsk-help]" }),
|
||||
expect(cdpSend).not.toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
"DOM.removeAttribute",
|
||||
expect.anything(),
|
||||
);
|
||||
});
|
||||
|
||||
it("tags ref targets via CDP and reports them matched", async () => {
|
||||
it("resolves ref targets to explicit viewport rectangles without mutating the page", async () => {
|
||||
const refStore = new RefStore();
|
||||
refStore.set("e1", 42, { tabId: 5 });
|
||||
const mgr = {
|
||||
get: (id: string) =>
|
||||
id === "abcd"
|
||||
? {
|
||||
sessionId: "abcd",
|
||||
agentWindowId: 99,
|
||||
refStore: { resolve: () => 42 },
|
||||
refStore,
|
||||
borrowedTabs: new Map(),
|
||||
}
|
||||
: null,
|
||||
@@ -366,7 +363,17 @@ describe("handleRequestHelp", () => {
|
||||
} as unknown as SessionManager;
|
||||
const deps = baseDeps({
|
||||
cdp: {
|
||||
send: vi.fn(async () => ({ object: { objectId: "obj-1" } })),
|
||||
send: vi.fn(async (_tabId, method) => {
|
||||
if (method === "DOM.getDocument") return { root: { nodeId: 1 } };
|
||||
if (method === "DOM.scrollIntoViewIfNeeded") return {};
|
||||
if (method === "DOM.getContentQuads") {
|
||||
return { quads: [[10, 20, 110, 20, 110, 60, 10, 60]] };
|
||||
}
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
|
||||
}
|
||||
throw new Error(`unexpected ${method}`);
|
||||
}),
|
||||
} as unknown as RequestHelpDeps["cdp"],
|
||||
});
|
||||
const res = await handleRequestHelp(
|
||||
@@ -375,14 +382,97 @@ describe("handleRequestHelp", () => {
|
||||
deps,
|
||||
);
|
||||
const sentMsg = (deps.sendToTab as ReturnType<typeof vi.fn>).mock.calls[0][1];
|
||||
expect(sentMsg.selectors).toContain('[data-bsk-help="0"]');
|
||||
expect(sentMsg.selectors).toEqual([]);
|
||||
expect(sentMsg.rects).toEqual([{ top: 20, left: 10, width: 100, height: 40 }]);
|
||||
expect(res).toMatchObject({
|
||||
outcome: "continued",
|
||||
tab_id: 5,
|
||||
resolved_targets: [{ matched: true, ref: "@e1" }],
|
||||
resolved_targets: [
|
||||
{
|
||||
matched: true,
|
||||
ref: "@e1",
|
||||
},
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
it("projects an OOPIF ref through its live content quad for the help overlay", async () => {
|
||||
const mgr = {
|
||||
get: (id: string) =>
|
||||
id === "abcd"
|
||||
? {
|
||||
sessionId: "abcd",
|
||||
agentWindowId: 99,
|
||||
refStore: {
|
||||
resolveEntry: () => ({
|
||||
backendNodeId: 42,
|
||||
tabId: 5,
|
||||
frameId: "child",
|
||||
cdpSessionId: "child-session",
|
||||
generation: 1,
|
||||
}),
|
||||
},
|
||||
borrowedTabs: new Map(),
|
||||
}
|
||||
: null,
|
||||
findByWindowId: (wid: number) => (wid === 99 ? { sessionId: "abcd" } : null),
|
||||
} as unknown as SessionManager;
|
||||
const send = vi.fn(async (_tabId, method) => {
|
||||
if (method === "DOM.getDocument") return { root: { nodeId: 1 } };
|
||||
if (method === "DOM.scrollIntoViewIfNeeded") return {};
|
||||
if (method === "DOM.getBoxModel") {
|
||||
return { model: { content: [204, 306, 604, 306, 604, 506, 204, 506] } };
|
||||
}
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
|
||||
}
|
||||
throw new Error(`unexpected root ${method}`);
|
||||
});
|
||||
const sendToTarget = vi.fn(async (_target, method) => {
|
||||
if (method === "DOM.scrollIntoViewIfNeeded") return {};
|
||||
if (method === "DOM.getContentQuads") {
|
||||
return { quads: [[10, 20, 110, 20, 110, 60, 10, 60]] };
|
||||
}
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
|
||||
}
|
||||
throw new Error(`unexpected child ${method}`);
|
||||
});
|
||||
const deps = baseDeps({
|
||||
cdp: {
|
||||
send,
|
||||
sendToTarget,
|
||||
getFrameGraph: vi.fn(async () => ({
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 5 } },
|
||||
{
|
||||
frameId: "child",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 99,
|
||||
target: { tabId: 5, sessionId: "child-session" },
|
||||
},
|
||||
],
|
||||
})),
|
||||
} as unknown as NonNullable<RequestHelpDeps["cdp"]>,
|
||||
});
|
||||
|
||||
const res = await handleRequestHelp(
|
||||
mgr,
|
||||
baseParams({ tab_id: 5, targets: [{ ref: "@e1" }] }),
|
||||
deps,
|
||||
);
|
||||
|
||||
const sentMsg = (deps.sendToTab as ReturnType<typeof vi.fn>).mock.calls[0][1];
|
||||
expect(sentMsg.rects).toEqual([{ top: 346, left: 224, width: 200, height: 80 }]);
|
||||
expect(res).toMatchObject({ resolved_targets: [{ matched: true, ref: "@e1" }] });
|
||||
expect(sendToTarget).toHaveBeenCalledWith(
|
||||
{ tabId: 5, sessionId: "child-session" },
|
||||
"DOM.getContentQuads",
|
||||
{ backendNodeId: 42 },
|
||||
);
|
||||
});
|
||||
|
||||
it("reports ref target unmatched when ref does not resolve", async () => {
|
||||
const deps = baseDeps();
|
||||
const res = await handleRequestHelp(
|
||||
@@ -398,16 +488,15 @@ describe("handleRequestHelp", () => {
|
||||
});
|
||||
|
||||
it("reports ref target unmatched when ref is for another tab", async () => {
|
||||
const refStore = new RefStore();
|
||||
refStore.set("e1", 42, { tabId: 4 });
|
||||
const mgr = {
|
||||
get: (id: string) =>
|
||||
id === "abcd"
|
||||
? {
|
||||
sessionId: "abcd",
|
||||
agentWindowId: 99,
|
||||
refStore: {
|
||||
resolve: (ref: string, opts: { tabId?: number }) =>
|
||||
ref === "e1" && opts.tabId === 4 ? 42 : null,
|
||||
},
|
||||
refStore,
|
||||
borrowedTabs: new Map(),
|
||||
}
|
||||
: null,
|
||||
|
||||
@@ -30,6 +30,9 @@ function makeFakeCdp(handlers: Record<string, (params: unknown) => unknown>) {
|
||||
const sendImpl = async (tabId: number, method: string, params?: object) => {
|
||||
sent.push({ tabId, method, params });
|
||||
const h = handlers[method];
|
||||
if (!h && method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
|
||||
}
|
||||
if (!h) throw new Error(`unexpected CDP call ${method}`);
|
||||
return h(params);
|
||||
};
|
||||
@@ -164,6 +167,63 @@ describe("handleClick", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("resolves frame refs in their CDP session and dispatches input in top coordinates", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.refStore.set("e3", 1234, {
|
||||
tabId: 4,
|
||||
frameId: "child-frame",
|
||||
cdpSessionId: "child-session",
|
||||
});
|
||||
const fake = makeFakeCdp({
|
||||
"Input.dispatchMouseEvent": () => ({}),
|
||||
"DOM.scrollIntoViewIfNeeded": () => ({}),
|
||||
"DOM.getBoxModel": () => ({
|
||||
model: { content: [204, 306, 604, 306, 604, 506, 204, 506] },
|
||||
}),
|
||||
});
|
||||
fake.cdp.getFrameGraph = vi.fn(async () => ({
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "child-frame",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 99,
|
||||
target: { tabId: 4, sessionId: "child-session" },
|
||||
},
|
||||
],
|
||||
}));
|
||||
const targetCalls: Array<{ sessionId?: string; method: string }> = [];
|
||||
fake.cdp.sendToTarget = vi.fn(async (target, method) => {
|
||||
targetCalls.push({ sessionId: target.sessionId, method });
|
||||
if (method === "DOM.scrollIntoViewIfNeeded") return {};
|
||||
if (method === "DOM.getContentQuads") {
|
||||
return { quads: [[10, 20, 110, 20, 110, 60, 10, 60]] };
|
||||
}
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
|
||||
}
|
||||
throw new Error(`unexpected child CDP call ${method}`);
|
||||
}) as CdpRunner["sendToTarget"];
|
||||
|
||||
const res = await handleClick(
|
||||
sm,
|
||||
{ session_id: "aa11", ref: "@e3" },
|
||||
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
||||
);
|
||||
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
// The iframe content box starts at (204, 306) after its border and is scaled 2x.
|
||||
expect(res).toMatchObject({ x: 324, y: 386 });
|
||||
expect(targetCalls).toEqual([
|
||||
{ sessionId: "child-session", method: "DOM.scrollIntoViewIfNeeded" },
|
||||
{ sessionId: "child-session", method: "DOM.getContentQuads" },
|
||||
{ sessionId: "child-session", method: "Page.getLayoutMetrics" },
|
||||
]);
|
||||
expect(fake.sent.filter((call) => call.method === "Input.dispatchMouseEvent")).toHaveLength(3);
|
||||
});
|
||||
|
||||
it("enables overlay bypass before mouse events when overlay blocks the click point", async () => {
|
||||
const order: string[] = [];
|
||||
const bypassOverlay = vi.fn(async (_tabId: number, enabled: boolean) => {
|
||||
|
||||
@@ -5,8 +5,10 @@ import { OVERLAY_HOST_MARKER_ATTR, OVERLAY_HOST_NAME } from "@/lib/overlay-bridg
|
||||
import { SessionManager } from "@/session-manager/manager";
|
||||
import type { CdpRunner } from "@/tools/shared";
|
||||
import {
|
||||
buildFrameVomScene,
|
||||
buildVomScene,
|
||||
type CdpAxNode,
|
||||
captureVomObservation,
|
||||
handleGetHtml,
|
||||
handleObserve,
|
||||
handleScreenshot,
|
||||
@@ -62,6 +64,9 @@ function makeFakeCdp(handlers: Record<string, (params?: object) => unknown>) {
|
||||
const send = vi.fn(async (_tabId: number, method: string, params?: object) => {
|
||||
sent.push({ method, params });
|
||||
const handler = handlers[method];
|
||||
if (!handler && method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
|
||||
}
|
||||
if (!handler) throw new Error(`unexpected CDP call ${method}`);
|
||||
return handler(params);
|
||||
});
|
||||
@@ -508,6 +513,292 @@ describe("handleScreenshot overlay suppression", () => {
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe("buildVomScene", () => {
|
||||
it("merges multiple sibling and nested frames without mixing their targets", () => {
|
||||
const node = (
|
||||
backendNodeId: number,
|
||||
parentBackendNodeId: number | null,
|
||||
frameId: string,
|
||||
tag: string,
|
||||
attrs: Record<string, string> = {},
|
||||
): CapturedNode => ({
|
||||
backendNodeId,
|
||||
parentBackendNodeId,
|
||||
frameId,
|
||||
tag,
|
||||
attrs,
|
||||
rect: { x: 0, y: 0, w: 100, h: 30 },
|
||||
localRect: { x: 0, y: 0, w: 100, h: 30 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
});
|
||||
const mainNodes = [
|
||||
node(1, null, "main", "body"),
|
||||
node(10, 1, "main", "iframe", { title: "First" }),
|
||||
node(20, 1, "main", "iframe", { title: "Second" }),
|
||||
];
|
||||
const firstNodes = [
|
||||
node(100, null, "first", "body"),
|
||||
node(101, 100, "first", "button", { "aria-label": "First action" }),
|
||||
];
|
||||
const secondNodes = [
|
||||
node(200, null, "second", "body"),
|
||||
node(201, 200, "second", "h2"),
|
||||
node(210, 200, "second", "iframe", { title: "Nested" }),
|
||||
];
|
||||
const nestedNodes = [
|
||||
node(300, null, "nested", "body"),
|
||||
node(301, 300, "nested", "input", { placeholder: "Nested field" }),
|
||||
];
|
||||
const captured: CapturedViewModel = {
|
||||
viewport: { width: 1200, height: 800 },
|
||||
nodes: mainNodes,
|
||||
iframeNodes: new Map([
|
||||
[10, firstNodes],
|
||||
[20, secondNodes],
|
||||
[210, nestedNodes],
|
||||
]),
|
||||
frameNodes: new Map([
|
||||
["main", mainNodes],
|
||||
["first", firstNodes],
|
||||
["second", secondNodes],
|
||||
["nested", nestedNodes],
|
||||
]),
|
||||
frameOwnerBackendNodeIds: new Map([
|
||||
["first", 10],
|
||||
["second", 20],
|
||||
["nested", 210],
|
||||
]),
|
||||
rootFrameId: "main",
|
||||
excludedBackendNodeIds: new Set(),
|
||||
};
|
||||
const axNode = (
|
||||
nodeId: string,
|
||||
backendDOMNodeId: number,
|
||||
role: string,
|
||||
parentId?: string,
|
||||
name?: string,
|
||||
): CdpAxNode => ({
|
||||
nodeId,
|
||||
backendDOMNodeId,
|
||||
role: { type: "role", value: role },
|
||||
...(parentId ? { parentId } : {}),
|
||||
...(name ? { name: { type: "computedString", value: name } } : {}),
|
||||
});
|
||||
const result = buildFrameVomScene(
|
||||
[
|
||||
{
|
||||
frameId: "main",
|
||||
contextScopeId: "main",
|
||||
target: { tabId: 7 },
|
||||
domNodes: mainNodes,
|
||||
axNodes: [
|
||||
axNode("root", 1, "RootWebArea"),
|
||||
axNode("frame-1", 10, "Iframe", "root", "First"),
|
||||
axNode("frame-2", 20, "Iframe", "root", "Second"),
|
||||
],
|
||||
},
|
||||
{
|
||||
frameId: "first",
|
||||
contextScopeId: "first",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 10,
|
||||
target: { tabId: 7, sessionId: "session-first" },
|
||||
domNodes: firstNodes,
|
||||
axNodes: [
|
||||
axNode("root", 100, "RootWebArea"),
|
||||
axNode("button", 101, "button", "root", "First action"),
|
||||
],
|
||||
},
|
||||
{
|
||||
frameId: "second",
|
||||
contextScopeId: "second",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 20,
|
||||
target: { tabId: 7, sessionId: "session-second" },
|
||||
domNodes: secondNodes,
|
||||
axNodes: [
|
||||
axNode("root", 200, "RootWebArea"),
|
||||
axNode("heading", 201, "heading", "root", "Second heading"),
|
||||
axNode("nested-frame", 210, "Iframe", "root", "Nested"),
|
||||
],
|
||||
},
|
||||
{
|
||||
frameId: "nested",
|
||||
contextScopeId: "nested",
|
||||
parentFrameId: "second",
|
||||
ownerBackendNodeId: 210,
|
||||
target: { tabId: 7, sessionId: "session-nested" },
|
||||
domNodes: nestedNodes,
|
||||
axNodes: [
|
||||
axNode("root", 300, "RootWebArea"),
|
||||
axNode("input", 301, "textbox", "root", "Nested field"),
|
||||
],
|
||||
},
|
||||
],
|
||||
captured,
|
||||
);
|
||||
|
||||
const rendered = renderVom(result);
|
||||
expect(rendered.text).toContain('Iframe "First"');
|
||||
expect(rendered.text).toContain('@e1 button "First action"');
|
||||
expect(rendered.text).toContain('heading "Second heading"');
|
||||
expect(rendered.text).toContain('Iframe "Nested"');
|
||||
expect(rendered.text).toContain('textbox "Nested field"');
|
||||
expect(rendered.refs.find((ref) => ref.backendNodeId === 101)?.frameId).toBe("first");
|
||||
expect(rendered.refs.find((ref) => ref.backendNodeId === 301)?.frameId).toBe("nested");
|
||||
|
||||
const firstAction = result.nodes.find((item) => item.backendNodeId === 101);
|
||||
const nestedInput = result.nodes.find((item) => item.backendNodeId === 301);
|
||||
expect(firstAction).toMatchObject({ frameId: "first", contextScopeId: "first" });
|
||||
expect(nestedInput).toMatchObject({ frameId: "nested", contextScopeId: "nested" });
|
||||
expect(nestedInput?.parentId).toBe(result.nodes.find((item) => item.backendNodeId === 210)?.id);
|
||||
});
|
||||
|
||||
it("does not place a child document at the page root when its frame boundary is unresolved", () => {
|
||||
const childNode: CapturedNode = {
|
||||
backendNodeId: 101,
|
||||
parentBackendNodeId: null,
|
||||
frameId: "child",
|
||||
tag: "button",
|
||||
attrs: {},
|
||||
rect: { x: 0, y: 0, w: 100, h: 30 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
};
|
||||
const captured: CapturedViewModel = {
|
||||
viewport: { width: 800, height: 600 },
|
||||
nodes: [],
|
||||
iframeNodes: new Map(),
|
||||
frameNodes: new Map([["child", [childNode]]]),
|
||||
rootFrameId: "main",
|
||||
excludedBackendNodeIds: new Set(),
|
||||
};
|
||||
|
||||
const result = buildFrameVomScene(
|
||||
[
|
||||
{
|
||||
frameId: "main",
|
||||
contextScopeId: "main",
|
||||
target: { tabId: 7 },
|
||||
domNodes: [],
|
||||
axNodes: [],
|
||||
},
|
||||
{
|
||||
frameId: "child",
|
||||
contextScopeId: "child",
|
||||
parentFrameId: "main",
|
||||
target: { tabId: 7 },
|
||||
domNodes: [childNode],
|
||||
axNodes: [
|
||||
{
|
||||
nodeId: "button",
|
||||
frameId: "child",
|
||||
backendDOMNodeId: 101,
|
||||
role: { type: "role", value: "button" },
|
||||
name: { type: "computedString", value: "Child action" },
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
captured,
|
||||
);
|
||||
|
||||
expect(result.nodes).toEqual([]);
|
||||
});
|
||||
|
||||
it("keeps AX-only frame content attached to its iframe boundary", () => {
|
||||
const mainNodes: CapturedNode[] = [
|
||||
{
|
||||
backendNodeId: 1,
|
||||
parentBackendNodeId: null,
|
||||
frameId: "main",
|
||||
tag: "body",
|
||||
attrs: {},
|
||||
rect: { x: 0, y: 0, w: 800, h: 600 },
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 10,
|
||||
parentBackendNodeId: 1,
|
||||
frameId: "main",
|
||||
tag: "iframe",
|
||||
attrs: { title: "Remote" },
|
||||
rect: { x: 100, y: 100, w: 400, h: 300 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
];
|
||||
const captured: CapturedViewModel = {
|
||||
viewport: { width: 800, height: 600 },
|
||||
nodes: mainNodes,
|
||||
iframeNodes: new Map(),
|
||||
frameNodes: new Map([
|
||||
["main", mainNodes],
|
||||
["child", []],
|
||||
]),
|
||||
rootFrameId: "main",
|
||||
excludedBackendNodeIds: new Set(),
|
||||
};
|
||||
|
||||
const result = buildFrameVomScene(
|
||||
[
|
||||
{
|
||||
frameId: "main",
|
||||
contextScopeId: "main",
|
||||
target: { tabId: 7 },
|
||||
domNodes: mainNodes,
|
||||
axNodes: [
|
||||
{
|
||||
nodeId: "main-root",
|
||||
backendDOMNodeId: 1,
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
},
|
||||
{
|
||||
nodeId: "frame-owner",
|
||||
parentId: "main-root",
|
||||
backendDOMNodeId: 10,
|
||||
role: { type: "role", value: "Iframe" },
|
||||
name: { type: "computedString", value: "Remote" },
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
frameId: "child",
|
||||
contextScopeId: "child",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 10,
|
||||
target: { tabId: 7, sessionId: "child-session" },
|
||||
domNodes: [],
|
||||
axNodes: [
|
||||
{
|
||||
nodeId: "child-root",
|
||||
backendDOMNodeId: 100,
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
},
|
||||
{
|
||||
nodeId: "child-button",
|
||||
parentId: "child-root",
|
||||
backendDOMNodeId: 101,
|
||||
role: { type: "role", value: "button" },
|
||||
name: { type: "computedString", value: "AX fallback action" },
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
captured,
|
||||
);
|
||||
|
||||
const childButton = result.nodes.find((node) => node.backendNodeId === 101);
|
||||
expect(childButton).toMatchObject({ frameId: "child", contextScopeId: "child" });
|
||||
expect(childButton?.parentId).toBe(result.nodes.find((node) => node.backendNodeId === 10)?.id);
|
||||
expect(renderVom(result).text).toContain('@e1 button "AX fallback action"');
|
||||
});
|
||||
|
||||
it("joins AX semantics with captured geometry by backendDOMNodeId", () => {
|
||||
const axNodes: CdpAxNode[] = [
|
||||
{
|
||||
@@ -816,6 +1107,7 @@ describe("buildVomScene", () => {
|
||||
parentId: "2",
|
||||
role: { type: "role", value: "textbox" },
|
||||
name: { type: "x", value: "输入密码" },
|
||||
value: { value: "iframe-secret" },
|
||||
backendDOMNodeId: 202,
|
||||
properties: [{ name: "inputType", value: { value: "password" } }],
|
||||
},
|
||||
@@ -870,6 +1162,10 @@ describe("buildVomScene", () => {
|
||||
|
||||
expect(passwordNodes).toHaveLength(1);
|
||||
expect(passwordNodes[0]).toEqual(expect.objectContaining({ id: 202, sensitive: true }));
|
||||
expect(passwordNodes[0].value).toBeUndefined();
|
||||
const rendered = renderVom(buildVomScene(axNodes, captured)).text;
|
||||
expect(rendered).toContain('textbox "输入密码" [filled] ="•••"');
|
||||
expect(rendered).not.toContain("iframe-secret");
|
||||
});
|
||||
|
||||
it("keeps unnamed iframe controls but skips unnamed iframe links", () => {
|
||||
@@ -1034,7 +1330,7 @@ describe("buildVomScene", () => {
|
||||
expect(rendered).toContain(' @e2 textbox "验证码"');
|
||||
});
|
||||
|
||||
it("does not mark non-password inputs sensitive from password-like labels", () => {
|
||||
it("marks current-password autocomplete as sensitive even for text inputs", () => {
|
||||
const axNodes: CdpAxNode[] = [
|
||||
{
|
||||
nodeId: "1",
|
||||
@@ -1069,7 +1365,7 @@ describe("buildVomScene", () => {
|
||||
expect(buildVomScene(axNodes, captured).nodes[0]).toEqual(
|
||||
expect.objectContaining({
|
||||
id: 10,
|
||||
sensitive: false,
|
||||
sensitive: true,
|
||||
}),
|
||||
);
|
||||
});
|
||||
@@ -1360,11 +1656,13 @@ describe("buildVomScene", () => {
|
||||
|
||||
const scene = buildVomScene(axNodes, captured);
|
||||
expect(scene.nodes.find((n) => n.id === 20)).toEqual(
|
||||
expect.objectContaining({ id: 20, role: "generic", cursor: "pointer" }),
|
||||
expect.objectContaining({ id: 20, role: "button", name: "close", cursor: "pointer" }),
|
||||
);
|
||||
const rendered = renderVom(scene);
|
||||
expect(rendered.text).toContain('@e1 button "close"');
|
||||
expect(rendered.refs).toEqual([{ ref: "e1", backendNodeId: 20 }]);
|
||||
expect(rendered.refs.map(({ ref, backendNodeId }) => ({ ref, backendNodeId }))).toEqual([
|
||||
{ ref: "e1", backendNodeId: 20 },
|
||||
]);
|
||||
});
|
||||
|
||||
it("does not promote a clickable container that wraps a real interactive control", () => {
|
||||
@@ -1434,7 +1732,7 @@ describe("buildVomScene", () => {
|
||||
};
|
||||
|
||||
const scene = buildVomScene(axNodes, captured);
|
||||
expect(scene.nodes.find((n) => n.id === 20)?.role).toBe("generic");
|
||||
expect(scene.nodes.find((n) => n.id === 20)).toBeUndefined();
|
||||
expect(scene.nodes.find((n) => n.id === 30)?.role).toBe("link");
|
||||
});
|
||||
|
||||
@@ -1503,12 +1801,14 @@ describe("buildVomScene", () => {
|
||||
|
||||
const scene = buildVomScene(axNodes, captured);
|
||||
expect(scene.nodes.find((n) => n.id === 20)).toEqual(
|
||||
expect.objectContaining({ id: 20, role: "generic", attrs: { "aria-label": "收藏" } }),
|
||||
expect.objectContaining({ id: 20, role: "button", attrs: { "aria-label": "收藏" } }),
|
||||
);
|
||||
expect(scene.nodes.find((n) => n.id === 30)?.role).toBe("generic");
|
||||
expect(scene.nodes.find((n) => n.id === 30)).toBeUndefined();
|
||||
const rendered = renderVom(scene);
|
||||
expect(rendered.text).toContain('@e1 button "收藏"');
|
||||
expect(rendered.refs).toEqual([{ ref: "e1", backendNodeId: 20 }]);
|
||||
expect(rendered.refs.map(({ ref, backendNodeId }) => ({ ref, backendNodeId }))).toEqual([
|
||||
{ ref: "e1", backendNodeId: 20 },
|
||||
]);
|
||||
});
|
||||
|
||||
it("builds active scope blocks from active aria-controls relationships", () => {
|
||||
@@ -1831,7 +2131,7 @@ describe("buildVomScene", () => {
|
||||
});
|
||||
|
||||
expect(scene.surfaces).toEqual([
|
||||
{ triggerId: 21, triggerAction: "hover", subItems: ["My profile", "Sign out"] },
|
||||
{ triggerId: 20, triggerAction: "hover", subItems: ["My profile", "Sign out"] },
|
||||
]);
|
||||
expect(renderVom(scene).text).toContain(
|
||||
'@e1 button "image" [hover first: My profile | Sign out]',
|
||||
@@ -2609,10 +2909,452 @@ describe("handleSnapshot", () => {
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
function secretsSnapshotReply(secrets: {
|
||||
email: string;
|
||||
password: string;
|
||||
otp: string;
|
||||
card: string;
|
||||
}) {
|
||||
const S = [
|
||||
"html",
|
||||
"body",
|
||||
"input",
|
||||
"position",
|
||||
"static",
|
||||
"pointer-events",
|
||||
"auto",
|
||||
"type",
|
||||
"text",
|
||||
"password",
|
||||
"autocomplete",
|
||||
"one-time-code",
|
||||
"cc-number",
|
||||
"value",
|
||||
secrets.email,
|
||||
secrets.password,
|
||||
secrets.otp,
|
||||
secrets.card,
|
||||
];
|
||||
const i = (s: string) => S.indexOf(s);
|
||||
const style = [i("static"), i("auto"), i("auto")];
|
||||
return {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1, 1, 1, 1],
|
||||
nodeName: [i("html"), i("body"), i("input"), i("input"), i("input"), i("input")],
|
||||
backendNodeId: [10, 11, 12, 13, 14, 15],
|
||||
attributes: [
|
||||
[],
|
||||
[],
|
||||
[i("type"), i("text"), i("value"), i(secrets.email)],
|
||||
[i("type"), i("password"), i("value"), i(secrets.password)],
|
||||
[
|
||||
i("type"),
|
||||
i("text"),
|
||||
i("autocomplete"),
|
||||
i("one-time-code"),
|
||||
i("value"),
|
||||
i(secrets.otp),
|
||||
],
|
||||
[
|
||||
i("type"),
|
||||
i("text"),
|
||||
i("autocomplete"),
|
||||
i("cc-number"),
|
||||
i("value"),
|
||||
i(secrets.card),
|
||||
],
|
||||
],
|
||||
inputValue: {
|
||||
index: [2, 3, 4, 5],
|
||||
value: [i(secrets.email), i(secrets.password), i(secrets.otp), i(secrets.card)],
|
||||
},
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [1, 2, 3, 4, 5],
|
||||
styles: [style, style, style, style, style],
|
||||
bounds: [
|
||||
[0, 0, 1000, 800],
|
||||
[10, 10, 200, 32],
|
||||
[10, 50, 200, 32],
|
||||
[10, 90, 200, 32],
|
||||
[10, 130, 200, 32],
|
||||
],
|
||||
paintOrders: [0, 1, 2, 3, 4],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
const VP_METRICS = {
|
||||
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
|
||||
};
|
||||
|
||||
function makeFrameAwareDeps() {
|
||||
const strings = ["html", "body", "iframe", "button", "title", "Remote", "static", "auto"];
|
||||
const i = (value: string) => strings.indexOf(value);
|
||||
const styles = [i("static"), i("auto"), i("auto")];
|
||||
const snapshot = {
|
||||
strings,
|
||||
documents: [
|
||||
{
|
||||
frameId: "main",
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1],
|
||||
nodeName: [i("html"), i("body"), i("iframe")],
|
||||
backendNodeId: [10, 11, 12],
|
||||
attributes: [[], [], [i("title"), i("Remote")]],
|
||||
contentDocumentIndex: { index: [2], value: [1] },
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [0, 1, 2],
|
||||
styles: [styles, styles, styles],
|
||||
bounds: [
|
||||
[0, 0, 1000, 800],
|
||||
[0, 0, 1000, 800],
|
||||
[100, 100, 400, 300],
|
||||
],
|
||||
paintOrders: [0, 0, 1],
|
||||
},
|
||||
},
|
||||
{
|
||||
frameId: "child",
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1],
|
||||
nodeName: [i("html"), i("body"), i("button")],
|
||||
backendNodeId: [20, 21, 22],
|
||||
attributes: [[], [], []],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [0, 1, 2],
|
||||
styles: [styles, styles, styles],
|
||||
bounds: [
|
||||
[0, 0, 400, 300],
|
||||
[0, 0, 400, 300],
|
||||
[20, 30, 120, 40],
|
||||
],
|
||||
paintOrders: [0, 0, 1],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
const mainAx: CdpAxNode[] = [
|
||||
{
|
||||
nodeId: "main-root",
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
backendDOMNodeId: 11,
|
||||
childIds: ["frame-owner"],
|
||||
},
|
||||
{
|
||||
nodeId: "frame-owner",
|
||||
parentId: "main-root",
|
||||
role: { type: "role", value: "Iframe" },
|
||||
name: { type: "computedString", value: "Remote" },
|
||||
backendDOMNodeId: 12,
|
||||
},
|
||||
];
|
||||
const childAx: CdpAxNode[] = [
|
||||
{
|
||||
nodeId: "child-root",
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
backendDOMNodeId: 21,
|
||||
childIds: ["child-button"],
|
||||
},
|
||||
{
|
||||
nodeId: "child-button",
|
||||
parentId: "child-root",
|
||||
role: { type: "role", value: "button" },
|
||||
name: { type: "computedString", value: "Frame action" },
|
||||
backendDOMNodeId: 22,
|
||||
},
|
||||
];
|
||||
const send = vi.fn(async (_tabId: number, method: string) => {
|
||||
if (method === "Page.getLayoutMetrics") return VP_METRICS;
|
||||
if (method === "DOMSnapshot.enable" || method === "Accessibility.enable") return {};
|
||||
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
|
||||
if (method === "Accessibility.getFullAXTree") return { nodes: mainAx };
|
||||
throw new Error(`unexpected root CDP method ${method}`);
|
||||
});
|
||||
const sendToTarget = vi.fn(async (_target, method: string) => {
|
||||
if (method === "Accessibility.enable") return {};
|
||||
if (method === "Accessibility.getFullAXTree") return { nodes: childAx };
|
||||
throw new Error(`unexpected child CDP method ${method}`);
|
||||
});
|
||||
const cdp = {
|
||||
send: send as unknown as CdpRunner["send"],
|
||||
sendToTarget: sendToTarget as unknown as NonNullable<CdpRunner["sendToTarget"]>,
|
||||
getFrameGraph: vi.fn(async () => ({
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "child",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 12,
|
||||
target: { tabId: 4, sessionId: "child-session" },
|
||||
},
|
||||
],
|
||||
})),
|
||||
trackSessionTab: vi.fn(),
|
||||
} satisfies CdpRunner;
|
||||
return {
|
||||
cdp,
|
||||
tabsApi: {
|
||||
get: vi.fn(
|
||||
async (tabId: number) => ({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab,
|
||||
),
|
||||
query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
it("exposes the frame-aware observation result for recording", async () => {
|
||||
const ax: CdpAxNode[] = [
|
||||
{
|
||||
nodeId: "root",
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
backendDOMNodeId: 11,
|
||||
childIds: ["password"],
|
||||
},
|
||||
{
|
||||
nodeId: "password",
|
||||
parentId: "root",
|
||||
role: { type: "role", value: "textbox" },
|
||||
name: { type: "computedString", value: "Password" },
|
||||
value: { value: "hunter2" },
|
||||
backendDOMNodeId: 13,
|
||||
},
|
||||
];
|
||||
const deps = makeOverlayDeps(ax, loginSnapshotReply(), VP_METRICS);
|
||||
|
||||
const result = await captureVomObservation(deps.cdp, 4, "https://example.com", {
|
||||
redactValues: true,
|
||||
});
|
||||
|
||||
expect(result.text).not.toContain("hunter2");
|
||||
expect(result.text).toContain("•••");
|
||||
expect(result.refs[0]).toMatchObject({
|
||||
backendNodeId: 13,
|
||||
role: "textbox",
|
||||
name: "Password",
|
||||
line: expect.any(Number),
|
||||
});
|
||||
expect(result.rootFrameId).toBe("root");
|
||||
expect(result).not.toHaveProperty("frameDocuments");
|
||||
expect(result.frames).toEqual([{ frameId: "root", target: { tabId: 4 } }]);
|
||||
expect(result.matchNodes.find((node) => node.backendNodeId === 13)).toMatchObject({
|
||||
frameId: "root",
|
||||
backendNodeId: 13,
|
||||
tag: "input",
|
||||
rect: { x: 400, y: 300, w: 200, h: 40 },
|
||||
localRect: { x: 400, y: 300, w: 200, h: 40 },
|
||||
});
|
||||
});
|
||||
|
||||
it("decorates live observations with active controlled content", async () => {
|
||||
const strings = [
|
||||
"html",
|
||||
"body",
|
||||
"button",
|
||||
"div",
|
||||
"#text",
|
||||
"role",
|
||||
"tab",
|
||||
"aria-selected",
|
||||
"true",
|
||||
"aria-controls",
|
||||
"reviews-panel",
|
||||
"id",
|
||||
"static",
|
||||
"auto",
|
||||
"visible",
|
||||
"1",
|
||||
"A detailed review",
|
||||
];
|
||||
const index = (value: string) => strings.indexOf(value);
|
||||
const style = [index("static"), index("auto"), index("auto"), index("visible"), index("1")];
|
||||
const snapshot = {
|
||||
strings,
|
||||
documents: [
|
||||
{
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1, 1, 3],
|
||||
nodeName: [index("html"), index("body"), index("button"), index("div"), index("#text")],
|
||||
backendNodeId: [10, 11, 12, 13, 14],
|
||||
attributes: [
|
||||
[],
|
||||
[],
|
||||
[
|
||||
index("role"),
|
||||
index("tab"),
|
||||
index("aria-selected"),
|
||||
index("true"),
|
||||
index("aria-controls"),
|
||||
index("reviews-panel"),
|
||||
],
|
||||
[index("id"), index("reviews-panel")],
|
||||
[],
|
||||
],
|
||||
nodeValue: [-1, -1, -1, -1, index("A detailed review")],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [0, 1, 2, 3, 4],
|
||||
styles: [style, style, style, style, style],
|
||||
bounds: [
|
||||
[0, 0, 1000, 800],
|
||||
[0, 0, 1000, 800],
|
||||
[20, 20, 120, 40],
|
||||
[20, 80, 500, 200],
|
||||
[20, 80, 200, 20],
|
||||
],
|
||||
paintOrders: [0, 0, 1, 1, 1],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
const ax: CdpAxNode[] = [
|
||||
{
|
||||
nodeId: "root",
|
||||
backendDOMNodeId: 11,
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
childIds: ["reviews"],
|
||||
},
|
||||
{
|
||||
nodeId: "reviews",
|
||||
parentId: "root",
|
||||
backendDOMNodeId: 12,
|
||||
role: { type: "role", value: "tab" },
|
||||
name: { type: "computedString", value: "Reviews" },
|
||||
properties: [
|
||||
{ name: "selected", value: { value: true } },
|
||||
{ name: "controls", value: { value: "reviews-panel" } },
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
const result = await captureVomObservation(
|
||||
makeOverlayDeps(ax, snapshot, VP_METRICS).cdp,
|
||||
4,
|
||||
"https://example.com",
|
||||
);
|
||||
|
||||
expect(result.text).toContain('@e1 tab "Reviews"');
|
||||
expect(result.text).toContain("[§ active: Reviews]");
|
||||
expect(result.text).toContain("A detailed review");
|
||||
});
|
||||
|
||||
it("does not leak form secrets through the record-safe observation payload", async () => {
|
||||
const secrets = {
|
||||
email: "user@example.com",
|
||||
password: "hunter2",
|
||||
otp: "847291",
|
||||
card: "4111111111111111",
|
||||
};
|
||||
const ax: CdpAxNode[] = [
|
||||
{
|
||||
nodeId: "root",
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
backendDOMNodeId: 11,
|
||||
childIds: ["email", "password", "otp", "card"],
|
||||
},
|
||||
{
|
||||
nodeId: "email",
|
||||
parentId: "root",
|
||||
role: { type: "role", value: "textbox" },
|
||||
name: { type: "computedString", value: "Email" },
|
||||
value: { value: secrets.email },
|
||||
backendDOMNodeId: 12,
|
||||
},
|
||||
{
|
||||
nodeId: "password",
|
||||
parentId: "root",
|
||||
role: { type: "role", value: "textbox" },
|
||||
name: { type: "computedString", value: "Password" },
|
||||
value: { value: secrets.password },
|
||||
backendDOMNodeId: 13,
|
||||
},
|
||||
{
|
||||
nodeId: "otp",
|
||||
parentId: "root",
|
||||
role: { type: "role", value: "textbox" },
|
||||
name: { type: "computedString", value: "Code" },
|
||||
value: { value: secrets.otp },
|
||||
backendDOMNodeId: 14,
|
||||
},
|
||||
{
|
||||
nodeId: "card",
|
||||
parentId: "root",
|
||||
role: { type: "role", value: "textbox" },
|
||||
name: { type: "computedString", value: "Card" },
|
||||
value: { value: secrets.card },
|
||||
backendDOMNodeId: 15,
|
||||
},
|
||||
];
|
||||
const deps = makeOverlayDeps(ax, secretsSnapshotReply(secrets), VP_METRICS);
|
||||
|
||||
const result = await captureVomObservation(deps.cdp, 4, "https://example.com", {
|
||||
redactValues: true,
|
||||
});
|
||||
const dumped = JSON.stringify(result);
|
||||
|
||||
expect(result).not.toHaveProperty("frameDocuments");
|
||||
expect(dumped).not.toContain(secrets.email);
|
||||
expect(dumped).not.toContain(secrets.password);
|
||||
expect(dumped).not.toContain(secrets.otp);
|
||||
expect(dumped).not.toContain(secrets.card);
|
||||
expect(dumped).not.toContain("formValue");
|
||||
expect(dumped).not.toContain("formDefaultValue");
|
||||
expect(dumped).not.toContain("axNodes");
|
||||
expect(dumped).not.toContain("domNodes");
|
||||
expect(dumped).not.toContain("attrs");
|
||||
expect(result.text).toContain("•••");
|
||||
});
|
||||
|
||||
it("keeps frames and rendered refs on the same frame identity", async () => {
|
||||
const deps = makeFrameAwareDeps();
|
||||
|
||||
const result = await captureVomObservation(deps.cdp, 4, "https://example.com");
|
||||
|
||||
expect(result.rootFrameId).toBe("main");
|
||||
expect(result.frames).toHaveLength(2);
|
||||
expect(result.frames.find((frame) => frame.frameId === "child")).toEqual({
|
||||
frameId: "child",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 12,
|
||||
target: { tabId: 4, sessionId: "child-session" },
|
||||
});
|
||||
expect(result.matchNodes.find((node) => node.backendNodeId === 22)).toMatchObject({
|
||||
frameId: "child",
|
||||
tag: "button",
|
||||
localRect: { x: 20, y: 30, w: 120, h: 40 },
|
||||
});
|
||||
expect(result.refs.find((ref) => ref.backendNodeId === 22)).toMatchObject({
|
||||
frameId: "child",
|
||||
name: "Frame action",
|
||||
});
|
||||
});
|
||||
|
||||
it("stores iframe refs with their owning CDP session", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
const deps = makeFrameAwareDeps();
|
||||
|
||||
const result = await handleSnapshot(sm, { session_id: "aa11" }, deps);
|
||||
|
||||
if ("code" in result) throw new Error(`unexpected error: ${JSON.stringify(result)}`);
|
||||
expect(result.text).toContain('@e1 button "Frame action"');
|
||||
const frameRef = [...ctx.refStore.entries()].find(([, entry]) => entry.backendNodeId === 22);
|
||||
expect(frameRef?.[1]).toMatchObject({
|
||||
tabId: 4,
|
||||
frameId: "child",
|
||||
cdpSessionId: "child-session",
|
||||
});
|
||||
});
|
||||
|
||||
it("renders a blocking login overlay as the focused top layer", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
await sm.start("aa11");
|
||||
@@ -3123,6 +3865,32 @@ describe("handleGetHtml", () => {
|
||||
expect(deps.send).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("reads a frame ref through its child CDP session", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.refStore.set("e7", 4242, {
|
||||
tabId: 4,
|
||||
frameId: "child-frame",
|
||||
cdpSessionId: "child-session",
|
||||
});
|
||||
const deps = makeDeps({});
|
||||
const sendToTarget = vi.fn(async (target, method, params) => {
|
||||
expect(target).toEqual({ tabId: 4, sessionId: "child-session" });
|
||||
expect(method).toBe("DOM.getOuterHTML");
|
||||
expect(params).toEqual({ backendNodeId: 4242 });
|
||||
return { outerHTML: "<button>frame</button>" };
|
||||
});
|
||||
(deps.cdp as CdpRunner).sendToTarget = sendToTarget as unknown as NonNullable<
|
||||
CdpRunner["sendToTarget"]
|
||||
>;
|
||||
|
||||
const res = await handleGetHtml(sm, { session_id: "aa11", ref: "@e7" }, deps);
|
||||
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
expect(res.html).toBe("<button>frame</button>");
|
||||
expect(deps.send).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("returns not_found when a ref belongs to another tab", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -66,6 +66,52 @@ function makeApis(
|
||||
}
|
||||
|
||||
describe("handleSessionStop with auto-return", () => {
|
||||
it("leaves the session untouched when cancellation arrived before teardown", async () => {
|
||||
const aw = fakeAgentWindow([100]);
|
||||
const sm = new SessionManager({ agentWindow: aw });
|
||||
await sm.start("aa11");
|
||||
const cdp = { detachSession: vi.fn(async () => {}) };
|
||||
const controller = new AbortController();
|
||||
controller.abort();
|
||||
|
||||
const res = await handleSessionStop(
|
||||
sm,
|
||||
{ session_id: "aa11" },
|
||||
{ cdp, signal: controller.signal },
|
||||
);
|
||||
|
||||
expect(res).toMatchObject({ code: "cancelled" });
|
||||
expect(sm.has("aa11")).toBe(true);
|
||||
expect(cdp.detachSession).not.toHaveBeenCalled();
|
||||
expect(aw.remove).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("does not close the Agent Window when cancellation lands during CDP detach", async () => {
|
||||
const aw = fakeAgentWindow([100]);
|
||||
const sm = new SessionManager({ agentWindow: aw });
|
||||
await sm.start("aa11");
|
||||
const controller = new AbortController();
|
||||
let finishDetach: () => void = () => {};
|
||||
const detach = new Promise<void>((resolve) => {
|
||||
finishDetach = resolve;
|
||||
});
|
||||
const cdp = { detachSession: vi.fn(() => detach) };
|
||||
|
||||
const stopping = handleSessionStop(
|
||||
sm,
|
||||
{ session_id: "aa11" },
|
||||
{ cdp, signal: controller.signal },
|
||||
);
|
||||
await vi.waitFor(() => expect(cdp.detachSession).toHaveBeenCalledWith("aa11"));
|
||||
controller.abort();
|
||||
finishDetach();
|
||||
|
||||
const res = await stopping;
|
||||
expect(res).toMatchObject({ code: "cancelled" });
|
||||
expect(sm.has("aa11")).toBe(true);
|
||||
expect(aw.remove).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("returns every borrowed tab and closes the Agent Window in the right order", async () => {
|
||||
const aw = fakeAgentWindow([100]);
|
||||
const sm = new SessionManager({ agentWindow: aw });
|
||||
|
||||
@@ -46,6 +46,25 @@ describe("lookupSnapshotRef", () => {
|
||||
});
|
||||
|
||||
describe("resolveSnapshotRef", () => {
|
||||
it("preserves the frame and child session needed to route iframe refs", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.refStore.set("e3", 1234, {
|
||||
tabId: 4,
|
||||
frameId: "child-frame",
|
||||
cdpSessionId: "child-session",
|
||||
});
|
||||
|
||||
const expected = {
|
||||
backendNodeId: 1234,
|
||||
refKey: "e3",
|
||||
frameId: "child-frame",
|
||||
cdpSessionId: "child-session",
|
||||
};
|
||||
expect(lookupSnapshotRef(ctx, "@e3", 4)).toEqual(expected);
|
||||
expect(resolveSnapshotRef(ctx, "@e3", 4)).toEqual(expected);
|
||||
});
|
||||
|
||||
it("returns not_found for unknown ref", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
|
||||
@@ -250,9 +250,10 @@ const DEFAULT_NOTIFICATION_COPY: BorrowNotificationCopy = {
|
||||
// URL injection-eligibility test
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const CHROME_WEB_STORE_RES = [
|
||||
const EXTENSION_STORE_RES = [
|
||||
/^https:\/\/chrome\.google\.com\/webstore/i,
|
||||
/^https:\/\/chromewebstore\.google\.com/i,
|
||||
/^https:\/\/microsoftedge\.microsoft\.com\/addons/i,
|
||||
];
|
||||
|
||||
/**
|
||||
@@ -264,14 +265,15 @@ const CHROME_WEB_STORE_RES = [
|
||||
* - `ftp://` no longer hosts content scripts reliably in modern Chrome.
|
||||
* - `about:` / `chrome:` / `chrome-extension:` / `edge:` / `devtools:` /
|
||||
* `view-source:` / `data:` / `blob:` are blocked by the platform.
|
||||
* - The Chrome Web Store (both legacy and new domains) is also blocked.
|
||||
* - Extension storefronts are blocked by their own browser: the Chrome Web
|
||||
* Store (legacy and new domains) under Chrome, and Edge Add-ons under Edge.
|
||||
*
|
||||
* Mirrors the implicit scheme list documented in
|
||||
* https://developer.chrome.com/docs/extensions/develop/concepts/match-patterns.
|
||||
*/
|
||||
export function isInjectableContentScriptUrl(url: string | undefined): boolean {
|
||||
if (!url) return false;
|
||||
for (const re of CHROME_WEB_STORE_RES) {
|
||||
for (const re of EXTENSION_STORE_RES) {
|
||||
if (re.test(url)) return false;
|
||||
}
|
||||
return /^https?:\/\//i.test(url);
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
export {
|
||||
type CaptureVomObservationOptions,
|
||||
captureVomObservation,
|
||||
} from "./observation";
|
||||
export {
|
||||
type CaptureVomFrame,
|
||||
type CaptureVomMatchNode,
|
||||
type CaptureVomObservationResult,
|
||||
} from "./vom/record-safe-observation";
|
||||
@@ -51,7 +51,12 @@ import {
|
||||
handleScreenshot,
|
||||
handleSnapshot,
|
||||
} from "./observation";
|
||||
import { handleRecordAwait, handleRecordStart, handleRecordStop } from "./record";
|
||||
import {
|
||||
handleRecordAwait,
|
||||
handleRecordStart,
|
||||
handleRecordStop,
|
||||
type RecordRuntimeDeps,
|
||||
} from "./record";
|
||||
import {
|
||||
handleSessionStart,
|
||||
handleSessionStop,
|
||||
@@ -99,6 +104,7 @@ export interface DispatcherDeps {
|
||||
transport: Transport;
|
||||
sessions: SessionManager;
|
||||
cdp?: DispatcherCdpRunner;
|
||||
recording?: RecordRuntimeDeps;
|
||||
/**
|
||||
* Invoked whenever a dispatched RPC may have changed the live
|
||||
* session set (currently `tool.session_start` and
|
||||
@@ -134,6 +140,7 @@ export class ToolDispatcher {
|
||||
private readonly transport: Transport;
|
||||
private readonly sessions: SessionManager;
|
||||
private readonly cdp?: DispatcherCdpRunner;
|
||||
private readonly recording?: RecordRuntimeDeps;
|
||||
private readonly onSessionsChanged?: () => void;
|
||||
private readonly onBrowserControlResumed?: (sessionId: string) => void;
|
||||
private readonly approveBorrow?: BorrowConfirmationApprover;
|
||||
@@ -153,6 +160,7 @@ export class ToolDispatcher {
|
||||
this.transport = deps.transport;
|
||||
this.sessions = deps.sessions;
|
||||
this.cdp = deps.cdp;
|
||||
this.recording = deps.recording;
|
||||
this.onSessionsChanged = deps.onSessionsChanged;
|
||||
this.onBrowserControlResumed = deps.onBrowserControlResumed;
|
||||
this.approveBorrow = deps.approveBorrow;
|
||||
@@ -290,6 +298,7 @@ export class ToolDispatcher {
|
||||
await this.releaseHoverLatch((req.params as SessionStopParams).session_id);
|
||||
return handleSessionStop(this.sessions, req.params as SessionStopParams, {
|
||||
cdp: this.cdp,
|
||||
signal,
|
||||
});
|
||||
}
|
||||
case "tool.tab_list":
|
||||
@@ -530,44 +539,26 @@ export class ToolDispatcher {
|
||||
signal,
|
||||
});
|
||||
case "tool.record_start":
|
||||
return handleRecordStart(this.sessions, req.params as RecordStartParams, {
|
||||
tabsApi: chromeTabsApi,
|
||||
sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
|
||||
bypassOverlay: async (tabId, enabled) => {
|
||||
try {
|
||||
await chrome.tabs.sendMessage(tabId, {
|
||||
type: OVERLAY_AUTOMATION_BYPASS,
|
||||
enabled,
|
||||
});
|
||||
} catch {
|
||||
// Content script may be unavailable on restricted pages.
|
||||
}
|
||||
},
|
||||
...(this.cdp ? { cdp: this.cdp } : {}),
|
||||
signal,
|
||||
});
|
||||
return this.recording
|
||||
? handleRecordStart(this.sessions, req.params as RecordStartParams, {
|
||||
...this.recording,
|
||||
signal,
|
||||
})
|
||||
: recordingRuntimeUnavailable();
|
||||
case "tool.record_stop":
|
||||
return handleRecordStop(this.sessions, req.params as RecordStopParams, {
|
||||
tabsApi: chromeTabsApi,
|
||||
sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
|
||||
bypassOverlay: async (tabId, enabled) => {
|
||||
try {
|
||||
await chrome.tabs.sendMessage(tabId, {
|
||||
type: OVERLAY_AUTOMATION_BYPASS,
|
||||
enabled,
|
||||
});
|
||||
} catch {
|
||||
// Content script may be unavailable on restricted pages.
|
||||
}
|
||||
},
|
||||
signal,
|
||||
});
|
||||
return this.recording
|
||||
? handleRecordStop(this.sessions, req.params as RecordStopParams, {
|
||||
...this.recording,
|
||||
signal,
|
||||
})
|
||||
: recordingRuntimeUnavailable();
|
||||
case "tool.record_await":
|
||||
return handleRecordAwait(this.sessions, req.params as RecordAwaitParams, {
|
||||
tabsApi: chromeTabsApi,
|
||||
sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
|
||||
signal,
|
||||
});
|
||||
return this.recording
|
||||
? handleRecordAwait(this.sessions, req.params as RecordAwaitParams, {
|
||||
...this.recording,
|
||||
signal,
|
||||
})
|
||||
: recordingRuntimeUnavailable();
|
||||
default:
|
||||
return {
|
||||
code: "unknown_method",
|
||||
@@ -703,6 +694,13 @@ function isRpcError(v: unknown): v is RpcError {
|
||||
);
|
||||
}
|
||||
|
||||
function recordingRuntimeUnavailable(): RpcError {
|
||||
return {
|
||||
code: "protocol_error",
|
||||
message: "recording runtime is unavailable",
|
||||
};
|
||||
}
|
||||
|
||||
function sessionIdForBrowserControlMethod(req: RequestFrame): string | null {
|
||||
switch (req.method) {
|
||||
case "tool.tab_create":
|
||||
|
||||
@@ -1,9 +1,15 @@
|
||||
// Shared CDP helpers for scrolling nodes into view and computing
|
||||
// viewport-space bounding rectangles. Used by interaction tools (click
|
||||
// centre) and observation tools (element screenshot clip).
|
||||
// Resolve and scroll nodes inside one CDP target. Cross-frame projection
|
||||
// belongs to frame-geometry.ts.
|
||||
|
||||
import type { RpcError } from "@/transport/types";
|
||||
import { rpcError } from "./errors";
|
||||
import {
|
||||
parseCdpQuad,
|
||||
polygonBounds,
|
||||
type Region,
|
||||
rectPolygon,
|
||||
type ViewportRect,
|
||||
} from "./geometry";
|
||||
import { type CdpRunner, isRpcError } from "./shared";
|
||||
|
||||
const ELEMENT_NOT_VISIBLE_MESSAGE =
|
||||
@@ -13,45 +19,6 @@ function elementNotVisibleError(): RpcError {
|
||||
return rpcError("permission_denied", "element_not_visible", ELEMENT_NOT_VISIBLE_MESSAGE);
|
||||
}
|
||||
|
||||
/** Viewport-space axis-aligned bounding box for CDP `Page.captureScreenshot` clip. */
|
||||
export interface ViewportRect {
|
||||
x: number;
|
||||
y: number;
|
||||
width: number;
|
||||
height: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute the axis-aligned bounding box of an 8-double CDP content
|
||||
* quad / box-model polygon. Exported for unit tests.
|
||||
*/
|
||||
export function quadBoundingRect(quad: number[]): ViewportRect | null {
|
||||
if (quad.length !== 8) return null;
|
||||
const xs = [quad[0], quad[2], quad[4], quad[6]];
|
||||
const ys = [quad[1], quad[3], quad[5], quad[7]];
|
||||
const minX = Math.min(...xs);
|
||||
const maxX = Math.max(...xs);
|
||||
const minY = Math.min(...ys);
|
||||
const maxY = Math.max(...ys);
|
||||
const width = maxX - minX;
|
||||
const height = maxY - minY;
|
||||
if (width <= 0 || height <= 0) return null;
|
||||
return { x: minX, y: minY, width, height };
|
||||
}
|
||||
|
||||
function rectToQuad(rect: ViewportRect): number[] {
|
||||
return [
|
||||
rect.x,
|
||||
rect.y,
|
||||
rect.x + rect.width,
|
||||
rect.y,
|
||||
rect.x + rect.width,
|
||||
rect.y + rect.height,
|
||||
rect.x,
|
||||
rect.y + rect.height,
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve `backendNodeId` → CDP `objectId` so we can invoke
|
||||
* `Runtime.callFunctionOn` against the live JS object.
|
||||
@@ -112,48 +79,24 @@ export async function scrollNodeIntoView(
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Viewport-space bounding box for `backendNodeId`. Prefers
|
||||
* `DOM.getContentQuads`, then `DOM.getBoxModel`, then the union of
|
||||
* visible descendant client rects for zero-size overflow containers.
|
||||
*/
|
||||
export async function nodeBoundingRect(
|
||||
function validCdpRegion(quads: number[][] | undefined): Region {
|
||||
return (quads ?? []).flatMap((raw) => {
|
||||
const polygon = parseCdpQuad(raw);
|
||||
return polygon && polygonBounds(polygon) ? [polygon] : [];
|
||||
});
|
||||
}
|
||||
|
||||
async function resolveVisibleContentRegion(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
backendNodeId: number,
|
||||
): Promise<ViewportRect | RpcError> {
|
||||
const polygon = await visibleContentPolygon(cdp, tabId, backendNodeId);
|
||||
if (isRpcError(polygon)) return polygon;
|
||||
// visibleContentPolygon only returns quads with positive area.
|
||||
return quadBoundingRect(polygon)!;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute the centroid of a CDP content quad. Quads are reported as
|
||||
* 8 doubles in clockwise order: (x1, y1, x2, y2, x3, y3, x4, y4).
|
||||
*/
|
||||
export function quadCentre(quad: number[]): { x: number; y: number } {
|
||||
const x = (quad[0] + quad[4]) / 2;
|
||||
const y = (quad[1] + quad[5]) / 2;
|
||||
return { x, y };
|
||||
}
|
||||
|
||||
/** Centre of a `DOM.getBoxModel` `content` polygon (also 8 doubles). */
|
||||
export function boxCentre(box: number[]): { x: number; y: number } {
|
||||
return quadCentre(box);
|
||||
}
|
||||
|
||||
async function visibleContentPolygon(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
backendNodeId: number,
|
||||
): Promise<number[] | RpcError> {
|
||||
): Promise<Region | RpcError> {
|
||||
try {
|
||||
const quads = await cdp.send<{ quads?: number[][] }>(tabId, "DOM.getContentQuads", {
|
||||
backendNodeId,
|
||||
});
|
||||
const quad = quads.quads?.find((q) => q.length === 8 && quadBoundingRect(q) !== null);
|
||||
if (quad) return quad;
|
||||
const visible = validCdpRegion(quads.quads);
|
||||
if (visible.length > 0) return visible;
|
||||
} catch (err) {
|
||||
console.debug("[bsk element-geometry] getContentQuads failed", err);
|
||||
}
|
||||
@@ -162,15 +105,34 @@ async function visibleContentPolygon(
|
||||
backendNodeId,
|
||||
});
|
||||
const content = box.model?.content;
|
||||
if (content && content.length === 8 && quadBoundingRect(content)) return content;
|
||||
const fallback = validCdpRegion(content ? [content] : undefined);
|
||||
if (fallback.length > 0) return fallback;
|
||||
} catch (err) {
|
||||
console.debug("[bsk element-geometry] getBoxModel failed", err);
|
||||
}
|
||||
const descendantRect = await descendantBoundingRect(cdp, tabId, backendNodeId);
|
||||
if (descendantRect) return rectToQuad(descendantRect);
|
||||
if (descendantRect) {
|
||||
return [
|
||||
rectPolygon({
|
||||
x: descendantRect.x,
|
||||
y: descendantRect.y,
|
||||
w: descendantRect.width,
|
||||
h: descendantRect.height,
|
||||
}),
|
||||
];
|
||||
}
|
||||
return elementNotVisibleError();
|
||||
}
|
||||
|
||||
/** Resolve every visible content region in the node's target-local viewport. */
|
||||
export async function nodeContentRegion(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
backendNodeId: number,
|
||||
): Promise<Region | RpcError> {
|
||||
return resolveVisibleContentRegion(cdp, tabId, backendNodeId);
|
||||
}
|
||||
|
||||
async function descendantBoundingRect(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
@@ -241,20 +203,3 @@ function parseViewportRect(value: unknown): ViewportRect | null {
|
||||
if (![x, y, width, height].every(Number.isFinite) || width <= 0 || height <= 0) return null;
|
||||
return { x, y, width, height };
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the click point for `backendNodeId`. Prefers
|
||||
* `DOM.getContentQuads` (richer for rotated / transformed elements);
|
||||
* falls back to `DOM.getBoxModel`, then visible descendant bounds.
|
||||
* Returns `permission_denied` with "element not visible" when every
|
||||
* geometry path comes back empty / fails.
|
||||
*/
|
||||
export async function nodeCentre(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
backendNodeId: number,
|
||||
): Promise<{ x: number; y: number } | RpcError> {
|
||||
const polygon = await visibleContentPolygon(cdp, tabId, backendNodeId);
|
||||
if (isRpcError(polygon)) return polygon;
|
||||
return quadCentre(polygon);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,290 @@
|
||||
import { type CdpFrame, type CdpFrameGraph, type CdpTarget } from "@/browser-driver/frame-graph";
|
||||
import type { RpcError } from "@/transport/types";
|
||||
import { nodeContentRegion, scrollNodeIntoView } from "./element-geometry";
|
||||
import {
|
||||
clipPolygon,
|
||||
type GeometryProjection,
|
||||
type Point,
|
||||
type Polygon,
|
||||
type ProjectiveEdge,
|
||||
parseCdpQuad,
|
||||
polygonArea,
|
||||
polygonCentroid,
|
||||
projectRegionToViewport,
|
||||
type Quad,
|
||||
type Region,
|
||||
rectPolygon,
|
||||
regionBounds,
|
||||
type Size,
|
||||
type ViewportRect,
|
||||
} from "./geometry";
|
||||
import { type CdpRunner, cdpRunnerForTarget, isRpcError, sendToCdpTarget } from "./shared";
|
||||
|
||||
export interface NodeAddress {
|
||||
target: CdpTarget;
|
||||
backendNodeId: number;
|
||||
frameId?: string;
|
||||
}
|
||||
|
||||
export interface ResolvedNodeGeometry {
|
||||
topVisibleRegions: Region;
|
||||
topBounds: ViewportRect;
|
||||
actionPoint: Point;
|
||||
}
|
||||
|
||||
function geometryError(message: string): RpcError {
|
||||
return { code: "cdp_failed", message };
|
||||
}
|
||||
|
||||
function frameMap(graph: CdpFrameGraph): Map<string, CdpFrame> {
|
||||
return new Map(graph.frames.map((frame) => [frame.frameId, frame]));
|
||||
}
|
||||
|
||||
function targetRootFrame(byId: Map<string, CdpFrame>, frame: CdpFrame): CdpFrame | null {
|
||||
const seen = new Set<string>();
|
||||
let current = frame;
|
||||
while (current.parentFrameId) {
|
||||
if (seen.has(current.frameId)) return null;
|
||||
seen.add(current.frameId);
|
||||
const parent = byId.get(current.parentFrameId);
|
||||
if (!parent || parent.target.sessionId !== current.target.sessionId) break;
|
||||
current = parent;
|
||||
}
|
||||
return current;
|
||||
}
|
||||
|
||||
function frameAncestry(graph: CdpFrameGraph, frameId: string): CdpFrame[] | null {
|
||||
const byId = frameMap(graph);
|
||||
const path: CdpFrame[] = [];
|
||||
const seen = new Set<string>();
|
||||
let current = byId.get(frameId);
|
||||
if (!current) return null;
|
||||
while (current.parentFrameId) {
|
||||
if (seen.has(current.frameId)) return null;
|
||||
seen.add(current.frameId);
|
||||
path.push(current);
|
||||
const parent = byId.get(current.parentFrameId);
|
||||
if (!parent) return null;
|
||||
current = parent;
|
||||
}
|
||||
return path;
|
||||
}
|
||||
|
||||
async function targetViewport(cdp: CdpRunner, target: CdpTarget): Promise<Size | null> {
|
||||
const metrics = await sendToCdpTarget<{
|
||||
cssLayoutViewport?: { clientWidth?: number; clientHeight?: number };
|
||||
layoutViewport?: { clientWidth?: number; clientHeight?: number };
|
||||
}>(cdp, target, "Page.getLayoutMetrics", {});
|
||||
const viewport = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
|
||||
const width = viewport.clientWidth ?? 0;
|
||||
const height = viewport.clientHeight ?? 0;
|
||||
return width > 0 && height > 0 ? { width, height } : null;
|
||||
}
|
||||
|
||||
async function ownerContentQuad(
|
||||
cdp: CdpRunner,
|
||||
parent: CdpFrame,
|
||||
ownerBackendNodeId: number,
|
||||
): Promise<Quad | null> {
|
||||
const result = await sendToCdpTarget<{ model?: { content?: number[] } }>(
|
||||
cdp,
|
||||
parent.target,
|
||||
"DOM.getBoxModel",
|
||||
{ backendNodeId: ownerBackendNodeId },
|
||||
);
|
||||
return parseCdpQuad(result.model?.content);
|
||||
}
|
||||
|
||||
async function sameTargetFrameClips(
|
||||
cdp: CdpRunner,
|
||||
byId: Map<string, CdpFrame>,
|
||||
frame: CdpFrame,
|
||||
targetRoot: CdpFrame,
|
||||
): Promise<Polygon[] | null> {
|
||||
const clips: Polygon[] = [];
|
||||
let current = frame;
|
||||
const seen = new Set<string>();
|
||||
while (current.frameId !== targetRoot.frameId) {
|
||||
if (seen.has(current.frameId) || !current.parentFrameId) return null;
|
||||
seen.add(current.frameId);
|
||||
const parent = byId.get(current.parentFrameId);
|
||||
if (!parent || current.ownerBackendNodeId === undefined) return null;
|
||||
const clip = await ownerContentQuad(cdp, parent, current.ownerBackendNodeId);
|
||||
if (!clip) return null;
|
||||
clips.push(clip);
|
||||
current = parent;
|
||||
}
|
||||
return clips;
|
||||
}
|
||||
|
||||
export async function resolveFrameProjection(
|
||||
cdp: CdpRunner,
|
||||
graph: CdpFrameGraph,
|
||||
frameId: string,
|
||||
): Promise<GeometryProjection | null> {
|
||||
const byId = frameMap(graph);
|
||||
const frame = byId.get(frameId);
|
||||
if (!frame) return null;
|
||||
let targetRoot = targetRootFrame(byId, frame);
|
||||
if (!targetRoot) return null;
|
||||
const sourceViewport = await targetViewport(cdp, targetRoot.target);
|
||||
if (!sourceViewport) return null;
|
||||
const sourceClips = await sameTargetFrameClips(cdp, byId, frame, targetRoot);
|
||||
if (!sourceClips) return null;
|
||||
|
||||
const edges: ProjectiveEdge[] = [];
|
||||
while (targetRoot.parentFrameId) {
|
||||
const parent = byId.get(targetRoot.parentFrameId);
|
||||
if (!parent || targetRoot.ownerBackendNodeId === undefined) return null;
|
||||
const destinationQuad = await ownerContentQuad(cdp, parent, targetRoot.ownerBackendNodeId);
|
||||
if (!destinationQuad) return null;
|
||||
const source =
|
||||
edges.length === 0 ? sourceViewport : await targetViewport(cdp, targetRoot.target);
|
||||
if (!source) return null;
|
||||
const parentTargetRoot = targetRootFrame(byId, parent);
|
||||
if (!parentTargetRoot) return null;
|
||||
const destinationClips = await sameTargetFrameClips(cdp, byId, parent, parentTargetRoot);
|
||||
if (!destinationClips) return null;
|
||||
edges.push({ sourceViewport: source, destinationQuad, destinationClips });
|
||||
targetRoot = parentTargetRoot;
|
||||
}
|
||||
|
||||
const topViewport =
|
||||
edges.length === 0 ? sourceViewport : await targetViewport(cdp, targetRoot.target);
|
||||
return topViewport ? { sourceClips, edges, topViewport } : null;
|
||||
}
|
||||
|
||||
async function loadFrameGraph(cdp: CdpRunner, tabId: number): Promise<CdpFrameGraph | null> {
|
||||
if (!cdp.getFrameGraph) return null;
|
||||
try {
|
||||
return await cdp.getFrameGraph(tabId);
|
||||
} catch (error) {
|
||||
console.debug("[bsk frame-geometry] frame graph resolution failed", error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function scrollFrameOwners(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
graph: CdpFrameGraph,
|
||||
frameId: string,
|
||||
): Promise<RpcError | null> {
|
||||
const byId = frameMap(graph);
|
||||
const ancestry = frameAncestry(graph, frameId);
|
||||
if (!ancestry) return geometryError(`could not resolve frame ancestry for ${frameId}`);
|
||||
for (const child of [...ancestry].reverse()) {
|
||||
const parent = child.parentFrameId ? byId.get(child.parentFrameId) : undefined;
|
||||
if (!parent || child.ownerBackendNodeId === undefined) {
|
||||
return geometryError(`could not resolve frame owner for ${child.frameId}`);
|
||||
}
|
||||
const error = await scrollNodeIntoView(
|
||||
cdpRunnerForTarget(cdp, parent.target),
|
||||
tabId,
|
||||
child.ownerBackendNodeId,
|
||||
);
|
||||
if (error) return error;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
async function scrollElementWithFrameGraph(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
target: CdpTarget,
|
||||
backendNodeId: number,
|
||||
frameId: string | undefined,
|
||||
graph: CdpFrameGraph | null,
|
||||
): Promise<RpcError | null> {
|
||||
if (frameId) {
|
||||
if (!graph) return geometryError(`could not resolve frame graph for ${frameId}`);
|
||||
const error = await scrollFrameOwners(cdp, tabId, graph, frameId);
|
||||
if (error) return error;
|
||||
} else if (target.sessionId) {
|
||||
return geometryError("an OOPIF node address requires frameId");
|
||||
}
|
||||
return scrollNodeIntoView(cdpRunnerForTarget(cdp, target), tabId, backendNodeId);
|
||||
}
|
||||
|
||||
export async function scrollElementAndFramesIntoView(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
target: CdpTarget,
|
||||
backendNodeId: number,
|
||||
frameId?: string,
|
||||
): Promise<RpcError | null> {
|
||||
const graph = frameId ? await loadFrameGraph(cdp, tabId) : null;
|
||||
return scrollElementWithFrameGraph(cdp, tabId, target, backendNodeId, frameId, graph);
|
||||
}
|
||||
|
||||
function largestRegion(regions: Region): Polygon | null {
|
||||
let largest: { polygon: Polygon; area: number } | null = null;
|
||||
for (const polygon of regions) {
|
||||
const area = polygonArea(polygon);
|
||||
if (area <= 0) continue;
|
||||
if (!largest || area > largest.area) largest = { polygon, area };
|
||||
}
|
||||
return largest?.polygon ?? null;
|
||||
}
|
||||
|
||||
export async function resolveNodeGeometry(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
address: NodeAddress,
|
||||
options: { scrollIntoView?: boolean } = {},
|
||||
): Promise<ResolvedNodeGeometry | RpcError> {
|
||||
try {
|
||||
if (address.target.sessionId && !address.frameId) {
|
||||
return geometryError("an OOPIF node address requires frameId");
|
||||
}
|
||||
const graph = address.frameId ? await loadFrameGraph(cdp, tabId) : null;
|
||||
if (address.frameId && !graph) {
|
||||
return geometryError(`could not resolve frame graph for ${address.frameId}`);
|
||||
}
|
||||
|
||||
if (options.scrollIntoView) {
|
||||
const scrollError = await scrollElementWithFrameGraph(
|
||||
cdp,
|
||||
tabId,
|
||||
address.target,
|
||||
address.backendNodeId,
|
||||
address.frameId,
|
||||
graph,
|
||||
);
|
||||
if (scrollError) return scrollError;
|
||||
}
|
||||
|
||||
const localRegion = await nodeContentRegion(
|
||||
cdpRunnerForTarget(cdp, address.target),
|
||||
tabId,
|
||||
address.backendNodeId,
|
||||
);
|
||||
if (isRpcError(localRegion)) return localRegion;
|
||||
|
||||
let topVisibleRegions: Region;
|
||||
if (address.frameId && graph) {
|
||||
const projection = await resolveFrameProjection(cdp, graph, address.frameId);
|
||||
if (!projection)
|
||||
return geometryError(`could not resolve frame geometry for ${address.frameId}`);
|
||||
topVisibleRegions = projectRegionToViewport(localRegion, projection);
|
||||
} else {
|
||||
const viewport = await targetViewport(cdp, address.target);
|
||||
if (!viewport) return geometryError("could not resolve top viewport geometry");
|
||||
topVisibleRegions = localRegion
|
||||
.map((polygon) =>
|
||||
clipPolygon(polygon, rectPolygon({ x: 0, y: 0, w: viewport.width, h: viewport.height })),
|
||||
)
|
||||
.filter((polygon) => polygon.length >= 3);
|
||||
}
|
||||
|
||||
const topBounds = regionBounds(topVisibleRegions);
|
||||
const actionRegion = largestRegion(topVisibleRegions);
|
||||
const actionPoint = actionRegion ? polygonCentroid(actionRegion) : null;
|
||||
if (!topBounds || !actionPoint) {
|
||||
return { code: "permission_denied", message: "element not visible" };
|
||||
}
|
||||
return { topVisibleRegions, topBounds, actionPoint };
|
||||
} catch (error) {
|
||||
return geometryError(error instanceof Error ? error.message : String(error));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,266 @@
|
||||
export interface Point {
|
||||
x: number;
|
||||
y: number;
|
||||
}
|
||||
|
||||
export type Polygon = Point[];
|
||||
export type Region = Polygon[];
|
||||
export type Quad = [Point, Point, Point, Point];
|
||||
|
||||
export interface Size {
|
||||
width: number;
|
||||
height: number;
|
||||
}
|
||||
|
||||
export interface ViewportRect {
|
||||
x: number;
|
||||
y: number;
|
||||
width: number;
|
||||
height: number;
|
||||
}
|
||||
|
||||
export interface ProjectiveEdge {
|
||||
sourceViewport: Size;
|
||||
destinationQuad: Quad;
|
||||
destinationClips?: Polygon[];
|
||||
}
|
||||
|
||||
export interface GeometryProjection {
|
||||
sourceClips: Polygon[];
|
||||
edges: ProjectiveEdge[];
|
||||
topViewport: Size;
|
||||
}
|
||||
|
||||
export function parseCdpQuad(raw: number[] | undefined): Quad | null {
|
||||
if (!raw || raw.length !== 8 || !raw.every(Number.isFinite)) return null;
|
||||
return [
|
||||
{ x: raw[0], y: raw[1] },
|
||||
{ x: raw[2], y: raw[3] },
|
||||
{ x: raw[4], y: raw[5] },
|
||||
{ x: raw[6], y: raw[7] },
|
||||
];
|
||||
}
|
||||
|
||||
export function rectPolygon(rect: { x: number; y: number; w: number; h: number }): Quad {
|
||||
return [
|
||||
{ x: rect.x, y: rect.y },
|
||||
{ x: rect.x + rect.w, y: rect.y },
|
||||
{ x: rect.x + rect.w, y: rect.y + rect.h },
|
||||
{ x: rect.x, y: rect.y + rect.h },
|
||||
];
|
||||
}
|
||||
|
||||
export function viewportPolygon(viewport: Size): Quad {
|
||||
return rectPolygon({ x: 0, y: 0, w: viewport.width, h: viewport.height });
|
||||
}
|
||||
|
||||
/** Project a normalized unit-square point into an arbitrary content quad. */
|
||||
export function projectUnitPoint(point: Point, quad: Quad): Point {
|
||||
const [p0, p1, p2, p3] = quad;
|
||||
const dx1 = p1.x - p2.x;
|
||||
const dx2 = p3.x - p2.x;
|
||||
const dx3 = p0.x - p1.x + p2.x - p3.x;
|
||||
const dy1 = p1.y - p2.y;
|
||||
const dy2 = p3.y - p2.y;
|
||||
const dy3 = p0.y - p1.y + p2.y - p3.y;
|
||||
|
||||
let a: number;
|
||||
let b: number;
|
||||
let c: number;
|
||||
let d: number;
|
||||
let e: number;
|
||||
let f: number;
|
||||
let g = 0;
|
||||
let h = 0;
|
||||
|
||||
if (Math.abs(dx3) < 1e-9 && Math.abs(dy3) < 1e-9) {
|
||||
a = p1.x - p0.x;
|
||||
b = p3.x - p0.x;
|
||||
c = p0.x;
|
||||
d = p1.y - p0.y;
|
||||
e = p3.y - p0.y;
|
||||
f = p0.y;
|
||||
} else {
|
||||
const denominator = dx1 * dy2 - dx2 * dy1;
|
||||
if (Math.abs(denominator) < 1e-9) {
|
||||
return {
|
||||
x: p0.x + (p1.x - p0.x) * point.x + (p3.x - p0.x) * point.y,
|
||||
y: p0.y + (p1.y - p0.y) * point.x + (p3.y - p0.y) * point.y,
|
||||
};
|
||||
}
|
||||
g = (dx3 * dy2 - dx2 * dy3) / denominator;
|
||||
h = (dx1 * dy3 - dx3 * dy1) / denominator;
|
||||
a = p1.x - p0.x + g * p1.x;
|
||||
b = p3.x - p0.x + h * p3.x;
|
||||
c = p0.x;
|
||||
d = p1.y - p0.y + g * p1.y;
|
||||
e = p3.y - p0.y + h * p3.y;
|
||||
f = p0.y;
|
||||
}
|
||||
|
||||
const scale = g * point.x + h * point.y + 1;
|
||||
return {
|
||||
x: (a * point.x + b * point.y + c) / scale,
|
||||
y: (d * point.x + e * point.y + f) / scale,
|
||||
};
|
||||
}
|
||||
|
||||
export function projectPolygon(polygon: Polygon, edge: ProjectiveEdge): Polygon {
|
||||
const { width, height } = edge.sourceViewport;
|
||||
if (width <= 0 || height <= 0) return [];
|
||||
return polygon.map((point) =>
|
||||
projectUnitPoint({ x: point.x / width, y: point.y / height }, edge.destinationQuad),
|
||||
);
|
||||
}
|
||||
|
||||
function polygonSignedArea(points: Polygon): number {
|
||||
let area = 0;
|
||||
for (let index = 0; index < points.length; index += 1) {
|
||||
const current = points[index];
|
||||
const next = points[(index + 1) % points.length];
|
||||
area += current.x * next.y - next.x * current.y;
|
||||
}
|
||||
return area / 2;
|
||||
}
|
||||
|
||||
function edgeIntersection(start: Point, end: Point, clipStart: Point, clipEnd: Point): Point {
|
||||
const subjectX = end.x - start.x;
|
||||
const subjectY = end.y - start.y;
|
||||
const clipX = clipEnd.x - clipStart.x;
|
||||
const clipY = clipEnd.y - clipStart.y;
|
||||
const denominator = subjectX * clipY - subjectY * clipX;
|
||||
if (Math.abs(denominator) < 1e-9) return end;
|
||||
const offsetX = clipStart.x - start.x;
|
||||
const offsetY = clipStart.y - start.y;
|
||||
const scale = (offsetX * clipY - offsetY * clipX) / denominator;
|
||||
return { x: start.x + scale * subjectX, y: start.y + scale * subjectY };
|
||||
}
|
||||
|
||||
/** Clip a polygon to a convex polygon, preserving every resulting vertex. */
|
||||
export function clipPolygon(subject: Polygon, clip: Polygon): Polygon {
|
||||
if (subject.length < 3 || clip.length < 3) return [];
|
||||
const orientation = polygonSignedArea(clip) >= 0 ? 1 : -1;
|
||||
let output = [...subject];
|
||||
for (let edge = 0; edge < clip.length && output.length > 0; edge += 1) {
|
||||
const clipStart = clip[edge];
|
||||
const clipEnd = clip[(edge + 1) % clip.length];
|
||||
const input = output;
|
||||
output = [];
|
||||
const inside = (point: Point) =>
|
||||
orientation *
|
||||
((clipEnd.x - clipStart.x) * (point.y - clipStart.y) -
|
||||
(clipEnd.y - clipStart.y) * (point.x - clipStart.x)) >=
|
||||
-1e-9;
|
||||
let start = input[input.length - 1];
|
||||
for (const end of input) {
|
||||
const startInside = inside(start);
|
||||
const endInside = inside(end);
|
||||
if (endInside) {
|
||||
if (!startInside) output.push(edgeIntersection(start, end, clipStart, clipEnd));
|
||||
output.push(end);
|
||||
} else if (startInside) {
|
||||
output.push(edgeIntersection(start, end, clipStart, clipEnd));
|
||||
}
|
||||
start = end;
|
||||
}
|
||||
}
|
||||
return output;
|
||||
}
|
||||
|
||||
export function polygonBounds(points: Polygon): ViewportRect | null {
|
||||
if (points.length < 3) return null;
|
||||
const xs = points.map((point) => point.x);
|
||||
const ys = points.map((point) => point.y);
|
||||
const x = Math.min(...xs);
|
||||
const y = Math.min(...ys);
|
||||
const right = Math.max(...xs);
|
||||
const bottom = Math.max(...ys);
|
||||
if (right <= x || bottom <= y) return null;
|
||||
return { x, y, width: right - x, height: bottom - y };
|
||||
}
|
||||
|
||||
export function regionBounds(region: Region): ViewportRect | null {
|
||||
return polygonBounds(region.flat());
|
||||
}
|
||||
|
||||
export function polygonArea(points: Polygon): number {
|
||||
return Math.abs(polygonSignedArea(points));
|
||||
}
|
||||
|
||||
export function polygonCentroid(points: Polygon): Point | null {
|
||||
if (points.length < 3) return null;
|
||||
let twiceArea = 0;
|
||||
let x = 0;
|
||||
let y = 0;
|
||||
for (let index = 0; index < points.length; index += 1) {
|
||||
const current = points[index];
|
||||
const next = points[(index + 1) % points.length];
|
||||
const cross = current.x * next.y - next.x * current.y;
|
||||
twiceArea += cross;
|
||||
x += (current.x + next.x) * cross;
|
||||
y += (current.y + next.y) * cross;
|
||||
}
|
||||
if (Math.abs(twiceArea) < 1e-9) return null;
|
||||
return { x: x / (3 * twiceArea), y: y / (3 * twiceArea) };
|
||||
}
|
||||
|
||||
export function projectAndClipRegion(
|
||||
region: Region,
|
||||
edges: ProjectiveEdge[],
|
||||
topViewport: Size,
|
||||
): Region {
|
||||
let projected = region;
|
||||
for (const edge of edges) {
|
||||
const sourceClip = viewportPolygon(edge.sourceViewport);
|
||||
projected = projected
|
||||
.map((polygon) => clipPolygon(polygon, sourceClip))
|
||||
.filter((polygon) => polygon.length >= 3)
|
||||
.map((polygon) =>
|
||||
[edge.destinationQuad, ...(edge.destinationClips ?? [])].reduce(
|
||||
(current, clip) => clipPolygon(current, clip),
|
||||
projectPolygon(polygon, edge),
|
||||
),
|
||||
)
|
||||
.filter((polygon) => polygon.length >= 3);
|
||||
}
|
||||
const topClip = viewportPolygon(topViewport);
|
||||
return projected
|
||||
.map((polygon) => clipPolygon(polygon, topClip))
|
||||
.filter((polygon) => polygon.length >= 3);
|
||||
}
|
||||
|
||||
export function projectRegionToViewport(region: Region, projection: GeometryProjection): Region {
|
||||
const clipped = region
|
||||
.map((polygon) =>
|
||||
projection.sourceClips.reduce((current, clip) => clipPolygon(current, clip), polygon),
|
||||
)
|
||||
.filter((polygon) => polygon.length >= 3);
|
||||
return projectAndClipRegion(clipped, projection.edges, projection.topViewport);
|
||||
}
|
||||
|
||||
export function projectRectToViewport(
|
||||
rect: { x: number; y: number; w: number; h: number } | null,
|
||||
projection: GeometryProjection,
|
||||
): ViewportRect | null {
|
||||
if (!rect) return null;
|
||||
return regionBounds(projectRegionToViewport([rectPolygon(rect)], projection));
|
||||
}
|
||||
|
||||
export function childFrameProjection(
|
||||
parent: GeometryProjection,
|
||||
ownerRectInParent: { x: number; y: number; w: number; h: number },
|
||||
): GeometryProjection {
|
||||
const destinationQuad = rectPolygon(ownerRectInParent);
|
||||
return {
|
||||
sourceClips: [],
|
||||
edges: [
|
||||
{
|
||||
sourceViewport: { width: ownerRectInParent.w, height: ownerRectInParent.h },
|
||||
destinationQuad,
|
||||
destinationClips: parent.sourceClips,
|
||||
},
|
||||
...parent.edges,
|
||||
],
|
||||
topViewport: parent.topViewport,
|
||||
};
|
||||
}
|
||||
@@ -8,6 +8,7 @@ import {
|
||||
HELP_REQUEST,
|
||||
type HelpCancelMessage,
|
||||
type HelpFinishMessage,
|
||||
type HelpHighlightRect,
|
||||
type HelpQueryResponse,
|
||||
type HelpRequestMessage,
|
||||
isHelpAckMessage,
|
||||
@@ -25,6 +26,7 @@ import type {
|
||||
ResolvedTarget,
|
||||
RpcError,
|
||||
} from "@/transport/types";
|
||||
import { resolveNodeGeometry } from "./frame-geometry";
|
||||
import {
|
||||
type CdpRunner,
|
||||
type ChromeTabsApi,
|
||||
@@ -37,7 +39,6 @@ import {
|
||||
import { lookupSnapshotRef } from "./snapshot-ref";
|
||||
|
||||
const DEFAULT_HELP_TIMEOUT_MS = 300_000;
|
||||
const HELP_ATTR = "data-bsk-help";
|
||||
const HELP_SEND_RETRIES = 3;
|
||||
const HELP_SEND_RETRY_DELAY_MS = 350;
|
||||
const HELP_REARM_DEBOUNCE_MS = 150;
|
||||
@@ -74,6 +75,7 @@ interface ActiveHelpRequest {
|
||||
title?: string;
|
||||
targets: HelpTarget[];
|
||||
selectors: string[];
|
||||
rects: HelpHighlightRect[];
|
||||
timeoutMs: number;
|
||||
notificationId: string;
|
||||
resolvedTargets?: ResolvedTarget[];
|
||||
@@ -134,6 +136,7 @@ function helpRequestMessage(help: ActiveHelpRequest, tabId: number): HelpRequest
|
||||
...(help.title ? { title: help.title } : {}),
|
||||
displayMode: isPrimaryTab ? "full" : "compact",
|
||||
selectors: isPrimaryTab ? help.selectors : [],
|
||||
rects: isPrimaryTab ? help.rects : [],
|
||||
timeoutMs: help.timeoutMs,
|
||||
};
|
||||
}
|
||||
@@ -165,33 +168,6 @@ async function sendHelpRequestWithAck(
|
||||
throw lastError ?? new Error("failed to show help overlay");
|
||||
}
|
||||
|
||||
async function tagRefTarget(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
backendNodeId: number,
|
||||
index: number,
|
||||
): Promise<string | null> {
|
||||
let objectId: string | undefined;
|
||||
try {
|
||||
const resolved = await cdp.send<{ object?: { objectId?: string } }>(tabId, "DOM.resolveNode", {
|
||||
backendNodeId,
|
||||
});
|
||||
objectId = resolved.object?.objectId;
|
||||
if (!objectId) return null;
|
||||
await cdp.send(tabId, "Runtime.callFunctionOn", {
|
||||
objectId,
|
||||
functionDeclaration: `function(){ this.setAttribute("${HELP_ATTR}", "${index}"); }`,
|
||||
});
|
||||
return `[${HELP_ATTR}="${index}"]`;
|
||||
} catch {
|
||||
return null;
|
||||
} finally {
|
||||
if (objectId) {
|
||||
await cdp.send(tabId, "Runtime.releaseObject", { objectId }).catch(() => {});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function selectorExists(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
@@ -209,36 +185,22 @@ async function selectorExists(
|
||||
}
|
||||
}
|
||||
|
||||
async function clearRefTags(cdp: CdpRunner | undefined, tabId: number): Promise<void> {
|
||||
if (!cdp) return;
|
||||
try {
|
||||
const doc = await cdp.send<{ root?: { nodeId?: number } }>(tabId, "DOM.getDocument", {
|
||||
depth: 0,
|
||||
});
|
||||
const rootId = doc.root?.nodeId;
|
||||
if (rootId === undefined) return;
|
||||
const { nodeIds } = await cdp.send<{ nodeIds: number[] }>(tabId, "DOM.querySelectorAll", {
|
||||
nodeId: rootId,
|
||||
selector: `[${HELP_ATTR}]`,
|
||||
});
|
||||
for (const nodeId of nodeIds ?? []) {
|
||||
await cdp.send(tabId, "DOM.removeAttribute", { nodeId, name: HELP_ATTR }).catch(() => {});
|
||||
}
|
||||
} catch {
|
||||
// Best-effort cleanup.
|
||||
}
|
||||
}
|
||||
|
||||
async function resolveHelpTargets(
|
||||
ctx: ReturnType<SessionManager["get"]>,
|
||||
tabId: number,
|
||||
targets: HelpTarget[],
|
||||
deps: RequestHelpDeps,
|
||||
): Promise<{ selectors: string[]; resolvedTargets?: ResolvedTarget[] }> {
|
||||
options: { scrollIntoView: boolean },
|
||||
): Promise<{
|
||||
selectors: string[];
|
||||
rects: HelpHighlightRect[];
|
||||
resolvedTargets?: ResolvedTarget[];
|
||||
}> {
|
||||
const selectors: string[] = [];
|
||||
const rects: HelpHighlightRect[] = [];
|
||||
const resolved: ResolvedTarget[] = [];
|
||||
let rootNodeId: number | undefined;
|
||||
if (deps.cdp && targets.length > 0) {
|
||||
if (deps.cdp && targets.some((target) => Boolean(target.selector))) {
|
||||
try {
|
||||
const doc = await deps.cdp.send<{ root?: { nodeId?: number } }>(tabId, "DOM.getDocument", {
|
||||
depth: 0,
|
||||
@@ -262,28 +224,55 @@ async function resolveHelpTargets(
|
||||
resolved.push({ matched, selector: tgt.selector });
|
||||
} else if (tgt.ref) {
|
||||
const looked = ctx ? lookupSnapshotRef(ctx, tgt.ref, tabId) : null;
|
||||
const backendNodeId = looked?.backendNodeId ?? null;
|
||||
let sel: string | null = null;
|
||||
if (backendNodeId !== null && deps.cdp) {
|
||||
sel = await tagRefTarget(deps.cdp, tabId, backendNodeId, i);
|
||||
let rect: HelpHighlightRect | null = null;
|
||||
if (looked && deps.cdp) {
|
||||
const target = {
|
||||
tabId,
|
||||
...(looked.cdpSessionId ? { sessionId: looked.cdpSessionId } : {}),
|
||||
};
|
||||
const geometry = await resolveNodeGeometry(
|
||||
deps.cdp,
|
||||
tabId,
|
||||
{
|
||||
target,
|
||||
backendNodeId: looked.backendNodeId,
|
||||
...(looked.frameId ? { frameId: looked.frameId } : {}),
|
||||
},
|
||||
{ scrollIntoView: options.scrollIntoView },
|
||||
);
|
||||
if (!isRpcError(geometry)) {
|
||||
rect = {
|
||||
top: geometry.topBounds.y,
|
||||
left: geometry.topBounds.x,
|
||||
width: geometry.topBounds.width,
|
||||
height: geometry.topBounds.height,
|
||||
};
|
||||
}
|
||||
}
|
||||
if (sel) selectors.push(sel);
|
||||
resolved.push({ matched: sel !== null, ref: tgt.ref });
|
||||
if (rect) rects.push(rect);
|
||||
resolved.push({
|
||||
matched: rect !== null,
|
||||
ref: tgt.ref,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return { selectors, resolvedTargets: resolved.length > 0 ? resolved : undefined };
|
||||
return { selectors, rects, resolvedTargets: resolved.length > 0 ? resolved : undefined };
|
||||
}
|
||||
|
||||
async function refreshHelpTargets(help: ActiveHelpRequest): Promise<void> {
|
||||
await clearRefTags(help.deps.cdp, help.primaryTabId);
|
||||
const { selectors, resolvedTargets } = await resolveHelpTargets(
|
||||
async function refreshHelpTargets(
|
||||
help: ActiveHelpRequest,
|
||||
options: { scrollIntoView: boolean } = { scrollIntoView: true },
|
||||
): Promise<void> {
|
||||
const { selectors, rects, resolvedTargets } = await resolveHelpTargets(
|
||||
help.ctx,
|
||||
help.primaryTabId,
|
||||
help.targets,
|
||||
help.deps,
|
||||
options,
|
||||
);
|
||||
help.selectors = selectors;
|
||||
help.rects = rects;
|
||||
help.resolvedTargets = resolvedTargets;
|
||||
}
|
||||
|
||||
@@ -294,12 +283,7 @@ async function cleanupHelp(help: ActiveHelpRequest): Promise<void> {
|
||||
const tabsToCancel = new Set([help.primaryTabId, ...help.overlayTabIds]);
|
||||
await Promise.all(
|
||||
[...tabsToCancel].map((tabId) =>
|
||||
Promise.all([
|
||||
clearRefTags(help.deps.cdp, tabId),
|
||||
help.deps
|
||||
.sendToTab(tabId, { type: HELP_CANCEL, requestId: help.requestId })
|
||||
.catch(() => {}),
|
||||
]),
|
||||
help.deps.sendToTab(tabId, { type: HELP_CANCEL, requestId: help.requestId }).catch(() => {}),
|
||||
),
|
||||
);
|
||||
}
|
||||
@@ -476,7 +460,9 @@ function attachHelpRuntimeListener(deps: RequestHelpDeps): () => void {
|
||||
sendResponse({ active: false });
|
||||
return;
|
||||
}
|
||||
if (tabId === help.primaryTabId) await refreshHelpTargets(help);
|
||||
if (tabId === help.primaryTabId) {
|
||||
await refreshHelpTargets(help, { scrollIntoView: false });
|
||||
}
|
||||
help.overlayTabIds.add(tabId);
|
||||
sendResponse({
|
||||
active: true,
|
||||
@@ -486,6 +472,7 @@ function attachHelpRuntimeListener(deps: RequestHelpDeps): () => void {
|
||||
...(help.title ? { title: help.title } : {}),
|
||||
displayMode: tabId === help.primaryTabId ? "full" : "compact",
|
||||
selectors: tabId === help.primaryTabId ? help.selectors : [],
|
||||
rects: tabId === help.primaryTabId ? help.rects : [],
|
||||
timeoutMs: help.timeoutMs,
|
||||
},
|
||||
});
|
||||
@@ -686,7 +673,13 @@ export async function handleRequestHelp(
|
||||
if (deps.autoAttachLifecycle !== false) ensureHelpLifecycleListeners(deps);
|
||||
const tabId = target.tabId;
|
||||
const initialTargets = params.targets ?? [];
|
||||
const { selectors, resolvedTargets } = await resolveHelpTargets(ctx, tabId, initialTargets, deps);
|
||||
const { selectors, rects, resolvedTargets } = await resolveHelpTargets(
|
||||
ctx,
|
||||
tabId,
|
||||
initialTargets,
|
||||
deps,
|
||||
{ scrollIntoView: true },
|
||||
);
|
||||
|
||||
await deps.windows.update(target.windowId, { focused: true }).catch(() => {});
|
||||
await deps.activateTab(tabId).catch(() => {});
|
||||
@@ -729,6 +722,7 @@ export async function handleRequestHelp(
|
||||
...(params.title ? { title: params.title } : {}),
|
||||
targets: initialTargets,
|
||||
selectors,
|
||||
rects,
|
||||
timeoutMs,
|
||||
notificationId,
|
||||
resolvedTargets,
|
||||
|
||||
@@ -3,14 +3,15 @@
|
||||
//
|
||||
// All interaction tools:
|
||||
// 1. Resolve target tab (sandbox: must be inside Agent Window).
|
||||
// 2. Resolve target element by `ref` (RefStore.resolve with tabId
|
||||
// binding) or `selector` (DOM.querySelector + describeNode).
|
||||
// 3. Scroll the node into view, then dispatch the appropriate
|
||||
// `Input.*` CDP events.
|
||||
// 2. Resolve target element by `ref` (compound frame/session identity)
|
||||
// or `selector` (DOM.querySelector + describeNode).
|
||||
// 3. Scroll the node and its frame owners into view, project its live
|
||||
// content quad into the top viewport, then dispatch `Input.*` events.
|
||||
// 4. Honour `AbortSignal` so canceled calls don't issue follow-up CDP
|
||||
// commands.
|
||||
|
||||
import { ChromiumCdp } from "@/browser-driver/chromium-cdp";
|
||||
import type { CdpTarget } from "@/browser-driver/frame-graph";
|
||||
import type { SessionContext, SessionManager } from "@/session-manager/manager";
|
||||
import type {
|
||||
ClickParams,
|
||||
@@ -28,17 +29,13 @@ import type {
|
||||
SelectResult,
|
||||
} from "@/transport/types";
|
||||
import { attachDialogs, markDialogCursor } from "./dialogs";
|
||||
import {
|
||||
backendNodeToObject,
|
||||
boxCentre,
|
||||
nodeCentre,
|
||||
quadCentre,
|
||||
scrollNodeIntoView,
|
||||
} from "./element-geometry";
|
||||
import { backendNodeToObject } from "./element-geometry";
|
||||
import { rpcError } from "./errors";
|
||||
import { resolveNodeGeometry, scrollElementAndFramesIntoView } from "./frame-geometry";
|
||||
import {
|
||||
type CdpRunner,
|
||||
type ChromeTabsApi,
|
||||
cdpRunnerForTarget,
|
||||
chromeTabsApi,
|
||||
enforceAgentWindow,
|
||||
isRpcError,
|
||||
@@ -137,7 +134,16 @@ async function resolveBackendNode(
|
||||
target: { tabId: number },
|
||||
params: { ref?: string; selector?: string },
|
||||
toolName: string,
|
||||
): Promise<{ backendNodeId: number; usedRef?: string; usedSelector?: string } | RpcError> {
|
||||
): Promise<
|
||||
| {
|
||||
backendNodeId: number;
|
||||
cdpTarget: CdpTarget;
|
||||
frameId?: string;
|
||||
usedRef?: string;
|
||||
usedSelector?: string;
|
||||
}
|
||||
| RpcError
|
||||
> {
|
||||
const hasRef = typeof params.ref === "string" && params.ref.length > 0;
|
||||
const hasSelector = typeof params.selector === "string" && params.selector.length > 0;
|
||||
if (hasRef && hasSelector) {
|
||||
@@ -155,7 +161,15 @@ async function resolveBackendNode(
|
||||
if (hasRef) {
|
||||
const resolved = resolveSnapshotRef(ctx, params.ref as string, target.tabId);
|
||||
if (isRpcError(resolved)) return resolved;
|
||||
return { backendNodeId: resolved.backendNodeId, usedRef: resolved.refKey };
|
||||
return {
|
||||
backendNodeId: resolved.backendNodeId,
|
||||
cdpTarget: {
|
||||
tabId: target.tabId,
|
||||
...(resolved.cdpSessionId ? { sessionId: resolved.cdpSessionId } : {}),
|
||||
},
|
||||
...(resolved.frameId ? { frameId: resolved.frameId } : {}),
|
||||
usedRef: resolved.refKey,
|
||||
};
|
||||
}
|
||||
// selector path
|
||||
try {
|
||||
@@ -192,7 +206,7 @@ async function resolveBackendNode(
|
||||
message: "DOM.describeNode returned no backendNodeId",
|
||||
};
|
||||
}
|
||||
return { backendNodeId, usedSelector: params.selector };
|
||||
return { backendNodeId, cdpTarget: { tabId: target.tabId }, usedSelector: params.selector };
|
||||
} catch (err) {
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
@@ -228,19 +242,19 @@ export async function handleClick(
|
||||
return { code: "cancelled", message: "click aborted" };
|
||||
}
|
||||
|
||||
try {
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
const scrollErr = await scrollNodeIntoView(deps.cdp, target.tabId, node.backendNodeId);
|
||||
if (scrollErr) return scrollErr;
|
||||
} catch (err) {
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
message: err instanceof Error ? err.message : String(err),
|
||||
};
|
||||
}
|
||||
|
||||
const centre = await nodeCentre(deps.cdp, target.tabId, node.backendNodeId);
|
||||
if (isRpcError(centre)) return centre;
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
const geometry = await resolveNodeGeometry(
|
||||
deps.cdp,
|
||||
target.tabId,
|
||||
{
|
||||
target: node.cdpTarget,
|
||||
backendNodeId: node.backendNodeId,
|
||||
...(node.frameId ? { frameId: node.frameId } : {}),
|
||||
},
|
||||
{ scrollIntoView: true },
|
||||
);
|
||||
if (isRpcError(geometry)) return geometry;
|
||||
const centre = geometry.actionPoint;
|
||||
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "click aborted" };
|
||||
@@ -353,19 +367,19 @@ export async function handleHover(
|
||||
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "hover");
|
||||
if (isRpcError(node)) return node;
|
||||
|
||||
try {
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
const scrollErr = await scrollNodeIntoView(deps.cdp, target.tabId, node.backendNodeId);
|
||||
if (scrollErr) return scrollErr;
|
||||
} catch (err) {
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
message: err instanceof Error ? err.message : String(err),
|
||||
};
|
||||
}
|
||||
|
||||
const centre = await nodeCentre(deps.cdp, target.tabId, node.backendNodeId);
|
||||
if (isRpcError(centre)) return centre;
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
const geometry = await resolveNodeGeometry(
|
||||
deps.cdp,
|
||||
target.tabId,
|
||||
{
|
||||
target: node.cdpTarget,
|
||||
backendNodeId: node.backendNodeId,
|
||||
...(node.frameId ? { frameId: node.frameId } : {}),
|
||||
},
|
||||
{ scrollIntoView: true },
|
||||
);
|
||||
if (isRpcError(geometry)) return geometry;
|
||||
const centre = geometry.actionPoint;
|
||||
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "hover aborted" };
|
||||
@@ -585,10 +599,11 @@ export async function handleFill(
|
||||
|
||||
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "fill");
|
||||
if (isRpcError(node)) return node;
|
||||
const nodeCdp = cdpRunnerForTarget(deps.cdp, node.cdpTarget);
|
||||
|
||||
try {
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
const described = await deps.cdp.send<{ node?: DescribedNode }>(
|
||||
const described = await nodeCdp.send<{ node?: DescribedNode }>(
|
||||
target.tabId,
|
||||
"DOM.describeNode",
|
||||
{
|
||||
@@ -602,12 +617,18 @@ export async function handleFill(
|
||||
`element ${described.node?.nodeName ?? "?"} not fillable (need input/textarea/contenteditable)`,
|
||||
);
|
||||
}
|
||||
const scrollErr = await scrollNodeIntoView(deps.cdp, target.tabId, node.backendNodeId);
|
||||
const scrollErr = await scrollElementAndFramesIntoView(
|
||||
deps.cdp,
|
||||
target.tabId,
|
||||
node.cdpTarget,
|
||||
node.backendNodeId,
|
||||
node.frameId,
|
||||
);
|
||||
if (scrollErr) return scrollErr;
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
await deps.cdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
|
||||
await nodeCdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
|
||||
} catch (err) {
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
@@ -619,7 +640,7 @@ export async function handleFill(
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
|
||||
const objectIdOrErr = await backendNodeToObject(deps.cdp, target.tabId, node.backendNodeId);
|
||||
const objectIdOrErr = await backendNodeToObject(nodeCdp, target.tabId, node.backendNodeId);
|
||||
if (isRpcError(objectIdOrErr)) return objectIdOrErr;
|
||||
const objectId = objectIdOrErr;
|
||||
const clearBefore = params.clear_before ?? true;
|
||||
@@ -628,7 +649,7 @@ export async function handleFill(
|
||||
if (clearBefore) {
|
||||
// Clear input/textarea value or wipe contenteditable innerText,
|
||||
// then fire `input` so frameworks observe the empty state.
|
||||
await deps.cdp.send(target.tabId, "Runtime.callFunctionOn", {
|
||||
await nodeCdp.send(target.tabId, "Runtime.callFunctionOn", {
|
||||
objectId,
|
||||
functionDeclaration: `function() {
|
||||
if (this.isContentEditable) { this.textContent = ''; }
|
||||
@@ -655,7 +676,7 @@ export async function handleFill(
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
// Fire `input` + `change` so React / Vue controlled inputs commit.
|
||||
await deps.cdp.send(target.tabId, "Runtime.callFunctionOn", {
|
||||
await nodeCdp.send(target.tabId, "Runtime.callFunctionOn", {
|
||||
objectId,
|
||||
functionDeclaration: `function() {
|
||||
this.dispatchEvent(new Event('input', { bubbles: true }));
|
||||
@@ -853,14 +874,21 @@ export async function handlePress(
|
||||
if (params.ref || params.selector) {
|
||||
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "press");
|
||||
if (isRpcError(node)) return node;
|
||||
const nodeCdp = cdpRunnerForTarget(deps.cdp, node.cdpTarget);
|
||||
try {
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
const scrollErr = await scrollNodeIntoView(deps.cdp, target.tabId, node.backendNodeId);
|
||||
const scrollErr = await scrollElementAndFramesIntoView(
|
||||
deps.cdp,
|
||||
target.tabId,
|
||||
node.cdpTarget,
|
||||
node.backendNodeId,
|
||||
node.frameId,
|
||||
);
|
||||
if (scrollErr) return scrollErr;
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "press aborted" };
|
||||
}
|
||||
await deps.cdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
|
||||
await nodeCdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
|
||||
} catch (err) {
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
@@ -965,10 +993,11 @@ export async function handleSelect(
|
||||
|
||||
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "select");
|
||||
if (isRpcError(node)) return node;
|
||||
const nodeCdp = cdpRunnerForTarget(deps.cdp, node.cdpTarget);
|
||||
|
||||
try {
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
const described = await deps.cdp.send<{ node?: DescribedNode }>(
|
||||
const described = await nodeCdp.send<{ node?: DescribedNode }>(
|
||||
target.tabId,
|
||||
"DOM.describeNode",
|
||||
{ backendNodeId: node.backendNodeId },
|
||||
@@ -991,12 +1020,18 @@ export async function handleSelect(
|
||||
"single-select <select> requires exactly one value",
|
||||
);
|
||||
}
|
||||
const scrollErr = await scrollNodeIntoView(deps.cdp, target.tabId, node.backendNodeId);
|
||||
const scrollErr = await scrollElementAndFramesIntoView(
|
||||
deps.cdp,
|
||||
target.tabId,
|
||||
node.cdpTarget,
|
||||
node.backendNodeId,
|
||||
node.frameId,
|
||||
);
|
||||
if (scrollErr) return scrollErr;
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "select aborted" };
|
||||
}
|
||||
await deps.cdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
|
||||
await nodeCdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
|
||||
} catch (err) {
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
@@ -1008,12 +1043,12 @@ export async function handleSelect(
|
||||
return { code: "cancelled", message: "select aborted" };
|
||||
}
|
||||
|
||||
const objectIdOrErr = await backendNodeToObject(deps.cdp, target.tabId, node.backendNodeId);
|
||||
const objectIdOrErr = await backendNodeToObject(nodeCdp, target.tabId, node.backendNodeId);
|
||||
if (isRpcError(objectIdOrErr)) return objectIdOrErr;
|
||||
const objectId = objectIdOrErr;
|
||||
|
||||
try {
|
||||
const evaluated = await deps.cdp.send<{
|
||||
const evaluated = await nodeCdp.send<{
|
||||
result?: { value?: SelectMutationResult | null };
|
||||
}>(target.tabId, "Runtime.callFunctionOn", {
|
||||
objectId,
|
||||
@@ -1072,9 +1107,6 @@ export async function handleSelect(
|
||||
|
||||
export const __testing__ = {
|
||||
DEFAULT_TIMEOUT_MS,
|
||||
quadCentre,
|
||||
boxCentre,
|
||||
resolveBackendNode,
|
||||
nodeCentre,
|
||||
isFillable,
|
||||
};
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -15,22 +15,37 @@ import {
|
||||
type RecordQueryResponse,
|
||||
type RecordStartAck,
|
||||
type RecordStartMessage,
|
||||
type RecordStepAck,
|
||||
type RecordStopMessage,
|
||||
} from "@/lib/record-bridge";
|
||||
import { appendRecordedPayload, observeRecordedNavigation } from "@/lib/recording-step-buffer";
|
||||
import { reduceTraceSteps, resolveTraceStartUrl } from "@/lib/trace-reducer";
|
||||
import {
|
||||
type RecordFrameCoordinator,
|
||||
type RecordingCaptureScope,
|
||||
recordFrameCoordinator,
|
||||
} from "@/lib/recording/frame-coordinator";
|
||||
import { RecordingObservationRuntime } from "@/lib/recording/recording-runtime";
|
||||
import {
|
||||
appendRecordedPayload,
|
||||
observeRecordedNavigation,
|
||||
type RecordingStepBuffer,
|
||||
} from "@/lib/recording/step-buffer";
|
||||
import { RecordingTabCoordinator, type TabActivation } from "@/lib/recording/tab-coordinator";
|
||||
import { buildTraceV2 } from "@/lib/recording/trace-reducer-v2";
|
||||
import type { RecordingDraftStep } from "@/lib/recording/types";
|
||||
import type { SessionManager } from "@/session-manager/manager";
|
||||
import { EXTENSION_VERSION } from "@/transport/handshake";
|
||||
import type {
|
||||
DraftTraceStep,
|
||||
RecordAwaitParams,
|
||||
RecordAwaitResult,
|
||||
RecordedTrace,
|
||||
RecordStartParams,
|
||||
RecordStartResult,
|
||||
RecordStopParams,
|
||||
RecordStopResult,
|
||||
RpcError,
|
||||
Trace,
|
||||
StopReason,
|
||||
} from "@/transport/types";
|
||||
import { TRACE_VERSION_V3 } from "@/transport/types";
|
||||
import { handleNavigate } from "./navigation";
|
||||
import {
|
||||
type CdpRunner,
|
||||
@@ -43,21 +58,46 @@ import {
|
||||
|
||||
interface ActiveRecording {
|
||||
requestId: string;
|
||||
tabId: number;
|
||||
tabs: RecordingTabCoordinator;
|
||||
agentWindowId: number;
|
||||
startUrl?: string;
|
||||
purpose?: string;
|
||||
steps: DraftTraceStep[];
|
||||
steps: RecordingDraftStep[];
|
||||
startedAt: string;
|
||||
startedAtMs: number;
|
||||
finishPromise: Promise<Trace>;
|
||||
resolveFinish: (trace: Trace) => void;
|
||||
traceVersion: 2 | 3;
|
||||
supportsTabSwitchSteps: boolean;
|
||||
finishPromise: Promise<RecordedTrace>;
|
||||
resolveFinish: (trace: RecordedTrace) => void;
|
||||
rejectFinish: (err: Error) => void;
|
||||
settled: boolean;
|
||||
finishing: boolean;
|
||||
currentUrl?: string;
|
||||
pendingNavigation: boolean;
|
||||
pendingNavigationDeadline?: number;
|
||||
finishAttempt: Promise<RecordedTrace | null> | null;
|
||||
observation: RecordingObservationRuntime | null;
|
||||
stoppedBy: StopReason;
|
||||
/** Navigation callbacks tracked from event receipt through action enqueue. */
|
||||
navigationCallbacks: Set<Promise<void>>;
|
||||
/** Synchronous intake gate closed only after finish drains to stability. */
|
||||
acceptingNavigation: boolean;
|
||||
/**
|
||||
* Serializes step appends with navigation observation so a click is always
|
||||
* in `steps` before a same-turn `webNavigation` tries to annotate it.
|
||||
*/
|
||||
actionQueue: Promise<void>;
|
||||
/** Last accepted sequence for each content-script document producer. */
|
||||
lastStepSequenceByProducer: Map<string, number>;
|
||||
}
|
||||
|
||||
function enqueueRecordingAction(
|
||||
recording: ActiveRecording,
|
||||
task: () => Promise<void>,
|
||||
): Promise<void> {
|
||||
const queued = recording.actionQueue.then(task, task);
|
||||
recording.actionQueue = queued.catch(() => {});
|
||||
return queued;
|
||||
}
|
||||
|
||||
function isRecordingFinishing(recording: ActiveRecording): boolean {
|
||||
return recording.settled || recording.finishAttempt !== null;
|
||||
}
|
||||
|
||||
const recordings = new Map<string, ActiveRecording>();
|
||||
@@ -78,6 +118,9 @@ function sleep(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
/** Recording producer version mirrored into trace.recorder.bsk. */
|
||||
export const BSK_TRACE_VERSION = EXTENSION_VERSION;
|
||||
|
||||
/** Injectable http(s) landing page when `tool.record_start` omits `url`. */
|
||||
export const RECORD_DEFAULT_START_URL = "https://example.com/";
|
||||
|
||||
@@ -137,9 +180,11 @@ async function sendRecordStartWithAck(
|
||||
tabId: number,
|
||||
msg: RecordStartMessage,
|
||||
sendToTab: RecordDeps["sendToTab"],
|
||||
cancelled?: () => boolean,
|
||||
): Promise<void> {
|
||||
let lastError: unknown;
|
||||
for (let attempt = 0; attempt < RECORD_START_RETRIES; attempt += 1) {
|
||||
if (cancelled?.()) throw new Error("record start cancelled");
|
||||
try {
|
||||
const response = await sendToTab(tabId, msg);
|
||||
if (isRecordStartAck(response)) return;
|
||||
@@ -147,6 +192,7 @@ async function sendRecordStartWithAck(
|
||||
} catch (err) {
|
||||
lastError = err;
|
||||
}
|
||||
if (cancelled?.()) throw new Error("record start cancelled");
|
||||
if (attempt + 1 < RECORD_START_RETRIES) {
|
||||
await sleep(RECORD_START_RETRY_DELAY_MS);
|
||||
}
|
||||
@@ -154,19 +200,83 @@ async function sendRecordStartWithAck(
|
||||
throw lastError ?? new Error("failed to start recording in content script");
|
||||
}
|
||||
|
||||
function buildTrace(recording: ActiveRecording): Trace {
|
||||
const { pages, steps } = reduceTraceSteps(recording.steps, recording.startUrl);
|
||||
const startUrl = resolveTraceStartUrl(recording.steps, recording.startUrl, pages);
|
||||
function negotiatedTraceVersion(params: RecordStartParams): 2 | 3 | RpcError {
|
||||
if (params.trace_version === undefined) return 2;
|
||||
if (params.trace_version === TRACE_VERSION_V3) return 3;
|
||||
return {
|
||||
...(recording.purpose ? { purpose: recording.purpose } : {}),
|
||||
recorded_at: new Date().toISOString(),
|
||||
started_at: recording.startedAt,
|
||||
entry: { start_url: startUrl },
|
||||
pages,
|
||||
steps,
|
||||
code: "invalid_params",
|
||||
message: `unsupported trace_version ${params.trace_version}; supported values are omitted (v2) or ${TRACE_VERSION_V3} (v3)`,
|
||||
};
|
||||
}
|
||||
|
||||
function buildTrace(recording: ActiveRecording): RecordedTrace {
|
||||
if (recording.traceVersion === 3 && recording.observation) {
|
||||
return recording.observation.buildTrace({
|
||||
drafts: recording.steps,
|
||||
startedAt: recording.startedAt,
|
||||
purpose: recording.purpose,
|
||||
startUrl: recording.startUrl,
|
||||
stoppedBy: recording.stoppedBy,
|
||||
bskVersion: BSK_TRACE_VERSION,
|
||||
includeTabSwitches: recording.supportsTabSwitchSteps,
|
||||
});
|
||||
}
|
||||
if (recording.traceVersion === 3) {
|
||||
throw new Error("trace v3 observation runtime is unavailable");
|
||||
}
|
||||
return buildTraceV2({
|
||||
steps: recording.steps,
|
||||
startedAt: recording.startedAt,
|
||||
...(recording.startUrl ? { startUrl: recording.startUrl } : {}),
|
||||
...(recording.purpose ? { purpose: recording.purpose } : {}),
|
||||
});
|
||||
}
|
||||
|
||||
function stepBufferFor(
|
||||
recording: ActiveRecording,
|
||||
tabId: number,
|
||||
fallbackUrl?: string,
|
||||
): RecordingStepBuffer {
|
||||
return {
|
||||
steps: recording.steps,
|
||||
navigation: recording.tabs.navigation(tabId, fallbackUrl),
|
||||
};
|
||||
}
|
||||
|
||||
async function activateRecordingTab(
|
||||
recording: ActiveRecording,
|
||||
targetTabId: number,
|
||||
): Promise<void> {
|
||||
const previousTabId = recording.tabs.currentTabId;
|
||||
if (targetTabId === previousTabId) return;
|
||||
|
||||
const transition = recording.observation
|
||||
? await recording.observation.captureTabTransition(previousTabId, targetTabId)
|
||||
: {};
|
||||
const { targetUrl, ...stateLinks } = transition;
|
||||
const draft: Extract<RecordingDraftStep, { op: "switch_tab" }> = {
|
||||
op: "switch_tab",
|
||||
...stateLinks,
|
||||
};
|
||||
recording.steps.push(draft);
|
||||
recording.observation?.bindTabTransition(draft, recording.steps.length);
|
||||
recording.tabs.commit(targetTabId, targetUrl);
|
||||
}
|
||||
|
||||
async function processRecordedStep(
|
||||
recording: ActiveRecording,
|
||||
draftIndex: number,
|
||||
tabId: number,
|
||||
producerId?: string,
|
||||
): Promise<void> {
|
||||
if (!recording.observation) return;
|
||||
try {
|
||||
await recording.observation.processDraft(tabId, recording.steps, draftIndex, producerId);
|
||||
} catch (err) {
|
||||
console.warn(`[bsk record] observation failed for step ${draftIndex + 1}`, err);
|
||||
}
|
||||
}
|
||||
|
||||
export interface RecordDeps {
|
||||
tabsApi: ChromeTabsApi;
|
||||
sendToTab(
|
||||
@@ -174,16 +284,25 @@ export interface RecordDeps {
|
||||
msg: RecordStartMessage | RecordStopMessage | RecordCancelMessage,
|
||||
): Promise<unknown>;
|
||||
bypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
|
||||
frameCoordinator?: Pick<
|
||||
RecordFrameCoordinator,
|
||||
"begin" | "armTab" | "sourceFor" | "stop" | "cancel"
|
||||
>;
|
||||
cdp?: CdpRunner;
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
export type RecordRuntimeDeps = Omit<RecordDeps, "frameCoordinator" | "signal"> & {
|
||||
frameCoordinator: NonNullable<RecordDeps["frameCoordinator"]>;
|
||||
};
|
||||
|
||||
let defaultDeps: RecordDeps | null = null;
|
||||
function getDefaultDeps(): RecordDeps {
|
||||
if (!defaultDeps) {
|
||||
defaultDeps = {
|
||||
tabsApi: chromeTabsApi,
|
||||
sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
|
||||
frameCoordinator: recordFrameCoordinator,
|
||||
};
|
||||
}
|
||||
return defaultDeps;
|
||||
@@ -241,16 +360,58 @@ export function releaseBrowserObservationListenersIfIdle(): void {
|
||||
detachBrowserObservation = null;
|
||||
}
|
||||
|
||||
export function attachRecordStepListener(): () => void {
|
||||
export function attachRecordStepListener(deps: RecordDeps = getDefaultDeps()): () => void {
|
||||
const listener = (
|
||||
message: unknown,
|
||||
_sender: chrome.runtime.MessageSender,
|
||||
_sendResponse: () => void,
|
||||
sender: chrome.runtime.MessageSender,
|
||||
sendResponse: (response: RecordStepAck) => void,
|
||||
) => {
|
||||
if (!isRecordStepMessage(message)) return false;
|
||||
for (const recording of recordings.values()) {
|
||||
if (recording.requestId !== message.requestId) continue;
|
||||
appendRecordedPayload(recording, message.step);
|
||||
const source: RecordingCaptureScope | null | undefined = deps.frameCoordinator
|
||||
? deps.frameCoordinator.sourceFor(message.requestId, message.producerId, sender)
|
||||
: undefined;
|
||||
if (deps.frameCoordinator && !source) return false;
|
||||
const sourceTabId = source?.tabId ?? sender.tab?.id ?? recording.tabs.currentTabId;
|
||||
const sourceWasActive = sender.tab?.active ?? sourceTabId === recording.tabs.activeTabId;
|
||||
const producerKey = source
|
||||
? `${source.tabId}:${source.documentId}:${source.producerId}`
|
||||
: `${sourceTabId}:${message.producerId}`;
|
||||
const expectedSequence = (recording.lastStepSequenceByProducer.get(producerKey) ?? 0) + 1;
|
||||
if (message.sequence < expectedSequence) {
|
||||
sendResponse({ ok: true, sequence: message.sequence });
|
||||
return false;
|
||||
}
|
||||
if (message.sequence > expectedSequence) {
|
||||
sendResponse({
|
||||
ok: false,
|
||||
expectedSequence,
|
||||
error: `out-of-order recorded step ${message.sequence}`,
|
||||
});
|
||||
return false;
|
||||
}
|
||||
|
||||
recording.lastStepSequenceByProducer.set(producerKey, message.sequence);
|
||||
enqueueRecordingAction(recording, async () => {
|
||||
if (sourceTabId !== recording.tabs.currentTabId) {
|
||||
if (!sourceWasActive) return;
|
||||
if (sourceTabId !== recording.tabs.activeTabId)
|
||||
recording.tabs.noteActivation(sourceTabId);
|
||||
await activateRecordingTab(recording, sourceTabId);
|
||||
}
|
||||
await recording.observation?.flushRedirects(sourceTabId);
|
||||
const targetHint = message.step.geometry ? { geometry: message.step.geometry } : undefined;
|
||||
const draftIndex = appendRecordedPayload(
|
||||
stepBufferFor(recording, sourceTabId, message.step.page_url),
|
||||
message.step,
|
||||
targetHint,
|
||||
);
|
||||
if (draftIndex !== null) {
|
||||
await processRecordedStep(recording, draftIndex, sourceTabId, source?.producerId);
|
||||
}
|
||||
});
|
||||
sendResponse({ ok: true, sequence: message.sequence });
|
||||
return false;
|
||||
}
|
||||
return false;
|
||||
@@ -277,7 +438,7 @@ export function attachRecordFinishListener(deps: RecordDeps = getDefaultDeps()):
|
||||
|
||||
function findRecordingByTabId(tabId: number): ActiveRecording | null {
|
||||
for (const recording of recordings.values()) {
|
||||
if (recording.tabId === tabId && !recording.settled) return recording;
|
||||
if (recording.tabs.currentTabId === tabId && !recording.settled) return recording;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
@@ -314,7 +475,7 @@ async function clearRearmTimersForRecording(
|
||||
recording: ActiveRecording,
|
||||
deps: RecordDeps,
|
||||
): Promise<void> {
|
||||
clearRearmTimer(recording.tabId);
|
||||
clearRearmTimer(recording.tabs.currentTabId);
|
||||
try {
|
||||
const tabs = await deps.tabsApi.query({ windowId: recording.agentWindowId });
|
||||
for (const tab of tabs) {
|
||||
@@ -329,13 +490,16 @@ async function stopRecordingOnAllAgentTabs(
|
||||
recording: ActiveRecording,
|
||||
deps: RecordDeps,
|
||||
): Promise<void> {
|
||||
if (deps.frameCoordinator && !(await deps.frameCoordinator.stop(recording.requestId))) {
|
||||
throw new Error("failed to flush one or more recording documents");
|
||||
}
|
||||
const stopMsg: RecordStopMessage = { type: RECORD_STOP, requestId: recording.requestId };
|
||||
let tabIds = [recording.tabId];
|
||||
let tabIds = [recording.tabs.currentTabId];
|
||||
try {
|
||||
const tabs = await deps.tabsApi.query({ windowId: recording.agentWindowId });
|
||||
tabIds = [
|
||||
...new Set([
|
||||
recording.tabId,
|
||||
recording.tabs.currentTabId,
|
||||
...tabs.flatMap((tab) => (typeof tab.id === "number" ? [tab.id] : [])),
|
||||
]),
|
||||
];
|
||||
@@ -346,11 +510,11 @@ async function stopRecordingOnAllAgentTabs(
|
||||
for (const tabId of tabIds) {
|
||||
try {
|
||||
const response = await deps.sendToTab(tabId, stopMsg);
|
||||
if (tabId === recording.tabId && !isRecordStartAck(response)) {
|
||||
if (tabId === recording.tabs.currentTabId && !isRecordStartAck(response)) {
|
||||
throw new Error("content script did not confirm recorded steps");
|
||||
}
|
||||
} catch {
|
||||
if (tabId === recording.tabId) {
|
||||
if (tabId === recording.tabs.currentTabId) {
|
||||
throw new Error("failed to flush recorded steps");
|
||||
}
|
||||
}
|
||||
@@ -368,22 +532,39 @@ async function rearmRecording(
|
||||
recording: ActiveRecording,
|
||||
targetTabId: number,
|
||||
deps: RecordDeps,
|
||||
activation?: TabActivation,
|
||||
): Promise<boolean> {
|
||||
// Do NOT toggle automation-bypass here: each retry used to increment the
|
||||
// content-script counter, and a single stop decrement left the ControlOverlay
|
||||
// stuck with pointer-events:none (page usable, Interrupt dead). RecordOverlay
|
||||
// already hides the control chrome while activeRecord is set.
|
||||
const isFinishing = () => isRecordingFinishing(recording);
|
||||
for (let attempt = 0; attempt < RECORD_REARM_MAX_ATTEMPTS; attempt += 1) {
|
||||
if (isFinishing()) return false;
|
||||
const startMsg: RecordStartMessage = {
|
||||
type: RECORD_START,
|
||||
requestId: recording.requestId,
|
||||
startedAtMs: recording.startedAtMs,
|
||||
};
|
||||
try {
|
||||
await sendRecordStartWithAck(targetTabId, startMsg, deps.sendToTab);
|
||||
recording.tabId = targetTabId;
|
||||
if (deps.frameCoordinator) {
|
||||
const frameStarted = await deps.frameCoordinator.armTab(recording.requestId, targetTabId);
|
||||
if (!frameStarted) throw new Error("recording document did not start");
|
||||
}
|
||||
await sendRecordStartWithAck(targetTabId, startMsg, deps.sendToTab, isFinishing);
|
||||
if (isFinishing()) return false;
|
||||
if (activation) {
|
||||
if (!recording.tabs.isLatest(activation)) return true;
|
||||
await enqueueRecordingAction(recording, async () => {
|
||||
if (isFinishing() || !recording.tabs.isLatest(activation)) {
|
||||
return;
|
||||
}
|
||||
await activateRecordingTab(recording, targetTabId);
|
||||
});
|
||||
}
|
||||
return true;
|
||||
} catch {
|
||||
if (isFinishing()) return false;
|
||||
if (attempt + 1 < RECORD_REARM_MAX_ATTEMPTS) {
|
||||
await sleep(RECORD_REARM_RETRY_DELAY_MS);
|
||||
}
|
||||
@@ -392,7 +573,7 @@ async function rearmRecording(
|
||||
return false;
|
||||
}
|
||||
|
||||
function scheduleRearmForTab(tabId: number, deps: RecordDeps): void {
|
||||
function scheduleRearmForTab(tabId: number, deps: RecordDeps, activation?: TabActivation): void {
|
||||
const existing = rearmTimers.get(tabId);
|
||||
if (existing) clearTimeout(existing);
|
||||
rearmTimers.set(
|
||||
@@ -401,7 +582,7 @@ function scheduleRearmForTab(tabId: number, deps: RecordDeps): void {
|
||||
rearmTimers.delete(tabId);
|
||||
void (async () => {
|
||||
const current = await findRecordingForTab(tabId, deps);
|
||||
if (current) await rearmRecording(current, tabId, deps);
|
||||
if (current) await rearmRecording(current, tabId, deps, activation);
|
||||
})();
|
||||
}, RECORD_REARM_DEBOUNCE_MS),
|
||||
);
|
||||
@@ -413,14 +594,20 @@ export function attachRecordTabListener(deps: RecordDeps = getDefaultDeps()): ()
|
||||
const windowId = tab.windowId;
|
||||
if (tabId === undefined || windowId === undefined) return;
|
||||
for (const recording of recordings.values()) {
|
||||
if (recording.settled || recording.agentWindowId !== windowId) continue;
|
||||
if (isRecordingFinishing(recording) || recording.agentWindowId !== windowId) continue;
|
||||
scheduleRearmForTab(tabId, deps);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
const onActivated = (activeInfo: chrome.tabs.TabActiveInfo) => {
|
||||
scheduleRearmForTab(activeInfo.tabId, deps);
|
||||
for (const recording of recordings.values()) {
|
||||
if (isRecordingFinishing(recording) || recording.agentWindowId !== activeInfo.windowId)
|
||||
continue;
|
||||
const activation = recording.tabs.noteActivation(activeInfo.tabId);
|
||||
scheduleRearmForTab(activeInfo.tabId, deps, activation);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
chrome.tabs.onCreated.addListener(onCreated);
|
||||
@@ -432,13 +619,47 @@ export function attachRecordTabListener(deps: RecordDeps = getDefaultDeps()): ()
|
||||
}
|
||||
|
||||
export function attachRecordNavigationListener(deps: RecordDeps = getDefaultDeps()): () => void {
|
||||
const observeMainFrame = (tabId: number, url?: string, causedByAction?: boolean) => {
|
||||
void (async () => {
|
||||
const recording = await findRecordingForTab(tabId, deps);
|
||||
if (recording && url) {
|
||||
observeRecordedNavigation(recording, url, causedByAction);
|
||||
}
|
||||
})();
|
||||
const observeMainFrame = (
|
||||
tabId: number,
|
||||
url?: string,
|
||||
causedByAction?: boolean,
|
||||
transitionType?: string,
|
||||
transitionQualifiers?: string[],
|
||||
) => {
|
||||
if (!url) return;
|
||||
const candidates = [...recordings.values()].filter(
|
||||
(recording) =>
|
||||
!recording.settled && recording.acceptingNavigation && recording.tabs.activeTabId === tabId,
|
||||
);
|
||||
for (const recording of candidates) {
|
||||
const queued = enqueueRecordingAction(recording, async () => {
|
||||
if (tabId !== recording.tabs.currentTabId) {
|
||||
await activateRecordingTab(recording, tabId);
|
||||
}
|
||||
const result = observeRecordedNavigation(
|
||||
stepBufferFor(recording, tabId, url),
|
||||
url,
|
||||
causedByAction,
|
||||
transitionType,
|
||||
transitionQualifiers,
|
||||
);
|
||||
if (result.kind === "coalesce_redirect") {
|
||||
recording.observation?.scheduleRedirect(tabId, recording.steps, result.url);
|
||||
return;
|
||||
}
|
||||
|
||||
if (result.kind === "noop") return;
|
||||
recording.observation?.clearRedirect(tabId);
|
||||
if (result.kind === "appended") {
|
||||
await processRecordedStep(recording, result.index, tabId);
|
||||
} else {
|
||||
recording.observation?.scheduleSettle(tabId, recording.steps, result.index);
|
||||
}
|
||||
});
|
||||
const tracked = queued.catch(() => {});
|
||||
recording.navigationCallbacks.add(tracked);
|
||||
void tracked.finally(() => recording.navigationCallbacks.delete(tracked));
|
||||
}
|
||||
};
|
||||
const onMainFrameComplete = (tabId: number, url?: string) => {
|
||||
void (async () => {
|
||||
@@ -458,18 +679,13 @@ export function attachRecordNavigationListener(deps: RecordDeps = getDefaultDeps
|
||||
details: chrome.webNavigation.WebNavigationTransitionCallbackDetails,
|
||||
) => {
|
||||
if (details.frameId !== 0) return;
|
||||
const causedByAction =
|
||||
details.transitionType === "link" || details.transitionType === "form_submit"
|
||||
? true
|
||||
: details.transitionType === "typed" ||
|
||||
details.transitionType === "auto_bookmark" ||
|
||||
details.transitionType === "generated" ||
|
||||
details.transitionType === "keyword" ||
|
||||
details.transitionType === "keyword_generated" ||
|
||||
details.transitionType === "reload"
|
||||
? false
|
||||
: undefined;
|
||||
observeMainFrame(details.tabId, details.url, causedByAction);
|
||||
observeMainFrame(
|
||||
details.tabId,
|
||||
details.url,
|
||||
undefined,
|
||||
details.transitionType,
|
||||
details.transitionQualifiers,
|
||||
);
|
||||
};
|
||||
chrome.webNavigation.onCompleted.addListener(completedListener);
|
||||
chrome.webNavigation.onCommitted?.addListener(committedListener);
|
||||
@@ -487,6 +703,26 @@ export function attachRecordNavigationListener(deps: RecordDeps = getDefaultDeps
|
||||
return () => chrome.tabs.onUpdated.removeListener(listener);
|
||||
}
|
||||
|
||||
const MAX_FINISH_DRAIN_ROUNDS = 10;
|
||||
|
||||
async function drainRecordingToStability(recording: ActiveRecording): Promise<boolean> {
|
||||
for (let round = 0; round < MAX_FINISH_DRAIN_ROUNDS; round += 1) {
|
||||
await Promise.all([...recording.navigationCallbacks]);
|
||||
const actionTail = recording.actionQueue;
|
||||
await actionTail;
|
||||
await recording.observation?.flush();
|
||||
|
||||
if (recording.navigationCallbacks.size === 0 && recording.actionQueue === actionTail) {
|
||||
// No event or promise continuation can interleave with this synchronous
|
||||
// check-and-close, so work accepted before the cutoff is fully drained.
|
||||
recording.acceptingNavigation = false;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
console.warn("[bsk record] navigation/action queues did not stabilize at stop");
|
||||
return false;
|
||||
}
|
||||
|
||||
export function attachRecordQueryListener(deps: RecordDeps = getDefaultDeps()): () => void {
|
||||
const listener = (
|
||||
message: unknown,
|
||||
@@ -505,7 +741,8 @@ export function attachRecordQueryListener(deps: RecordDeps = getDefaultDeps()):
|
||||
sendResponse({ active: false });
|
||||
return;
|
||||
}
|
||||
await rearmRecording(recording, tabId, deps);
|
||||
const activation = sender.tab?.active ? recording.tabs.noteActivation(tabId) : undefined;
|
||||
await rearmRecording(recording, tabId, deps, activation);
|
||||
sendResponse({
|
||||
active: true,
|
||||
requestId: recording.requestId,
|
||||
@@ -527,23 +764,50 @@ async function finishRecordingByRequest(
|
||||
if (recording.requestId !== requestId || recording.settled) continue;
|
||||
const match = await findRecordingForTab(tabId, deps);
|
||||
if (match !== recording) continue;
|
||||
await finishRecording(sessionId, deps);
|
||||
await finishRecording(sessionId, deps, "user_finish");
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
async function finishRecording(sessionId: string, deps: RecordDeps): Promise<Trace | null> {
|
||||
async function finishRecording(
|
||||
sessionId: string,
|
||||
deps: RecordDeps,
|
||||
stoppedBy: StopReason,
|
||||
): Promise<RecordedTrace | null> {
|
||||
const recording = recordings.get(sessionId);
|
||||
if (!recording || recording.settled || recording.finishing) return null;
|
||||
recording.finishing = true;
|
||||
if (!recording || recording.settled) return null;
|
||||
if (recording.finishAttempt) return recording.finishAttempt;
|
||||
recording.stoppedBy = stoppedBy;
|
||||
|
||||
const attempt = finishRecordingAttempt(sessionId, recording, deps);
|
||||
recording.finishAttempt = attempt;
|
||||
try {
|
||||
return await attempt;
|
||||
} finally {
|
||||
if (!recording.settled) {
|
||||
recording.finishAttempt = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function finishRecordingAttempt(
|
||||
sessionId: string,
|
||||
recording: ActiveRecording,
|
||||
deps: RecordDeps,
|
||||
): Promise<RecordedTrace | null> {
|
||||
await clearRearmTimersForRecording(recording, deps);
|
||||
try {
|
||||
// Disposing content capture may commit a final dirty fill. Flush content
|
||||
// first so all resulting RECORD_STEP messages enter actionQueue before it
|
||||
// and the observation queues are drained.
|
||||
await stopRecordingOnAllAgentTabs(recording, deps);
|
||||
} catch {
|
||||
recording.finishing = false;
|
||||
return null;
|
||||
}
|
||||
if (!(await drainRecordingToStability(recording))) {
|
||||
return null;
|
||||
}
|
||||
await recording.observation?.settleTrailing(recording.tabs.currentTabId, recording.steps);
|
||||
|
||||
recording.settled = true;
|
||||
recordings.delete(sessionId);
|
||||
@@ -558,6 +822,15 @@ export async function handleRecordStart(
|
||||
params: RecordStartParams,
|
||||
deps: RecordDeps = getDefaultDeps(),
|
||||
): Promise<RecordStartResult | RpcError> {
|
||||
const traceVersionOrErr = negotiatedTraceVersion(params);
|
||||
if (typeof traceVersionOrErr !== "number") return traceVersionOrErr;
|
||||
if (traceVersionOrErr === 3 && !deps.cdp) {
|
||||
return {
|
||||
code: "protocol_error",
|
||||
message: "trace v3 recording requires an active CDP connection",
|
||||
};
|
||||
}
|
||||
|
||||
const ctxOrErr = lookupSession(manager, params, "record_start");
|
||||
if (isRpcError(ctxOrErr)) return ctxOrErr;
|
||||
const ctx = ctxOrErr;
|
||||
@@ -574,37 +847,64 @@ export async function handleRecordStart(
|
||||
// on the destination page can RECORD_QUERY → rearm → show RecordOverlay
|
||||
// instead of flashing ControlOverlay ("Agent 正在控制").
|
||||
const requestId = makeRequestId(target.tabId);
|
||||
let resolveFinish!: (trace: Trace) => void;
|
||||
let resolveFinish!: (trace: RecordedTrace) => void;
|
||||
let rejectFinish!: (err: Error) => void;
|
||||
const finishPromise = new Promise<Trace>((resolve, reject) => {
|
||||
const finishPromise = new Promise<RecordedTrace>((resolve, reject) => {
|
||||
resolveFinish = resolve;
|
||||
rejectFinish = reject;
|
||||
});
|
||||
const navigateUrl = params.url ?? RECORD_DEFAULT_START_URL;
|
||||
const startedAtMs = Date.now();
|
||||
const maxPageTokens = params.max_page_tokens;
|
||||
const redactValues = params.redact_values ?? false;
|
||||
recordings.set(params.session_id, {
|
||||
requestId,
|
||||
tabId: target.tabId,
|
||||
tabs: new RecordingTabCoordinator(target.tabId, navigateUrl),
|
||||
agentWindowId: ctx.agentWindowId,
|
||||
startUrl: navigateUrl,
|
||||
...(params.purpose ? { purpose: params.purpose } : {}),
|
||||
steps: [],
|
||||
startedAt: new Date(startedAtMs).toISOString(),
|
||||
startedAtMs,
|
||||
traceVersion: traceVersionOrErr,
|
||||
supportsTabSwitchSteps: params.supports_tab_switch_steps === true,
|
||||
finishPromise,
|
||||
resolveFinish,
|
||||
rejectFinish,
|
||||
settled: false,
|
||||
finishing: false,
|
||||
currentUrl: navigateUrl,
|
||||
pendingNavigation: false,
|
||||
pendingNavigationDeadline: undefined,
|
||||
finishAttempt: null,
|
||||
observation:
|
||||
traceVersionOrErr === 3 && deps.cdp
|
||||
? new RecordingObservationRuntime({
|
||||
cdp: deps.cdp,
|
||||
tabsApi: deps.tabsApi,
|
||||
maxTokens: maxPageTokens,
|
||||
redactValues,
|
||||
})
|
||||
: null,
|
||||
stoppedBy: "user_finish",
|
||||
navigationCallbacks: new Set(),
|
||||
acceptingNavigation: true,
|
||||
actionQueue: Promise.resolve(),
|
||||
lastStepSequenceByProducer: new Map(),
|
||||
});
|
||||
deps.frameCoordinator?.begin(requestId, startedAtMs, target.tabId, async (scope) => {
|
||||
const recording = recordings.get(params.session_id);
|
||||
if (!recording || recording.requestId !== requestId || recording.settled) return;
|
||||
try {
|
||||
await recording.observation?.refreshDocument(scope.tabId, scope.producerId);
|
||||
} catch {
|
||||
// The normal action-time capture remains available if a child Document
|
||||
// is replaced while its readiness refresh is in flight.
|
||||
}
|
||||
});
|
||||
// Observe navigations for the whole recording lifetime; attach before
|
||||
// optional navigate so the destination load can rearm capture.
|
||||
ensureBrowserObservationListeners(deps);
|
||||
|
||||
const abortPending = async (notifyContent: boolean) => {
|
||||
recordings.get(params.session_id)?.observation?.cancel();
|
||||
deps.frameCoordinator?.cancel(requestId);
|
||||
recordings.delete(params.session_id);
|
||||
releaseBrowserObservationListenersIfIdle();
|
||||
if (notifyContent) {
|
||||
@@ -686,7 +986,7 @@ export async function handleRecordStart(
|
||||
return cancelledError();
|
||||
}
|
||||
active.startUrl = startUrl;
|
||||
active.currentUrl = startUrl;
|
||||
active.tabs.navigation(target.tabId, startUrl).currentUrl = startUrl;
|
||||
|
||||
if (isContentScriptRestrictedUrl(startUrl)) {
|
||||
await abortPending(false);
|
||||
@@ -694,7 +994,7 @@ export async function handleRecordStart(
|
||||
code: "invalid_params",
|
||||
message: params.url
|
||||
? `cannot record on restricted URL (${startUrl}); use an http(s) page`
|
||||
: `cannot record on restricted URL (${startUrl}); default start page must be injectable http(s)`,
|
||||
: `cannot record on restricted URL (${startUrl}); default start page https://example.com/ did not load — pass --url with a page you can open`,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -721,6 +1021,10 @@ export async function handleRecordStart(
|
||||
const startMsg: RecordStartMessage = { type: RECORD_START, requestId, startedAtMs };
|
||||
|
||||
try {
|
||||
if (deps.frameCoordinator) {
|
||||
const frameStarted = await deps.frameCoordinator.armTab(requestId, target.tabId);
|
||||
if (!frameStarted) throw new Error("top recording document did not start");
|
||||
}
|
||||
await sendRecordStartWithAck(target.tabId, startMsg, deps.sendToTab);
|
||||
} catch {
|
||||
await abortPending(true);
|
||||
@@ -731,6 +1035,17 @@ export async function handleRecordStart(
|
||||
};
|
||||
}
|
||||
|
||||
if (active.observation) {
|
||||
const activeRecording = recordings.get(params.session_id);
|
||||
if (activeRecording) {
|
||||
try {
|
||||
await activeRecording.observation?.captureInitial(target.tabId);
|
||||
} catch {
|
||||
// Proceed without initial observation; steps may be dropped by reducer.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
const cancelled = await abortIfCancelled(true);
|
||||
if (cancelled) return cancelled;
|
||||
@@ -755,7 +1070,7 @@ export async function handleRecordStop(
|
||||
};
|
||||
}
|
||||
|
||||
const trace = await finishRecording(params.session_id, deps);
|
||||
const trace = await finishRecording(params.session_id, deps, "cli_stop");
|
||||
if (!trace) {
|
||||
return {
|
||||
code: "protocol_error",
|
||||
@@ -785,9 +1100,9 @@ export async function handleRecordAwait(
|
||||
return { code: "cancelled", message: "record_await aborted" };
|
||||
}
|
||||
|
||||
const outcome = await new Promise<{ trace: Trace } | { error: RpcError }>((resolve) => {
|
||||
const outcome = await new Promise<{ trace: RecordedTrace } | { error: RpcError }>((resolve) => {
|
||||
let settled = false;
|
||||
const finish = (result: { trace: Trace } | { error: RpcError }) => {
|
||||
const finish = (result: { trace: RecordedTrace } | { error: RpcError }) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
if (timer) clearTimeout(timer);
|
||||
@@ -829,6 +1144,8 @@ export function clearRecordingForSession(sessionId: string): void {
|
||||
}
|
||||
void clearRearmTimersForRecording(recording, getDefaultDeps());
|
||||
if (!recording.settled) {
|
||||
getDefaultDeps().frameCoordinator?.cancel(recording.requestId);
|
||||
recording.observation?.cancel();
|
||||
recording.settled = true;
|
||||
recording.rejectFinish(new Error("recording cleared"));
|
||||
}
|
||||
|
||||
@@ -76,6 +76,7 @@ export interface SessionStopResult {
|
||||
}
|
||||
|
||||
export interface SessionStopDeps {
|
||||
signal?: AbortSignal;
|
||||
cdp?: {
|
||||
detachSession(sessionId: string): Promise<void>;
|
||||
};
|
||||
@@ -177,6 +178,9 @@ export async function handleSessionStop(
|
||||
message: `session ${params.session_id} unknown`,
|
||||
};
|
||||
}
|
||||
if (deps.signal?.aborted) {
|
||||
return { code: "cancelled", message: "session_stop aborted before teardown" };
|
||||
}
|
||||
|
||||
// Step 1: auto-return borrowed tabs. Iterate over a snapshot of the
|
||||
// ids so deletions during iteration do not break the Map iterator.
|
||||
@@ -187,6 +191,7 @@ export async function handleSessionStop(
|
||||
try {
|
||||
const tabManagement = {
|
||||
...(deps.tabManagement ?? {}),
|
||||
signal: deps.signal,
|
||||
isAgentWindowId:
|
||||
deps.tabManagement?.isAgentWindowId ??
|
||||
((windowId: number) => manager.findByWindowId(windowId) !== null),
|
||||
@@ -214,6 +219,17 @@ export async function handleSessionStop(
|
||||
}
|
||||
}
|
||||
|
||||
if (deps.signal?.aborted) {
|
||||
const nonCancelFailures = returnFailures?.filter((failure) => failure.code !== "cancelled");
|
||||
if (nonCancelFailures && nonCancelFailures.length > 0) {
|
||||
return {
|
||||
...(returnedTabIds.length > 0 ? { returned_tab_ids: returnedTabIds } : {}),
|
||||
return_failures: nonCancelFailures,
|
||||
};
|
||||
}
|
||||
return { code: "cancelled", message: "session_stop aborted during tab return" };
|
||||
}
|
||||
|
||||
const result: SessionStopResult = {};
|
||||
if (returnedTabIds.length > 0) result.returned_tab_ids = returnedTabIds;
|
||||
if (returnFailures && returnFailures.length > 0) {
|
||||
@@ -233,6 +249,10 @@ export async function handleSessionStop(
|
||||
// Step 3: detach CDP sessions this session opened (no-op if none).
|
||||
await deps.cdp?.detachSession(params.session_id);
|
||||
|
||||
if (deps.signal?.aborted) {
|
||||
return { code: "cancelled", message: "session_stop aborted before window close" };
|
||||
}
|
||||
|
||||
// Step 4: close the Agent Window and drop the context.
|
||||
await manager.stop(params.session_id);
|
||||
|
||||
|
||||
@@ -6,7 +6,8 @@
|
||||
// exactly the same sandbox + visibility rules as the M6 observation
|
||||
// handlers (review parity).
|
||||
|
||||
import type { DialogCursor } from "@/browser-driver/chromium-cdp";
|
||||
import type { CdpDebuggee, DialogCursor } from "@/browser-driver/chromium-cdp";
|
||||
import type { CdpFrameGraph, CdpTarget } from "@/browser-driver/frame-graph";
|
||||
import type { SessionContext, SessionManager } from "@/session-manager/manager";
|
||||
import { normaliseRef } from "@/session-manager/ref-store";
|
||||
import type { ConsoleResult, JavaScriptDialogInfo, RpcError } from "@/transport/types";
|
||||
@@ -51,9 +52,11 @@ export type { DialogCursor };
|
||||
*/
|
||||
export interface CdpRunner {
|
||||
send<T = unknown>(tabId: number, method: string, params?: object): Promise<T>;
|
||||
sendToTarget?<T = unknown>(target: CdpTarget, method: string, params?: object): Promise<T>;
|
||||
getFrameGraph?(tabId: number): Promise<CdpFrameGraph>;
|
||||
ensureAttachedToUrl?(tabId: number, expectedUrl: string | undefined): Promise<void>;
|
||||
trackSessionTab?(sessionId: string, tabId: number): void;
|
||||
onEvent?(handler: (source: chrome.debugger.Debuggee, method: string, params: unknown) => void): {
|
||||
onEvent?(handler: (source: CdpDebuggee, method: string, params: unknown) => void): {
|
||||
dispose(): void;
|
||||
};
|
||||
dialogCursor?(tabId: number): DialogCursor;
|
||||
@@ -68,6 +71,25 @@ export interface CdpRunner {
|
||||
): ConsoleResult;
|
||||
}
|
||||
|
||||
export function sendToCdpTarget<T = unknown>(
|
||||
cdp: CdpRunner,
|
||||
target: CdpTarget,
|
||||
method: string,
|
||||
params?: object,
|
||||
): Promise<T> {
|
||||
if (target.sessionId && cdp.sendToTarget) {
|
||||
return cdp.sendToTarget<T>(target, method, params);
|
||||
}
|
||||
return cdp.send<T>(target.tabId, method, params);
|
||||
}
|
||||
|
||||
export function cdpRunnerForTarget(cdp: CdpRunner, target: CdpTarget): CdpRunner {
|
||||
return {
|
||||
send: <T = unknown>(_tabId: number, method: string, params?: object) =>
|
||||
sendToCdpTarget<T>(cdp, target, method, params),
|
||||
};
|
||||
}
|
||||
|
||||
export interface BufferedReadBounds {
|
||||
since: number | undefined;
|
||||
limit: number;
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// Snapshot ref resolution — normalise `@eN` / `eN`, look up
|
||||
// `backendNodeId` via the session RefStore, and build stable
|
||||
// compound CDP node identity via the session RefStore, and build stable
|
||||
// `ref_not_found` errors for hard-failure tool paths.
|
||||
|
||||
import type { SessionContext } from "@/session-manager/manager";
|
||||
@@ -10,6 +10,13 @@ import { rpcError } from "./errors";
|
||||
export interface SnapshotRefLookup {
|
||||
backendNodeId: number;
|
||||
refKey: string;
|
||||
frameId?: string;
|
||||
cdpSessionId?: string;
|
||||
}
|
||||
|
||||
function refEntryForTab(ctx: SessionContext, refKey: string, tabId: number) {
|
||||
const entry = ctx.refStore.resolveEntry(refKey);
|
||||
return entry?.tabId === tabId ? entry : null;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -23,9 +30,14 @@ export function lookupSnapshotRef(
|
||||
tabId: number,
|
||||
): SnapshotRefLookup | null {
|
||||
const refKey = normaliseRef(ref);
|
||||
const backendNodeId = ctx.refStore.resolve(refKey, { tabId });
|
||||
if (backendNodeId === null) return null;
|
||||
return { backendNodeId, refKey };
|
||||
const entry = refEntryForTab(ctx, refKey, tabId);
|
||||
if (!entry) return null;
|
||||
return {
|
||||
backendNodeId: entry.backendNodeId,
|
||||
refKey,
|
||||
...(entry.frameId ? { frameId: entry.frameId } : {}),
|
||||
...(entry.cdpSessionId ? { cdpSessionId: entry.cdpSessionId } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -37,7 +49,7 @@ export function resolveSnapshotRef(
|
||||
ctx: SessionContext,
|
||||
ref: string,
|
||||
tabId: number,
|
||||
): { backendNodeId: number; refKey: string } | RpcError {
|
||||
): SnapshotRefLookup | RpcError {
|
||||
const looked = lookupSnapshotRef(ctx, ref, tabId);
|
||||
if (looked === null) {
|
||||
return rpcError(
|
||||
|
||||
@@ -18,6 +18,8 @@ function fakeSnapshotReply() {
|
||||
"auto",
|
||||
"type",
|
||||
"password",
|
||||
"value",
|
||||
"hunter2",
|
||||
];
|
||||
const i = (s: string) => S.indexOf(s);
|
||||
return {
|
||||
@@ -29,11 +31,12 @@ function fakeSnapshotReply() {
|
||||
nodeType: [1, 1, 1, 1],
|
||||
nodeName: [i("html"), i("body"), i("div"), i("input")],
|
||||
backendNodeId: [10, 11, 12, 13],
|
||||
attributes: [[], [], [], [i("type"), i("password")]],
|
||||
attributes: [[], [], [], [i("type"), i("password"), i("value"), i("hunter2")]],
|
||||
inputValue: { index: [3], value: [i("hunter2")] },
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [1, 2, 3],
|
||||
// styles columns follow requested computedStyles order: [position, pointer-events]
|
||||
// Legacy fixture: omitted style columns fall back to visible defaults.
|
||||
styles: [
|
||||
[i("static"), i("auto")],
|
||||
[i("fixed"), i("auto")],
|
||||
@@ -217,7 +220,8 @@ function makeCdp(snapshot: unknown) {
|
||||
{
|
||||
state: "filled",
|
||||
sensitive: true,
|
||||
defaultValue: "",
|
||||
defaultValue: "hunter2",
|
||||
value: "hunter2",
|
||||
placeholder: "secret",
|
||||
},
|
||||
],
|
||||
@@ -262,10 +266,11 @@ describe("captureViewModel", () => {
|
||||
|
||||
expect(input).toMatchObject({
|
||||
formState: "filled",
|
||||
formDefaultValue: "",
|
||||
formPlaceholder: "secret",
|
||||
attrs: { type: "password" },
|
||||
});
|
||||
expect(input?.formValue).toBeUndefined();
|
||||
expect(input?.formDefaultValue).toBeUndefined();
|
||||
expect(cdp.send).toHaveBeenCalledWith(
|
||||
4,
|
||||
"Runtime.evaluate",
|
||||
@@ -677,6 +682,46 @@ describe("captureViewModel", () => {
|
||||
expect(nodes.find((n) => n.backendNodeId === 11)?.cursor).toBe("auto");
|
||||
});
|
||||
|
||||
it("captures painted visibility as DOM fallback evidence", async () => {
|
||||
const S = ["html", "body", "button", "static", "auto", "visible", "hidden", "1", "0"];
|
||||
const i = (value: string) => S.indexOf(value);
|
||||
const snapshot = {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1, 1, 1],
|
||||
nodeName: [i("html"), i("body"), i("button"), i("button"), i("button")],
|
||||
backendNodeId: [10, 11, 12, 13, 14],
|
||||
attributes: [[], [], [], [], []],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [1, 2, 3, 4],
|
||||
styles: [
|
||||
[i("static"), i("auto"), i("auto"), i("visible"), i("1")],
|
||||
[i("static"), i("auto"), i("auto"), i("visible"), i("1")],
|
||||
[i("static"), i("auto"), i("auto"), i("hidden"), i("1")],
|
||||
[i("static"), i("auto"), i("auto"), i("visible"), i("0")],
|
||||
],
|
||||
bounds: [
|
||||
[0, 0, 1000, 800],
|
||||
[10, 10, 100, 30],
|
||||
[10, 50, 100, 30],
|
||||
[10, 90, 100, 30],
|
||||
],
|
||||
paintOrders: [0, 1, 2, 3],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const { nodes } = await captureViewModel(makeCdp(snapshot), 4);
|
||||
|
||||
expect(nodes.find((node) => node.backendNodeId === 12)?.rendered).toBe(true);
|
||||
expect(nodes.find((node) => node.backendNodeId === 13)?.rendered).toBe(false);
|
||||
expect(nodes.find((node) => node.backendNodeId === 14)?.rendered).toBe(false);
|
||||
});
|
||||
|
||||
it("subtracts scroll offset to make rects viewport-relative", async () => {
|
||||
const cdp = {
|
||||
send: vi.fn(async (_t: number, method: string) => {
|
||||
@@ -692,9 +737,7 @@ describe("captureViewModel", () => {
|
||||
};
|
||||
const { nodes } = await captureViewModel(cdp, 4);
|
||||
const div = nodes.find((n) => n.backendNodeId === 12);
|
||||
// local rect preserves viewport-relative scroll math.
|
||||
expect(div?.localRect?.y).toBe(-200);
|
||||
// exported rect is clipped to the top-level viewport before VOM consumes it.
|
||||
expect(div?.rect).toMatchObject({ y: 0, h: 600 });
|
||||
});
|
||||
|
||||
@@ -839,6 +882,7 @@ describe("captureViewModel", () => {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
frameId: "main-frame",
|
||||
scrollOffsetX: 0,
|
||||
scrollOffsetY: 0,
|
||||
nodes: {
|
||||
@@ -866,6 +910,7 @@ describe("captureViewModel", () => {
|
||||
},
|
||||
// documents[1]: the iframe's sub-document with a login input
|
||||
{
|
||||
frameId: "child-frame",
|
||||
scrollOffsetX: 0,
|
||||
scrollOffsetY: 0,
|
||||
nodes: {
|
||||
@@ -895,7 +940,14 @@ describe("captureViewModel", () => {
|
||||
throw new Error(method);
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
const { nodes, iframeNodes } = await captureViewModel(cdp, 4);
|
||||
const {
|
||||
nodes,
|
||||
iframeNodes,
|
||||
frameNodes,
|
||||
frameOwnerBackendNodeIds,
|
||||
frameParentIds,
|
||||
rootFrameId,
|
||||
} = await captureViewModel(cdp, 4);
|
||||
// Main frame has 4 nodes; iframe is included
|
||||
expect(nodes.find((n) => n.backendNodeId === 13)?.tag).toBe("iframe");
|
||||
// iframeNodes keyed by the <iframe> element's backendNodeId=13
|
||||
@@ -908,9 +960,13 @@ describe("captureViewModel", () => {
|
||||
expect(input?.ownerFrameBackendNodeId).toBe(13);
|
||||
expect(input?.localRect).toEqual({ x: 0, y: 0, w: 200, h: 40 });
|
||||
expect(input?.rect).toEqual({ x: 100, y: 300, w: 200, h: 40 });
|
||||
expect(rootFrameId).toBe("main-frame");
|
||||
expect(frameNodes?.get("child-frame")).toBe(subNodes);
|
||||
expect(frameOwnerBackendNodeIds?.get("child-frame")).toBe(13);
|
||||
expect(frameParentIds?.get("child-frame")).toBe("main-frame");
|
||||
});
|
||||
|
||||
it("clips iframe sub-document rects to the iframe viewport in top coordinates", async () => {
|
||||
it("captures iframe documents without requiring owner-frame geometry", async () => {
|
||||
const S = [
|
||||
"html",
|
||||
"body",
|
||||
@@ -938,16 +994,10 @@ describe("captureViewModel", () => {
|
||||
contentDocumentIndex: { index: [2], value: [1] },
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [1, 2],
|
||||
styles: [
|
||||
[i("static"), i("auto")],
|
||||
[i("static"), i("auto")],
|
||||
],
|
||||
bounds: [
|
||||
[0, 0, 1000, 800],
|
||||
[950, 760, 200, 100],
|
||||
],
|
||||
paintOrders: [0, 1],
|
||||
nodeIndex: [1],
|
||||
styles: [[i("static"), i("auto")]],
|
||||
bounds: [[0, 0, 1000, 800]],
|
||||
paintOrders: [0],
|
||||
},
|
||||
},
|
||||
{
|
||||
@@ -985,7 +1035,7 @@ describe("captureViewModel", () => {
|
||||
const input = iframeNodes.get(13)?.find((n) => n.backendNodeId === 101);
|
||||
|
||||
expect(input?.localRect).toEqual({ x: 20, y: 20, w: 120, h: 60 });
|
||||
expect(input?.rect).toEqual({ x: 970, y: 780, w: 30, h: 20 });
|
||||
expect(input?.rect).toBeNull();
|
||||
});
|
||||
|
||||
it("recursively normalizes nested iframe sub-documents", async () => {
|
||||
@@ -1080,8 +1130,10 @@ describe("captureViewModel", () => {
|
||||
const nestedIframe = iframeNodes.get(13)?.find((n) => n.backendNodeId === 23);
|
||||
const input = iframeNodes.get(23)?.find((n) => n.backendNodeId === 31);
|
||||
|
||||
expect(nestedIframe?.localRect).toEqual({ x: 5, y: 6, w: 100, h: 80 });
|
||||
expect(nestedIframe?.rect).toEqual({ x: 15, y: 26, w: 100, h: 80 });
|
||||
expect(input?.ownerFrameBackendNodeId).toBe(23);
|
||||
expect(input?.localRect).toEqual({ x: 1, y: 2, w: 40, h: 20 });
|
||||
expect(input?.rect).toEqual({ x: 16, y: 28, w: 40, h: 20 });
|
||||
});
|
||||
|
||||
|
||||
@@ -0,0 +1,198 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import type { CdpRunner } from "../../shared";
|
||||
import type { CapturedNode, CapturedViewModel } from "../capture";
|
||||
import { captureFrameData } from "../frame-capture";
|
||||
|
||||
function ownerNode(backendNodeId: number, x: number): CapturedNode {
|
||||
return {
|
||||
backendNodeId,
|
||||
parentBackendNodeId: 1,
|
||||
frameId: "main",
|
||||
ownerFrameBackendNodeId: null,
|
||||
tag: "iframe",
|
||||
attrs: {},
|
||||
rect: { x, y: 100, w: 300, h: 200 },
|
||||
localRect: { x, y: 100, w: 300, h: 200 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
};
|
||||
}
|
||||
|
||||
function childSnapshot(frameId: string, backendNodeId: number) {
|
||||
const strings = [frameId, "body", "button", "static", "auto", "pointer"];
|
||||
return {
|
||||
strings,
|
||||
documents: [
|
||||
{
|
||||
frameId,
|
||||
nodes: {
|
||||
parentIndex: [-1, 0],
|
||||
nodeName: [1, 2],
|
||||
backendNodeId: [backendNodeId - 1, backendNodeId],
|
||||
attributes: [[], []],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [0, 1],
|
||||
styles: [
|
||||
[3, 4, 4],
|
||||
[3, 4, 5],
|
||||
],
|
||||
bounds: [
|
||||
[0, 0, 300, 200],
|
||||
[10, 20, 100, 40],
|
||||
],
|
||||
paintOrders: [0, 1],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
describe("captureFrameData", () => {
|
||||
it("captures and positions multiple OOPIF documents missing from the root snapshot", async () => {
|
||||
const leftOwner = ownerNode(10, 50);
|
||||
const rightOwner = ownerNode(20, 500);
|
||||
const captured: CapturedViewModel = {
|
||||
nodes: [leftOwner, rightOwner],
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
frameNodes: new Map([["main", [leftOwner, rightOwner]]]),
|
||||
frameOwnerBackendNodeIds: new Map(),
|
||||
rootFrameId: "main",
|
||||
excludedBackendNodeIds: new Set(),
|
||||
};
|
||||
const sendToTarget = vi.fn(async (target, method) => {
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 300, clientHeight: 200, pageX: 0, pageY: 0 } };
|
||||
}
|
||||
if (method === "DOMSnapshot.enable" || method === "Accessibility.enable") return {};
|
||||
if (method === "DOMSnapshot.captureSnapshot") {
|
||||
return target.sessionId === "left-session"
|
||||
? childSnapshot("left", 101)
|
||||
: childSnapshot("right", 201);
|
||||
}
|
||||
if (method === "Accessibility.getFullAXTree") return { nodes: [] };
|
||||
throw new Error(`unexpected ${method}`);
|
||||
});
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async (_tabId, method, params) => {
|
||||
if (method === "Accessibility.enable") return {};
|
||||
if (method === "Accessibility.getFullAXTree") return { nodes: [] };
|
||||
if (method === "DOM.getBoxModel") {
|
||||
const backendNodeId = (params as { backendNodeId?: number })?.backendNodeId;
|
||||
const x = backendNodeId === 10 ? 50 : 500;
|
||||
return { model: { content: [x, 100, x + 300, 100, x + 300, 300, x, 300] } };
|
||||
}
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } };
|
||||
}
|
||||
throw new Error(`unexpected root ${method}`);
|
||||
}) as CdpRunner["send"],
|
||||
sendToTarget: sendToTarget as unknown as NonNullable<CdpRunner["sendToTarget"]>,
|
||||
getFrameGraph: vi.fn(async () => ({
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "left",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 10,
|
||||
target: { tabId: 4, sessionId: "left-session" },
|
||||
},
|
||||
{
|
||||
frameId: "right",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 20,
|
||||
target: { tabId: 4, sessionId: "right-session" },
|
||||
},
|
||||
],
|
||||
})),
|
||||
};
|
||||
|
||||
const trees = await captureFrameData(cdp, 4, captured);
|
||||
|
||||
expect(trees.map((tree) => tree.frameId)).toEqual(["main", "left", "right"]);
|
||||
expect(
|
||||
captured.frameNodes?.get("left")?.find((node) => node.backendNodeId === 101)?.rect,
|
||||
).toEqual({ x: 60, y: 120, w: 100, h: 40 });
|
||||
expect(
|
||||
captured.frameNodes?.get("right")?.find((node) => node.backendNodeId === 201)?.rect,
|
||||
).toEqual({ x: 510, y: 120, w: 100, h: 40 });
|
||||
expect(captured.frameOwnerBackendNodeIds).toEqual(
|
||||
new Map([
|
||||
["left", 10],
|
||||
["right", 20],
|
||||
]),
|
||||
);
|
||||
expect(captured.frameParentIds).toEqual(
|
||||
new Map([
|
||||
["left", "main"],
|
||||
["right", "main"],
|
||||
]),
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps OOPIF semantics when viewport projection is unavailable", async () => {
|
||||
const owner = { ...ownerNode(10, 50), rect: null, localRect: null };
|
||||
const captured: CapturedViewModel = {
|
||||
nodes: [owner],
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
frameNodes: new Map([["main", [owner]]]),
|
||||
rootFrameId: "main",
|
||||
excludedBackendNodeIds: new Set(),
|
||||
};
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async (_tabId, method) => {
|
||||
if (method === "Accessibility.enable") return {};
|
||||
if (method === "Accessibility.getFullAXTree") return { nodes: [] };
|
||||
if (method === "DOM.getBoxModel") throw new Error("owner geometry unavailable");
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } };
|
||||
}
|
||||
throw new Error(`unexpected root ${method}`);
|
||||
}) as CdpRunner["send"],
|
||||
sendToTarget: vi.fn(async (_target, method) => {
|
||||
if (method === "Page.getLayoutMetrics") throw new Error("viewport unavailable");
|
||||
if (method === "DOMSnapshot.enable" || method === "Accessibility.enable") return {};
|
||||
if (method === "DOMSnapshot.captureSnapshot") return childSnapshot("child", 101);
|
||||
if (method === "Accessibility.getFullAXTree") {
|
||||
return {
|
||||
nodes: [
|
||||
{
|
||||
nodeId: "button",
|
||||
backendDOMNodeId: 101,
|
||||
role: { type: "role", value: "button" },
|
||||
name: { type: "computedString", value: "Continue" },
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
throw new Error(`unexpected ${method}`);
|
||||
}) as unknown as NonNullable<CdpRunner["sendToTarget"]>,
|
||||
getFrameGraph: vi.fn(async () => ({
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "child",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 10,
|
||||
target: { tabId: 4, sessionId: "child-session" },
|
||||
},
|
||||
],
|
||||
})),
|
||||
};
|
||||
|
||||
const documents = await captureFrameData(cdp, 4, captured);
|
||||
const childDocument = documents.find((document) => document.frameId === "child");
|
||||
|
||||
expect(childDocument?.axNodes).toEqual([
|
||||
expect.objectContaining({ backendDOMNodeId: 101, frameId: "child" }),
|
||||
]);
|
||||
expect(childDocument?.domNodes.find((node) => node.backendNodeId === 101)).toEqual(
|
||||
expect.objectContaining({ rect: null, localRect: { x: 10, y: 20, w: 100, h: 40 } }),
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,174 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { CdpFrameGraph } from "@/browser-driver/frame-graph";
|
||||
import type { CapturedNode, CapturedViewModel } from "../capture";
|
||||
import { buildFrameDocuments, type FrameAxBatch, type FrameOwnedAxNode } from "../frame-document";
|
||||
|
||||
function domNode(frameId: string, backendNodeId: number): CapturedNode {
|
||||
return {
|
||||
backendNodeId,
|
||||
parentBackendNodeId: null,
|
||||
frameId,
|
||||
tag: "div",
|
||||
attrs: {},
|
||||
rect: { x: 0, y: 0, w: 20, h: 20 },
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
};
|
||||
}
|
||||
|
||||
function captured(frameNodes: Map<string, CapturedNode[]>): CapturedViewModel {
|
||||
return {
|
||||
nodes: frameNodes.get("main") ?? [],
|
||||
viewport: { width: 800, height: 600 },
|
||||
iframeNodes: new Map(),
|
||||
frameNodes,
|
||||
rootFrameId: "main",
|
||||
excludedBackendNodeIds: new Set(),
|
||||
};
|
||||
}
|
||||
|
||||
describe("buildFrameDocuments", () => {
|
||||
it("partitions mixed AX results by node ownership and cuts cross-frame parent edges", () => {
|
||||
const graph: CdpFrameGraph = {
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{ frameId: "child", parentFrameId: "main", target: { tabId: 4 } },
|
||||
],
|
||||
};
|
||||
const batches: FrameAxBatch<FrameOwnedAxNode>[] = [
|
||||
{
|
||||
frame: graph.frames[0],
|
||||
nodes: [
|
||||
{ nodeId: "root", frameId: "main", backendDOMNodeId: 1, childIds: ["h"] },
|
||||
{
|
||||
nodeId: "h",
|
||||
frameId: "child",
|
||||
parentId: "root",
|
||||
backendDOMNodeId: 100,
|
||||
childIds: ["virtual"],
|
||||
},
|
||||
{ nodeId: "virtual", parentId: "h" },
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
const documents = buildFrameDocuments(
|
||||
graph,
|
||||
batches,
|
||||
captured(
|
||||
new Map([
|
||||
["main", [domNode("main", 1)]],
|
||||
["child", [domNode("child", 100)]],
|
||||
]),
|
||||
),
|
||||
);
|
||||
|
||||
expect(documents.find((document) => document.frameId === "main")?.axNodes).toHaveLength(1);
|
||||
const childNodes = documents.find((document) => document.frameId === "child")?.axNodes ?? [];
|
||||
expect(childNodes.map((node) => node.nodeId)).toEqual(["h", "virtual"]);
|
||||
expect(childNodes.find((node) => node.nodeId === "h")?.parentId).toBeUndefined();
|
||||
expect(childNodes.find((node) => node.nodeId === "virtual")?.frameId).toBe("child");
|
||||
});
|
||||
|
||||
it("deduplicates overlapping same-target AX results using the strongest ownership", () => {
|
||||
const graph: CdpFrameGraph = {
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{ frameId: "child", parentFrameId: "main", target: { tabId: 4 } },
|
||||
],
|
||||
};
|
||||
const childNode = { nodeId: "shared", backendDOMNodeId: 100 };
|
||||
const batches: FrameAxBatch<FrameOwnedAxNode>[] = [
|
||||
{
|
||||
frame: graph.frames[0],
|
||||
nodes: [{ ...childNode, frameId: "child" }],
|
||||
},
|
||||
{
|
||||
frame: graph.frames[1],
|
||||
nodes: [childNode],
|
||||
},
|
||||
];
|
||||
|
||||
const documents = buildFrameDocuments(
|
||||
graph,
|
||||
batches,
|
||||
captured(
|
||||
new Map([
|
||||
["main", [domNode("main", 1)]],
|
||||
["child", [domNode("child", 100)]],
|
||||
]),
|
||||
),
|
||||
);
|
||||
|
||||
expect(documents.find((document) => document.frameId === "main")?.axNodes).toEqual([]);
|
||||
expect(documents.find((document) => document.frameId === "child")?.axNodes).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("uses target-scoped backend ownership when OOPIF backend ids collide", () => {
|
||||
const graph: CdpFrameGraph = {
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "left",
|
||||
parentFrameId: "main",
|
||||
target: { tabId: 4, sessionId: "left-session" },
|
||||
},
|
||||
{
|
||||
frameId: "right",
|
||||
parentFrameId: "main",
|
||||
target: { tabId: 4, sessionId: "right-session" },
|
||||
},
|
||||
],
|
||||
};
|
||||
const batches: FrameAxBatch<FrameOwnedAxNode>[] = graph.frames.slice(1).map((frame) => ({
|
||||
frame,
|
||||
nodes: [{ nodeId: `${frame.frameId}-node`, backendDOMNodeId: 7 }],
|
||||
}));
|
||||
|
||||
const documents = buildFrameDocuments(
|
||||
graph,
|
||||
batches,
|
||||
captured(
|
||||
new Map([
|
||||
["main", []],
|
||||
["left", [domNode("left", 7)]],
|
||||
["right", [domNode("right", 7)]],
|
||||
]),
|
||||
),
|
||||
);
|
||||
|
||||
expect(documents.find((document) => document.frameId === "left")?.axNodes[0]?.nodeId).toBe(
|
||||
"left-node",
|
||||
);
|
||||
expect(documents.find((document) => document.frameId === "right")?.axNodes[0]?.nodeId).toBe(
|
||||
"right-node",
|
||||
);
|
||||
});
|
||||
|
||||
it("uses captured frame parent identity instead of guessing from backend ids", () => {
|
||||
const frameNodes = new Map([
|
||||
["main", [domNode("main", 1)]],
|
||||
["left", [domNode("left", 7)]],
|
||||
["right", [domNode("right", 7)]],
|
||||
["child", [domNode("child", 9)]],
|
||||
]);
|
||||
const capture = captured(frameNodes);
|
||||
capture.frameOwnerBackendNodeIds = new Map([["child", 7]]);
|
||||
capture.frameParentIds = new Map([["child", "left"]]);
|
||||
const frames = ["main", "left", "right", "child"].map((frameId) => ({
|
||||
frame: { frameId, target: { tabId: 4 } },
|
||||
nodes: [],
|
||||
}));
|
||||
|
||||
const documents = buildFrameDocuments(null, frames, capture);
|
||||
|
||||
expect(documents.find((document) => document.frameId === "child")).toMatchObject({
|
||||
parentFrameId: "left",
|
||||
ownerBackendNodeId: 7,
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,183 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import type { CdpTarget } from "@/browser-driver/frame-graph";
|
||||
import type { CdpRunner } from "../../shared";
|
||||
import type { CapturedNode } from "../capture";
|
||||
import type { FrameDocument, FrameOwnedAxNode } from "../frame-document";
|
||||
import { probeTooltipNames } from "../name-enrichment";
|
||||
import { buildSemanticGraph, resolveSemanticGraph } from "../semantic-graph";
|
||||
import { stableIdentifierName } from "../semantic-graph/name-evidence";
|
||||
import { frameBackendKey } from "../semantic-graph/types";
|
||||
|
||||
function document(
|
||||
target: CdpTarget,
|
||||
backendNodeId: number,
|
||||
accessibleName?: string,
|
||||
frameId = "child",
|
||||
x = 10,
|
||||
): FrameDocument<FrameOwnedAxNode> {
|
||||
const node: CapturedNode = {
|
||||
backendNodeId,
|
||||
parentBackendNodeId: null,
|
||||
frameId,
|
||||
tag: "button",
|
||||
attrs: {
|
||||
id: "toolInsertRecord",
|
||||
"data-test-id": "toolInsertRecord",
|
||||
...(accessibleName ? { "aria-label": accessibleName } : {}),
|
||||
},
|
||||
rect: { x, y: 20, w: 40, h: 30 },
|
||||
localRect: { x: 2, y: 3, w: 40, h: 30 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
cursor: "pointer",
|
||||
};
|
||||
return {
|
||||
frameId,
|
||||
parentFrameId: "main",
|
||||
contextScopeId: "child",
|
||||
target,
|
||||
axNodes: [],
|
||||
domNodes: [node],
|
||||
url: "https://example.com/frame",
|
||||
};
|
||||
}
|
||||
|
||||
function semantics(documents: FrameDocument<FrameOwnedAxNode>[]) {
|
||||
return resolveSemanticGraph(
|
||||
buildSemanticGraph({
|
||||
documents,
|
||||
viewport: { width: 800, height: 600 },
|
||||
rootFrameId: documents[0]?.frameId,
|
||||
}),
|
||||
{ identifierFallback: false },
|
||||
);
|
||||
}
|
||||
|
||||
describe("tooltip name enrichment", () => {
|
||||
it("uses the owning frame target and upgrades an identifier name from a tooltip", async () => {
|
||||
const target = { tabId: 7, sessionId: "oopif-session" };
|
||||
let hovered = false;
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async (_tabId, method, params) => {
|
||||
if (method === "Input.dispatchMouseEvent") {
|
||||
hovered = (params as { x?: number }).x === 30;
|
||||
}
|
||||
return {};
|
||||
}) as CdpRunner["send"],
|
||||
sendToTarget: vi.fn(async (_target, method) => {
|
||||
if (method === "Page.createIsolatedWorld") return { executionContextId: 11 };
|
||||
if (method === "DOM.resolveNode") return { object: { objectId: "button-501" } };
|
||||
if (method === "Runtime.callFunctionOn") {
|
||||
return { result: { value: hovered ? ["插入行"] : [] } };
|
||||
}
|
||||
return {};
|
||||
}) as CdpRunner["sendToTarget"],
|
||||
};
|
||||
|
||||
const documents = [document(target, 501)];
|
||||
const graph = semantics(documents);
|
||||
const names = await probeTooltipNames(cdp, 7, documents, graph);
|
||||
|
||||
expect(names.get(frameBackendKey("child", 501))).toBe("插入行");
|
||||
expect(cdp.sendToTarget).toHaveBeenCalledWith(
|
||||
target,
|
||||
"Page.createIsolatedWorld",
|
||||
expect.objectContaining({ frameId: "child" }),
|
||||
);
|
||||
|
||||
const sendsBeforeCache = vi.mocked(cdp.send).mock.calls.length;
|
||||
const targetSendsBeforeCache = vi.mocked(
|
||||
cdp.sendToTarget as NonNullable<CdpRunner["sendToTarget"]>,
|
||||
).mock.calls.length;
|
||||
const cached = await probeTooltipNames(cdp, 7, documents, graph);
|
||||
expect(cached.get(frameBackendKey("child", 501))).toBe("插入行");
|
||||
expect(cdp.send).toHaveBeenCalledTimes(sendsBeforeCache);
|
||||
expect(cdp.sendToTarget).toHaveBeenCalledTimes(targetSendsBeforeCache);
|
||||
});
|
||||
|
||||
it("does not probe controls that already have a higher-confidence name", async () => {
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async () => ({})) as CdpRunner["send"],
|
||||
sendToTarget: vi.fn(async () => ({ executionContextId: 12 })) as CdpRunner["sendToTarget"],
|
||||
};
|
||||
|
||||
const documents = [document({ tabId: 8 }, 601, "Add a new record")];
|
||||
const names = await probeTooltipNames(cdp, 8, documents, semantics(documents));
|
||||
|
||||
expect(names.size).toBe(0);
|
||||
expect(cdp.send).not.toHaveBeenCalled();
|
||||
expect(cdp.sendToTarget).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("does not collect supplemental names from form controls", async () => {
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async () => ({})) as CdpRunner["send"],
|
||||
sendToTarget: vi.fn(async () => ({ executionContextId: 12 })) as CdpRunner["sendToTarget"],
|
||||
};
|
||||
const inputDocument = document({ tabId: 8 }, 602);
|
||||
inputDocument.domNodes[0] = {
|
||||
...inputDocument.domNodes[0],
|
||||
tag: "input",
|
||||
attrs: { type: "email" },
|
||||
formValue: "user@example.com",
|
||||
formDefaultValue: "",
|
||||
formState: "filled",
|
||||
};
|
||||
|
||||
const names = await probeTooltipNames(cdp, 8, [inputDocument], semantics([inputDocument]));
|
||||
|
||||
expect(names.size).toBe(0);
|
||||
expect(cdp.send).not.toHaveBeenCalled();
|
||||
expect(cdp.sendToTarget).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("keeps tooltip evidence isolated across sibling frame sessions", async () => {
|
||||
let hoveredX = -10;
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async (_tabId, method, params) => {
|
||||
if (method === "Input.dispatchMouseEvent") {
|
||||
hoveredX = (params as { x: number }).x;
|
||||
}
|
||||
return {};
|
||||
}) as CdpRunner["send"],
|
||||
sendToTarget: vi.fn(async (target, method, params) => {
|
||||
if (method === "Page.createIsolatedWorld") {
|
||||
return { executionContextId: target.sessionId === "left-session" ? 21 : 22 };
|
||||
}
|
||||
if (method === "DOM.resolveNode") {
|
||||
return { object: { objectId: target.sessionId === "left-session" ? "left" : "right" } };
|
||||
}
|
||||
if (method === "Runtime.callFunctionOn") {
|
||||
const objectId = (params as { objectId: string }).objectId;
|
||||
const value =
|
||||
objectId === "left" && hoveredX === 30
|
||||
? ["Left action"]
|
||||
: objectId === "right" && hoveredX === 130
|
||||
? ["Right action"]
|
||||
: [];
|
||||
return { result: { value } };
|
||||
}
|
||||
return {};
|
||||
}) as CdpRunner["sendToTarget"],
|
||||
};
|
||||
const documents = [
|
||||
document({ tabId: 9, sessionId: "left-session" }, 701, undefined, "left", 10),
|
||||
document({ tabId: 9, sessionId: "right-session" }, 701, undefined, "right", 110),
|
||||
];
|
||||
|
||||
const names = await probeTooltipNames(cdp, 9, documents, semantics(documents));
|
||||
|
||||
expect(names.get(frameBackendKey("left", 701))).toBe("Left action");
|
||||
expect(names.get(frameBackendKey("right", 701))).toBe("Right action");
|
||||
});
|
||||
});
|
||||
|
||||
describe("stable identifier evidence", () => {
|
||||
it("accepts readable action identifiers and rejects framework-generated ids", () => {
|
||||
expect(stableIdentifierName({ id: "toolInsertRecord" })).toBe("Insert record");
|
||||
expect(stableIdentifierName({ id: "reactivateAccount" })).toBe("Reactivate account");
|
||||
expect(stableIdentifierName({ id: "headlessui-menu-button-r1" })).toBeUndefined();
|
||||
expect(stableIdentifierName({ id: "7f15a3bc91" })).toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,82 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
RECORD_DOCUMENT_ATTRIBUTE,
|
||||
recordingDocumentMarkerValue,
|
||||
} from "@/shared/recording-document-identity";
|
||||
import { projectRecordSafeObservation } from "../record-safe-observation";
|
||||
|
||||
describe("projectRecordSafeObservation", () => {
|
||||
it("projects the opaque recording identity without exposing captured attributes", () => {
|
||||
const recordingDocumentId = "123e4567-e89b-42d3-a456-426614174000";
|
||||
const result = projectRecordSafeObservation({
|
||||
rootFrameId: "child",
|
||||
frameDocuments: [
|
||||
{
|
||||
frameId: "child",
|
||||
target: { tabId: 4, sessionId: "oopif-session" },
|
||||
contextScopeId: "scope-1",
|
||||
axNodes: [],
|
||||
domNodes: [
|
||||
{
|
||||
backendNodeId: 1,
|
||||
parentBackendNodeId: null,
|
||||
tag: "html",
|
||||
attrs: {
|
||||
[RECORD_DOCUMENT_ATTRIBUTE]: recordingDocumentMarkerValue(recordingDocumentId),
|
||||
value: "user@example.com",
|
||||
},
|
||||
rect: null,
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
rendered: { text: '@vom 1\nRootWebArea "Example"', refs: [], truncated: false },
|
||||
});
|
||||
|
||||
expect(result.frames).toEqual([
|
||||
{
|
||||
frameId: "child",
|
||||
target: { tabId: 4, sessionId: "oopif-session" },
|
||||
recordingDocumentId,
|
||||
},
|
||||
]);
|
||||
const dumped = JSON.stringify(result);
|
||||
expect(dumped).not.toContain("user@example.com");
|
||||
expect(dumped).not.toContain("attrs");
|
||||
expect(dumped).not.toContain("domNodes");
|
||||
expect(dumped).not.toContain("axNodes");
|
||||
});
|
||||
|
||||
it("does not project a page-controlled marker value as recording identity", () => {
|
||||
const result = projectRecordSafeObservation({
|
||||
rootFrameId: "root",
|
||||
frameDocuments: [
|
||||
{
|
||||
frameId: "root",
|
||||
target: { tabId: 4 },
|
||||
contextScopeId: "scope-1",
|
||||
axNodes: [],
|
||||
domNodes: [
|
||||
{
|
||||
backendNodeId: 1,
|
||||
parentBackendNodeId: null,
|
||||
tag: "html",
|
||||
attrs: { [RECORD_DOCUMENT_ATTRIBUTE]: "bsk:user@example.com" },
|
||||
rect: null,
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
rendered: { text: '@vom 1\nRootWebArea "Example"', refs: [], truncated: false },
|
||||
});
|
||||
|
||||
expect(result.frames).toEqual([{ frameId: "root", target: { tabId: 4 } }]);
|
||||
expect(JSON.stringify(result)).not.toContain("user@example.com");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,782 @@
|
||||
import { isVomReferenceNode, renderVom } from "@browser-skill/vom";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { CapturedNode } from "../capture";
|
||||
import type { FrameDocument } from "../frame-document";
|
||||
import { buildSemanticVomScene, type SemanticAxNode } from "../semantic-graph";
|
||||
|
||||
function dom(
|
||||
backendNodeId: number,
|
||||
parentBackendNodeId: number | null,
|
||||
tag: string,
|
||||
attrs: Record<string, string> = {},
|
||||
textContent?: string,
|
||||
): CapturedNode {
|
||||
return {
|
||||
backendNodeId,
|
||||
parentBackendNodeId,
|
||||
tag,
|
||||
attrs,
|
||||
rect: { x: 0, y: 0, w: 100, h: 30 },
|
||||
paintOrder: backendNodeId,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
...(textContent ? { textContent } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
function document(
|
||||
frameId: string,
|
||||
axNodes: SemanticAxNode[],
|
||||
domNodes: CapturedNode[],
|
||||
frame?: Pick<FrameDocument<SemanticAxNode>, "parentFrameId" | "ownerBackendNodeId">,
|
||||
): FrameDocument<SemanticAxNode> {
|
||||
return {
|
||||
frameId,
|
||||
contextScopeId: frameId,
|
||||
target: { tabId: 1 },
|
||||
axNodes,
|
||||
domNodes,
|
||||
...frame,
|
||||
};
|
||||
}
|
||||
|
||||
describe("semantic VOM graph", () => {
|
||||
it("preserves backend-less AX structure without making it referenceable", () => {
|
||||
const scene = buildSemanticVomScene({
|
||||
viewport: { width: 800, height: 600 },
|
||||
rootFrameId: "main",
|
||||
documents: [
|
||||
document(
|
||||
"main",
|
||||
[
|
||||
{
|
||||
nodeId: "root",
|
||||
backendDOMNodeId: 1,
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
childIds: ["toolbar"],
|
||||
},
|
||||
{
|
||||
nodeId: "toolbar",
|
||||
parentId: "root",
|
||||
role: { type: "role", value: "toolbar" },
|
||||
name: { type: "computedString", value: "Formatting" },
|
||||
childIds: ["bold", "italic"],
|
||||
},
|
||||
{
|
||||
nodeId: "bold",
|
||||
parentId: "toolbar",
|
||||
backendDOMNodeId: 2,
|
||||
role: { type: "role", value: "button" },
|
||||
name: { type: "computedString", value: "Bold" },
|
||||
},
|
||||
{
|
||||
nodeId: "italic",
|
||||
parentId: "toolbar",
|
||||
backendDOMNodeId: 3,
|
||||
role: { type: "role", value: "button" },
|
||||
name: { type: "computedString", value: "Italic" },
|
||||
},
|
||||
],
|
||||
[dom(1, null, "body"), dom(2, 1, "button"), dom(3, 1, "button")],
|
||||
),
|
||||
],
|
||||
});
|
||||
|
||||
const toolbar = scene.nodes.find((node) => node.role === "toolbar");
|
||||
const buttons = scene.nodes.filter((node) => node.role === "button");
|
||||
expect(toolbar).toEqual(expect.objectContaining({ name: "Formatting", referenceable: false }));
|
||||
expect(buttons.map((node) => node.parentId)).toEqual([toolbar?.id, toolbar?.id]);
|
||||
expect(toolbar && isVomReferenceNode(toolbar)).toBe(false);
|
||||
expect(renderVom(scene).refs.map((ref) => ref.backendNodeId)).toEqual([2, 3]);
|
||||
});
|
||||
|
||||
it("resolves missing AX names through one DOM naming pipeline", () => {
|
||||
const scene = buildSemanticVomScene({
|
||||
viewport: { width: 800, height: 600 },
|
||||
rootFrameId: "main",
|
||||
documents: [
|
||||
document(
|
||||
"main",
|
||||
[
|
||||
{
|
||||
nodeId: "root",
|
||||
backendDOMNodeId: 1,
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
childIds: ["labelled", "titled", "text"],
|
||||
},
|
||||
{
|
||||
nodeId: "labelled",
|
||||
parentId: "root",
|
||||
backendDOMNodeId: 2,
|
||||
role: { type: "role", value: "button" },
|
||||
},
|
||||
{
|
||||
nodeId: "titled",
|
||||
parentId: "root",
|
||||
backendDOMNodeId: 3,
|
||||
role: { type: "role", value: "button" },
|
||||
},
|
||||
{
|
||||
nodeId: "text",
|
||||
parentId: "root",
|
||||
backendDOMNodeId: 4,
|
||||
role: { type: "role", value: "button" },
|
||||
},
|
||||
],
|
||||
[
|
||||
dom(1, null, "body"),
|
||||
dom(2, 1, "button", { "aria-label": "Add row" }),
|
||||
dom(3, 1, "button", { title: "Settings" }),
|
||||
dom(4, 1, "button"),
|
||||
dom(5, 4, "span", {}, "Save"),
|
||||
],
|
||||
),
|
||||
],
|
||||
});
|
||||
|
||||
expect(scene.nodes.filter((node) => node.role === "button").map((node) => node.name)).toEqual([
|
||||
"Add row",
|
||||
"Settings",
|
||||
"Save",
|
||||
]);
|
||||
});
|
||||
|
||||
it("aggregates AX text at the semantic text frontier", () => {
|
||||
const scene = buildSemanticVomScene({
|
||||
viewport: { width: 800, height: 600 },
|
||||
rootFrameId: "main",
|
||||
documents: [
|
||||
document(
|
||||
"main",
|
||||
[
|
||||
{
|
||||
nodeId: "root",
|
||||
backendDOMNodeId: 1,
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
childIds: ["covered", "inline-only", "repeated", "labelled"],
|
||||
},
|
||||
{
|
||||
nodeId: "covered",
|
||||
parentId: "root",
|
||||
backendDOMNodeId: 2,
|
||||
role: { type: "role", value: "paragraph" },
|
||||
childIds: ["covered-static"],
|
||||
},
|
||||
{
|
||||
nodeId: "covered-static",
|
||||
parentId: "covered",
|
||||
role: { type: "role", value: "StaticText" },
|
||||
name: { type: "computedString", value: "Covered text" },
|
||||
childIds: ["covered-inline"],
|
||||
},
|
||||
{
|
||||
nodeId: "covered-inline",
|
||||
parentId: "covered-static",
|
||||
role: { type: "role", value: "InlineTextBox" },
|
||||
name: { type: "computedString", value: "Covered text" },
|
||||
},
|
||||
{
|
||||
nodeId: "inline-only",
|
||||
parentId: "root",
|
||||
backendDOMNodeId: 3,
|
||||
role: { type: "role", value: "paragraph" },
|
||||
childIds: ["inline-leaf"],
|
||||
},
|
||||
{
|
||||
nodeId: "inline-leaf",
|
||||
parentId: "inline-only",
|
||||
role: { type: "role", value: "InlineTextBox" },
|
||||
name: { type: "computedString", value: "Inline fallback" },
|
||||
},
|
||||
{
|
||||
nodeId: "repeated",
|
||||
parentId: "root",
|
||||
backendDOMNodeId: 4,
|
||||
role: { type: "role", value: "paragraph" },
|
||||
childIds: ["first-go", "second-go"],
|
||||
},
|
||||
{
|
||||
nodeId: "first-go",
|
||||
parentId: "repeated",
|
||||
role: { type: "role", value: "StaticText" },
|
||||
name: { type: "computedString", value: "Go" },
|
||||
},
|
||||
{
|
||||
nodeId: "second-go",
|
||||
parentId: "repeated",
|
||||
role: { type: "role", value: "StaticText" },
|
||||
name: { type: "computedString", value: "Go" },
|
||||
},
|
||||
{
|
||||
nodeId: "labelled",
|
||||
parentId: "root",
|
||||
backendDOMNodeId: 5,
|
||||
role: { type: "role", value: "button" },
|
||||
childIds: ["visible-label"],
|
||||
},
|
||||
{
|
||||
nodeId: "visible-label",
|
||||
parentId: "labelled",
|
||||
role: { type: "role", value: "StaticText" },
|
||||
name: { type: "computedString", value: "Visible text" },
|
||||
},
|
||||
],
|
||||
[
|
||||
dom(1, null, "body"),
|
||||
dom(2, 1, "p"),
|
||||
dom(3, 1, "p"),
|
||||
dom(4, 1, "p"),
|
||||
dom(5, 1, "button", { "aria-label": "Explicit action" }),
|
||||
],
|
||||
),
|
||||
],
|
||||
});
|
||||
|
||||
expect(scene.nodes.find((node) => node.backendNodeId === 2)?.name).toBe("Covered text");
|
||||
expect(scene.nodes.find((node) => node.backendNodeId === 3)?.name).toBe("Inline fallback");
|
||||
expect(scene.nodes.find((node) => node.backendNodeId === 4)?.name).toBe("Go Go");
|
||||
expect(scene.nodes.find((node) => node.backendNodeId === 5)?.name).toBe("Explicit action");
|
||||
expect(renderVom(scene).text).not.toContain("Covered text Covered text");
|
||||
});
|
||||
|
||||
it("uses stable identifiers only as a final name fallback", () => {
|
||||
const base = document(
|
||||
"main",
|
||||
[
|
||||
{
|
||||
nodeId: "root",
|
||||
backendDOMNodeId: 1,
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
childIds: ["fallback", "labelled", "container"],
|
||||
},
|
||||
{
|
||||
nodeId: "fallback",
|
||||
parentId: "root",
|
||||
backendDOMNodeId: 2,
|
||||
role: { type: "role", value: "button" },
|
||||
},
|
||||
{
|
||||
nodeId: "labelled",
|
||||
parentId: "root",
|
||||
backendDOMNodeId: 3,
|
||||
role: { type: "role", value: "button" },
|
||||
name: { type: "computedString", value: "Localized name" },
|
||||
},
|
||||
{
|
||||
nodeId: "container",
|
||||
parentId: "root",
|
||||
backendDOMNodeId: 4,
|
||||
role: { type: "role", value: "generic" },
|
||||
},
|
||||
],
|
||||
[
|
||||
dom(1, null, "body"),
|
||||
dom(2, 1, "button", { "data-test-id": "toolInsertRecord" }),
|
||||
dom(3, 1, "button", { id: "toolIgnoredFallback" }),
|
||||
dom(4, 1, "div", { id: "toolbarContainer" }),
|
||||
],
|
||||
);
|
||||
const staticScene = buildSemanticVomScene({
|
||||
viewport: { width: 800, height: 600 },
|
||||
rootFrameId: "main",
|
||||
documents: [base],
|
||||
});
|
||||
const enrichedScene = buildSemanticVomScene({
|
||||
viewport: { width: 800, height: 600 },
|
||||
rootFrameId: "main",
|
||||
documents: [base],
|
||||
supplementalNames: new Map([["main\u00002", "插入行"]]),
|
||||
});
|
||||
|
||||
expect(staticScene.nodes.find((node) => node.backendNodeId === 2)?.name).toBe("Insert record");
|
||||
expect(staticScene.nodes.find((node) => node.backendNodeId === 3)?.name).toBe("Localized name");
|
||||
expect(staticScene.nodes.find((node) => node.backendNodeId === 4)?.name).toBeUndefined();
|
||||
expect(enrichedScene.nodes.find((node) => node.backendNodeId === 2)?.name).toBe("插入行");
|
||||
expect(enrichedScene.nodes.find((node) => node.backendNodeId === 3)?.name).toBe(
|
||||
"Localized name",
|
||||
);
|
||||
});
|
||||
|
||||
it("isolates equal backend node ids in sibling frames", () => {
|
||||
const scene = buildSemanticVomScene({
|
||||
viewport: { width: 1000, height: 800 },
|
||||
rootFrameId: "main",
|
||||
documents: [
|
||||
document(
|
||||
"main",
|
||||
[
|
||||
{
|
||||
nodeId: "root",
|
||||
backendDOMNodeId: 1,
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
childIds: ["owner-a", "owner-b"],
|
||||
},
|
||||
{
|
||||
nodeId: "owner-a",
|
||||
parentId: "root",
|
||||
backendDOMNodeId: 10,
|
||||
role: { type: "role", value: "Iframe" },
|
||||
},
|
||||
{
|
||||
nodeId: "owner-b",
|
||||
parentId: "root",
|
||||
backendDOMNodeId: 20,
|
||||
role: { type: "role", value: "Iframe" },
|
||||
},
|
||||
],
|
||||
[dom(1, null, "body"), dom(10, 1, "iframe"), dom(20, 1, "iframe")],
|
||||
),
|
||||
document(
|
||||
"frame-a",
|
||||
[
|
||||
{
|
||||
nodeId: "button-a",
|
||||
backendDOMNodeId: 5,
|
||||
role: { type: "role", value: "button" },
|
||||
childIds: ["text-a"],
|
||||
},
|
||||
{
|
||||
nodeId: "text-a",
|
||||
parentId: "button-a",
|
||||
role: { type: "role", value: "StaticText" },
|
||||
name: { type: "computedString", value: "Accept" },
|
||||
childIds: ["inline-a"],
|
||||
},
|
||||
{
|
||||
nodeId: "inline-a",
|
||||
parentId: "text-a",
|
||||
role: { type: "role", value: "InlineTextBox" },
|
||||
name: { type: "computedString", value: "Accept" },
|
||||
},
|
||||
],
|
||||
[dom(5, null, "button")],
|
||||
{ parentFrameId: "main", ownerBackendNodeId: 10 },
|
||||
),
|
||||
document(
|
||||
"frame-b",
|
||||
[
|
||||
{
|
||||
nodeId: "button-b",
|
||||
backendDOMNodeId: 5,
|
||||
role: { type: "role", value: "button" },
|
||||
childIds: ["text-b"],
|
||||
},
|
||||
{
|
||||
nodeId: "text-b",
|
||||
parentId: "button-b",
|
||||
role: { type: "role", value: "StaticText" },
|
||||
name: { type: "computedString", value: "Decline" },
|
||||
childIds: ["inline-b"],
|
||||
},
|
||||
{
|
||||
nodeId: "inline-b",
|
||||
parentId: "text-b",
|
||||
role: { type: "role", value: "InlineTextBox" },
|
||||
name: { type: "computedString", value: "Decline" },
|
||||
},
|
||||
],
|
||||
[dom(5, null, "button")],
|
||||
{ parentFrameId: "main", ownerBackendNodeId: 20 },
|
||||
),
|
||||
],
|
||||
});
|
||||
|
||||
const frameButtons = scene.nodes.filter((node) => node.backendNodeId === 5);
|
||||
const owners = new Map(
|
||||
scene.nodes
|
||||
.filter((node) => node.backendNodeId === 10 || node.backendNodeId === 20)
|
||||
.map((node) => [node.backendNodeId, node.id]),
|
||||
);
|
||||
expect(frameButtons.map((node) => node.frameId)).toEqual(["frame-a", "frame-b"]);
|
||||
expect(frameButtons.map((node) => node.name)).toEqual(["Accept", "Decline"]);
|
||||
expect(frameButtons.map((node) => node.parentId)).toEqual([owners.get(10), owners.get(20)]);
|
||||
expect(renderVom(scene).refs.map((ref) => [ref.frameId, ref.backendNodeId])).toEqual([
|
||||
["frame-a", 5],
|
||||
["frame-b", 5],
|
||||
]);
|
||||
});
|
||||
|
||||
it("requires render evidence before promoting DOM fallback semantics", () => {
|
||||
const hiddenOwner = {
|
||||
...dom(10, 1, "iframe", { hidden: "" }),
|
||||
rect: null,
|
||||
localRect: null,
|
||||
rendered: false,
|
||||
};
|
||||
const hiddenButton = {
|
||||
...dom(2, 1, "button", { "aria-label": "Hidden action" }),
|
||||
rendered: false,
|
||||
};
|
||||
const scene = buildSemanticVomScene({
|
||||
viewport: { width: 800, height: 600 },
|
||||
rootFrameId: "main",
|
||||
documents: [
|
||||
document(
|
||||
"main",
|
||||
[
|
||||
{
|
||||
nodeId: "root",
|
||||
backendDOMNodeId: 1,
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
},
|
||||
],
|
||||
[dom(1, null, "body"), hiddenButton, hiddenOwner],
|
||||
),
|
||||
document("hidden-frame", [], [dom(20, null, "button", {}, "Frame action")], {
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 10,
|
||||
}),
|
||||
],
|
||||
});
|
||||
|
||||
expect(scene.nodes.some((node) => node.backendNodeId === 2)).toBe(false);
|
||||
expect(scene.nodes.some((node) => node.backendNodeId === 10)).toBe(false);
|
||||
expect(scene.nodes.some((node) => node.frameId === "hidden-frame")).toBe(false);
|
||||
expect(renderVom(scene).refs).toEqual([]);
|
||||
});
|
||||
|
||||
it("retains rendered DOM fallback controls when AX semantics are unavailable", () => {
|
||||
const scene = buildSemanticVomScene({
|
||||
viewport: { width: 800, height: 600 },
|
||||
rootFrameId: "main",
|
||||
documents: [
|
||||
document(
|
||||
"main",
|
||||
[],
|
||||
[dom(1, null, "body"), { ...dom(2, 1, "button", {}, "Save"), rendered: true }],
|
||||
),
|
||||
],
|
||||
});
|
||||
|
||||
expect(renderVom(scene).refs).toEqual([
|
||||
expect.objectContaining({ backendNodeId: 2, role: "button", name: "Save" }),
|
||||
]);
|
||||
});
|
||||
|
||||
it("lets explicit DOM structure override ignored AX metadata", () => {
|
||||
const scene = buildSemanticVomScene({
|
||||
viewport: { width: 800, height: 600 },
|
||||
rootFrameId: "main",
|
||||
documents: [
|
||||
document(
|
||||
"main",
|
||||
[
|
||||
{
|
||||
nodeId: "root",
|
||||
backendDOMNodeId: 1,
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
childIds: ["toolbar"],
|
||||
},
|
||||
{
|
||||
nodeId: "toolbar",
|
||||
parentId: "root",
|
||||
backendDOMNodeId: 2,
|
||||
ignored: true,
|
||||
role: { type: "role", value: "generic" },
|
||||
childIds: ["wrapper"],
|
||||
},
|
||||
{
|
||||
nodeId: "wrapper",
|
||||
parentId: "toolbar",
|
||||
backendDOMNodeId: 5,
|
||||
role: { type: "role", value: "generic" },
|
||||
childIds: ["bold", "italic"],
|
||||
},
|
||||
{
|
||||
nodeId: "bold",
|
||||
parentId: "wrapper",
|
||||
backendDOMNodeId: 3,
|
||||
role: { type: "role", value: "button" },
|
||||
name: { type: "computedString", value: "Bold" },
|
||||
},
|
||||
{
|
||||
nodeId: "italic",
|
||||
parentId: "wrapper",
|
||||
backendDOMNodeId: 4,
|
||||
role: { type: "role", value: "button" },
|
||||
name: { type: "computedString", value: "Italic" },
|
||||
},
|
||||
],
|
||||
[
|
||||
dom(1, null, "body"),
|
||||
dom(2, 1, "div", { role: "toolbar" }),
|
||||
dom(5, 2, "div"),
|
||||
dom(3, 5, "button"),
|
||||
dom(4, 5, "button"),
|
||||
],
|
||||
),
|
||||
],
|
||||
});
|
||||
|
||||
const toolbar = scene.nodes.find((node) => node.backendNodeId === 2);
|
||||
const buttons = scene.nodes.filter((node) => [3, 4].includes(node.backendNodeId ?? -1));
|
||||
expect(toolbar).toEqual(expect.objectContaining({ role: "toolbar" }));
|
||||
expect(toolbar && isVomReferenceNode(toolbar)).toBe(false);
|
||||
expect(scene.nodes.some((node) => node.role === "group")).toBe(false);
|
||||
expect(buttons.map((node) => node.parentId)).toEqual([toolbar?.id, toolbar?.id]);
|
||||
});
|
||||
|
||||
it("infers an unnamed group only from independent interaction branches", () => {
|
||||
const scene = buildSemanticVomScene({
|
||||
viewport: { width: 800, height: 600 },
|
||||
rootFrameId: "main",
|
||||
documents: [
|
||||
document(
|
||||
"main",
|
||||
[
|
||||
{
|
||||
nodeId: "root",
|
||||
backendDOMNodeId: 1,
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
childIds: ["wrapper"],
|
||||
},
|
||||
{
|
||||
nodeId: "wrapper",
|
||||
parentId: "root",
|
||||
backendDOMNodeId: 2,
|
||||
role: { type: "role", value: "generic" },
|
||||
childIds: ["first", "second"],
|
||||
},
|
||||
{
|
||||
nodeId: "first",
|
||||
parentId: "wrapper",
|
||||
backendDOMNodeId: 3,
|
||||
role: { type: "role", value: "button" },
|
||||
name: { type: "computedString", value: "First" },
|
||||
},
|
||||
{
|
||||
nodeId: "second",
|
||||
parentId: "wrapper",
|
||||
backendDOMNodeId: 4,
|
||||
role: { type: "role", value: "button" },
|
||||
name: { type: "computedString", value: "Second" },
|
||||
},
|
||||
],
|
||||
[dom(1, null, "body"), dom(2, 1, "div"), dom(3, 2, "button"), dom(4, 2, "button")],
|
||||
),
|
||||
],
|
||||
});
|
||||
|
||||
const group = scene.nodes.find((node) => node.backendNodeId === 2);
|
||||
const buttons = scene.nodes.filter((node) => [3, 4].includes(node.backendNodeId ?? -1));
|
||||
expect(group).toEqual(expect.objectContaining({ role: "group" }));
|
||||
expect(group?.name).toBeUndefined();
|
||||
expect(group && isVomReferenceNode(group)).toBe(false);
|
||||
expect(buttons.map((node) => node.parentId)).toEqual([group?.id, group?.id]);
|
||||
|
||||
const rendered = renderVom(scene);
|
||||
const lines = rendered.text.split("\n");
|
||||
const groupLine = lines.findIndex((line) => line.trim() === "group");
|
||||
expect(groupLine).toBeGreaterThanOrEqual(0);
|
||||
for (const ref of rendered.refs) {
|
||||
expect(ref.line).toBeGreaterThan(groupLine);
|
||||
expect(lines[ref.line]).toMatch(/^\s+@e\d+ button/);
|
||||
}
|
||||
});
|
||||
|
||||
it("never promotes a form value into its semantic name through supplemental evidence", () => {
|
||||
const secret = "user@example.com";
|
||||
const input = {
|
||||
...dom(2, 1, "input", { type: "email" }),
|
||||
formValue: secret,
|
||||
formDefaultValue: "",
|
||||
formState: "filled" as const,
|
||||
};
|
||||
const scene = buildSemanticVomScene({
|
||||
viewport: { width: 800, height: 600 },
|
||||
rootFrameId: "main",
|
||||
supplementalNames: new Map([["main\u00002", secret]]),
|
||||
identifierFallback: false,
|
||||
documents: [
|
||||
document(
|
||||
"main",
|
||||
[
|
||||
{
|
||||
nodeId: "root",
|
||||
backendDOMNodeId: 1,
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
childIds: ["email"],
|
||||
},
|
||||
{
|
||||
nodeId: "email",
|
||||
parentId: "root",
|
||||
backendDOMNodeId: 2,
|
||||
role: { type: "role", value: "textbox" },
|
||||
value: { value: secret },
|
||||
},
|
||||
],
|
||||
[dom(1, null, "body"), input],
|
||||
),
|
||||
],
|
||||
});
|
||||
|
||||
const field = scene.nodes.find((node) => node.backendNodeId === 2);
|
||||
expect(field?.name).toBeUndefined();
|
||||
const rendered = renderVom(scene, { redactValues: true });
|
||||
expect(rendered.text).toContain("•••");
|
||||
expect(JSON.stringify(rendered)).not.toContain(secret);
|
||||
});
|
||||
|
||||
it("keeps uncertain single-branch wrappers flat", () => {
|
||||
const scene = buildSemanticVomScene({
|
||||
viewport: { width: 800, height: 600 },
|
||||
rootFrameId: "main",
|
||||
documents: [
|
||||
document(
|
||||
"main",
|
||||
[
|
||||
{
|
||||
nodeId: "root",
|
||||
backendDOMNodeId: 1,
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
childIds: ["wrapper"],
|
||||
},
|
||||
{
|
||||
nodeId: "wrapper",
|
||||
parentId: "root",
|
||||
backendDOMNodeId: 2,
|
||||
role: { type: "role", value: "generic" },
|
||||
childIds: ["button"],
|
||||
},
|
||||
{
|
||||
nodeId: "button",
|
||||
parentId: "wrapper",
|
||||
backendDOMNodeId: 3,
|
||||
role: { type: "role", value: "button" },
|
||||
name: { type: "computedString", value: "Only action" },
|
||||
},
|
||||
],
|
||||
[dom(1, null, "body"), dom(2, 1, "div"), dom(3, 2, "button")],
|
||||
),
|
||||
],
|
||||
});
|
||||
|
||||
expect(scene.nodes.some((node) => node.role === "group")).toBe(false);
|
||||
expect(scene.nodes.find((node) => node.backendNodeId === 3)?.parentId).toBe(
|
||||
scene.nodes.find((node) => node.backendNodeId === 1)?.id,
|
||||
);
|
||||
});
|
||||
|
||||
it("does not infer across mixed content boundaries", () => {
|
||||
const scene = buildSemanticVomScene({
|
||||
viewport: { width: 800, height: 600 },
|
||||
rootFrameId: "main",
|
||||
documents: [
|
||||
document(
|
||||
"main",
|
||||
[
|
||||
{
|
||||
nodeId: "root",
|
||||
backendDOMNodeId: 1,
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
childIds: ["wrapper"],
|
||||
},
|
||||
{
|
||||
nodeId: "wrapper",
|
||||
parentId: "root",
|
||||
backendDOMNodeId: 2,
|
||||
role: { type: "role", value: "generic" },
|
||||
childIds: ["first", "second", "paragraph"],
|
||||
},
|
||||
{
|
||||
nodeId: "first",
|
||||
parentId: "wrapper",
|
||||
backendDOMNodeId: 3,
|
||||
role: { type: "role", value: "button" },
|
||||
name: { type: "computedString", value: "First" },
|
||||
},
|
||||
{
|
||||
nodeId: "second",
|
||||
parentId: "wrapper",
|
||||
backendDOMNodeId: 4,
|
||||
role: { type: "role", value: "button" },
|
||||
name: { type: "computedString", value: "Second" },
|
||||
},
|
||||
{
|
||||
nodeId: "paragraph",
|
||||
parentId: "wrapper",
|
||||
backendDOMNodeId: 5,
|
||||
role: { type: "role", value: "paragraph" },
|
||||
},
|
||||
],
|
||||
[
|
||||
dom(1, null, "body"),
|
||||
dom(2, 1, "div"),
|
||||
dom(3, 2, "button"),
|
||||
dom(4, 2, "button"),
|
||||
dom(5, 2, "p", {}, "Description"),
|
||||
],
|
||||
),
|
||||
],
|
||||
});
|
||||
|
||||
expect(scene.nodes.some((node) => node.role === "group")).toBe(false);
|
||||
});
|
||||
|
||||
it("keeps only the innermost reliable inferred group", () => {
|
||||
const scene = buildSemanticVomScene({
|
||||
viewport: { width: 800, height: 600 },
|
||||
rootFrameId: "main",
|
||||
documents: [
|
||||
document(
|
||||
"main",
|
||||
[
|
||||
{
|
||||
nodeId: "root",
|
||||
backendDOMNodeId: 1,
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
childIds: ["outer"],
|
||||
},
|
||||
{
|
||||
nodeId: "outer",
|
||||
parentId: "root",
|
||||
backendDOMNodeId: 2,
|
||||
role: { type: "role", value: "generic" },
|
||||
childIds: ["inner", "third"],
|
||||
},
|
||||
{
|
||||
nodeId: "inner",
|
||||
parentId: "outer",
|
||||
backendDOMNodeId: 3,
|
||||
role: { type: "role", value: "generic" },
|
||||
childIds: ["first", "second"],
|
||||
},
|
||||
{
|
||||
nodeId: "first",
|
||||
parentId: "inner",
|
||||
backendDOMNodeId: 4,
|
||||
role: { type: "role", value: "button" },
|
||||
name: { type: "computedString", value: "First" },
|
||||
},
|
||||
{
|
||||
nodeId: "second",
|
||||
parentId: "inner",
|
||||
backendDOMNodeId: 5,
|
||||
role: { type: "role", value: "button" },
|
||||
name: { type: "computedString", value: "Second" },
|
||||
},
|
||||
{
|
||||
nodeId: "third",
|
||||
parentId: "outer",
|
||||
backendDOMNodeId: 6,
|
||||
role: { type: "role", value: "button" },
|
||||
name: { type: "computedString", value: "Third" },
|
||||
},
|
||||
],
|
||||
[
|
||||
dom(1, null, "body"),
|
||||
dom(2, 1, "div"),
|
||||
dom(3, 2, "div"),
|
||||
dom(4, 3, "button"),
|
||||
dom(5, 3, "button"),
|
||||
dom(6, 2, "button"),
|
||||
],
|
||||
),
|
||||
],
|
||||
});
|
||||
|
||||
const groups = scene.nodes.filter((node) => node.role === "group");
|
||||
expect(groups.map((node) => node.backendNodeId)).toEqual([3]);
|
||||
expect(scene.nodes.find((node) => node.backendNodeId === 6)?.parentId).not.toBe(groups[0]?.id);
|
||||
});
|
||||
});
|
||||
@@ -1,15 +1,17 @@
|
||||
// CDP capture adapter: DOMSnapshot.captureSnapshot + Page.getLayoutMetrics
|
||||
// → CapturedNode[] + Viewport. This is the ONLY VOM module that touches
|
||||
// raw CDP. The captureSnapshot reply is columnar (parallel arrays + a
|
||||
// shared string table); we request exactly three computed styles so the
|
||||
// `styles` columns are [position, pointer-events, cursor] in that order.
|
||||
// shared string table); requested computed styles are decoded through
|
||||
// STYLE_COL so capture and visibility policy share one explicit contract.
|
||||
|
||||
import type { Rect, Viewport } from "@browser-skill/vom";
|
||||
import { evaluateHoverTrigger } from "@/lib/hover-trigger-policy";
|
||||
import { isOverlayHostNode, OVERLAY_HOST_SELECTOR } from "../../lib/overlay-bridge";
|
||||
import { childFrameProjection, type GeometryProjection, projectRectToViewport } from "../geometry";
|
||||
import type { CdpRunner } from "../shared";
|
||||
import { clearHover, waitForHover } from "./hover-perception";
|
||||
|
||||
const REQUESTED_STYLES = ["position", "pointer-events", "cursor"] as const;
|
||||
const REQUESTED_STYLES = ["position", "pointer-events", "cursor", "visibility", "opacity"] as const;
|
||||
const STYLE_COL = Object.fromEntries(
|
||||
REQUESTED_STYLES.map((name, index) => [name, index]),
|
||||
) as Record<(typeof REQUESTED_STYLES)[number], number>;
|
||||
@@ -17,13 +19,14 @@ const STYLE_COL = Object.fromEntries(
|
||||
export interface CapturedNode {
|
||||
backendNodeId: number;
|
||||
parentBackendNodeId: number | null;
|
||||
frameId?: string;
|
||||
/** Owning iframe backend node id; `null` for the top-level document. */
|
||||
ownerFrameBackendNodeId?: number | null;
|
||||
tag: string;
|
||||
attrs: Record<string, string>;
|
||||
/** Top-level viewport-relative CSS px, clipped to the owning frame viewport. */
|
||||
rect: Rect | null;
|
||||
/** Frame-local viewport-relative CSS px before top-level translation/clipping. */
|
||||
/** Frame-local viewport-relative CSS px before top-level projection. */
|
||||
localRect?: Rect | null;
|
||||
paintOrder: number;
|
||||
position: string;
|
||||
@@ -36,6 +39,12 @@ export interface CapturedNode {
|
||||
* `textContent`: the live parser always sets it, hand-built fixtures may not.
|
||||
*/
|
||||
cursor?: string;
|
||||
/**
|
||||
* Whether the live DOM snapshot provides a painted, non-hidden box for this
|
||||
* node. Semantic resolution uses this only for DOM fallback nodes; AX-backed
|
||||
* nodes remain authoritative even when they are outside the viewport.
|
||||
*/
|
||||
rendered?: boolean;
|
||||
textContent?: string;
|
||||
formState?: "empty" | "filled" | "default";
|
||||
formValue?: string;
|
||||
@@ -57,6 +66,11 @@ export interface CapturedViewModel {
|
||||
nodes: CapturedNode[];
|
||||
viewport: Viewport;
|
||||
iframeNodes: CapturedIframeNodes;
|
||||
frameNodes?: Map<string, CapturedNode[]>;
|
||||
frameOwnerBackendNodeIds?: Map<string, number>;
|
||||
/** Explicit DOMSnapshot frame ancestry; never inferred from backend node ids. */
|
||||
frameParentIds?: Map<string, string>;
|
||||
rootFrameId?: string;
|
||||
surfaceProbes?: CapturedSurfaceProbe[];
|
||||
/** Backend node ids belonging to the agent overlay host + its shadow subtree. */
|
||||
excludedBackendNodeIds: Set<number>;
|
||||
@@ -92,7 +106,12 @@ interface SparseArray {
|
||||
value: number[];
|
||||
}
|
||||
|
||||
interface RareBooleanData {
|
||||
index: number[];
|
||||
}
|
||||
|
||||
interface SnapshotDocument {
|
||||
frameId?: string | number;
|
||||
scrollOffsetX?: number;
|
||||
scrollOffsetY?: number;
|
||||
nodes?: {
|
||||
@@ -107,6 +126,10 @@ interface SnapshotDocument {
|
||||
nodeValue?: number[];
|
||||
/** Maps node array index → index into `documents[]` for frame content. */
|
||||
contentDocumentIndex?: SparseArray;
|
||||
inputValue?: SparseArray;
|
||||
textValue?: SparseArray;
|
||||
inputChecked?: RareBooleanData;
|
||||
optionSelected?: RareBooleanData;
|
||||
};
|
||||
layout?: {
|
||||
nodeIndex?: number[];
|
||||
@@ -116,6 +139,12 @@ interface SnapshotDocument {
|
||||
};
|
||||
}
|
||||
|
||||
function snapshotFrameId(document: SnapshotDocument, strings: string[]): string | undefined {
|
||||
if (typeof document.frameId === "string") return document.frameId || undefined;
|
||||
if (typeof document.frameId === "number") return str(strings, document.frameId) || undefined;
|
||||
return undefined;
|
||||
}
|
||||
|
||||
interface SnapshotReply {
|
||||
strings?: string[];
|
||||
documents?: SnapshotDocument[];
|
||||
@@ -135,6 +164,25 @@ function isFormControlTag(tag: string): boolean {
|
||||
return tag === "input" || tag === "textarea" || tag === "select";
|
||||
}
|
||||
|
||||
function isSensitiveFormControl(node: Pick<CapturedNode, "tag" | "attrs">): boolean {
|
||||
return node.tag === "input" && (node.attrs.type ?? "").toLowerCase() === "password";
|
||||
}
|
||||
|
||||
function snapshotFormState(
|
||||
value: string | undefined,
|
||||
defaultValue: string,
|
||||
hasDefaultValue: boolean,
|
||||
sensitive: boolean,
|
||||
): CapturedNode["formState"] {
|
||||
if (sensitive) {
|
||||
if (value === undefined && !hasDefaultValue) return undefined;
|
||||
return (value ?? defaultValue) === "" ? "empty" : "filled";
|
||||
}
|
||||
if (value === undefined) return undefined;
|
||||
if (value === "") return "empty";
|
||||
return value === defaultValue ? "default" : "filled";
|
||||
}
|
||||
|
||||
const MAX_FORM_ENRICH_CONTROLS = 250;
|
||||
|
||||
interface CapturedFormState {
|
||||
@@ -166,9 +214,8 @@ function formStateBatchExpression(maxControls: number): string {
|
||||
return {
|
||||
state,
|
||||
sensitive,
|
||||
defaultValue,
|
||||
placeholder,
|
||||
...(sensitive ? {} : { value: rawValue }),
|
||||
...(sensitive ? {} : { value: rawValue, defaultValue }),
|
||||
};
|
||||
};
|
||||
const collect = (doc) => {
|
||||
@@ -210,10 +257,14 @@ function applyFormStates(nodes: CapturedNode[], states: CapturedFormState[]): vo
|
||||
index += 1;
|
||||
if (!state) continue;
|
||||
node.formState = state.state;
|
||||
node.formDefaultValue = state.defaultValue ?? "";
|
||||
node.formPlaceholder = state.placeholder ?? "";
|
||||
if (!state.sensitive && state.value !== undefined) {
|
||||
node.formValue = state.value;
|
||||
if (state.sensitive || isSensitiveFormControl(node)) {
|
||||
delete node.formValue;
|
||||
delete node.formDefaultValue;
|
||||
delete node.attrs.value;
|
||||
} else {
|
||||
node.formDefaultValue = state.defaultValue ?? "";
|
||||
if (state.value !== undefined) node.formValue = state.value;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -272,9 +323,9 @@ interface ParseDocumentResult {
|
||||
}
|
||||
|
||||
interface FrameContext {
|
||||
frameId?: string;
|
||||
ownerFrameBackendNodeId: number | null;
|
||||
originInTop: { x: number; y: number };
|
||||
clipRectInTop: Rect;
|
||||
projection: GeometryProjection | null;
|
||||
scrollX: number;
|
||||
scrollY: number;
|
||||
}
|
||||
@@ -293,38 +344,6 @@ function devicePixelRatio(metrics: LayoutMetricsReply): number {
|
||||
return dpr >= 1 ? dpr : 1;
|
||||
}
|
||||
|
||||
function viewportRect(viewport: Viewport): Rect {
|
||||
return { x: 0, y: 0, w: Math.max(0, viewport.width), h: Math.max(0, viewport.height) };
|
||||
}
|
||||
|
||||
function intersectRects(a: Rect, b: Rect): Rect | null {
|
||||
const x1 = Math.max(a.x, b.x);
|
||||
const y1 = Math.max(a.y, b.y);
|
||||
const x2 = Math.min(a.x + a.w, b.x + b.w);
|
||||
const y2 = Math.min(a.y + a.h, b.y + b.h);
|
||||
if (x2 <= x1 || y2 <= y1) return null;
|
||||
return { x: x1, y: y1, w: x2 - x1, h: y2 - y1 };
|
||||
}
|
||||
|
||||
function rectInTop(localRect: Rect, context: FrameContext): Rect | null {
|
||||
const translated = {
|
||||
x: context.originInTop.x + localRect.x,
|
||||
y: context.originInTop.y + localRect.y,
|
||||
w: localRect.w,
|
||||
h: localRect.h,
|
||||
};
|
||||
return intersectRects(translated, context.clipRectInTop);
|
||||
}
|
||||
|
||||
function unclipRectInTop(localRect: Rect, context: FrameContext): Rect {
|
||||
return {
|
||||
x: context.originInTop.x + localRect.x,
|
||||
y: context.originInTop.y + localRect.y,
|
||||
w: localRect.w,
|
||||
h: localRect.h,
|
||||
};
|
||||
}
|
||||
|
||||
function sparseIndexMap(sparse: SparseArray | undefined): Map<number, number> {
|
||||
const out = new Map<number, number>();
|
||||
if (!sparse?.index || !sparse.value) return out;
|
||||
@@ -448,36 +467,6 @@ function hoverStateExpression(): string {
|
||||
})()`;
|
||||
}
|
||||
|
||||
async function wait(ms: number, signal?: AbortSignal): Promise<void> {
|
||||
throwIfAborted(signal);
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const timer = setTimeout(() => {
|
||||
signal?.removeEventListener("abort", onAbort);
|
||||
resolve();
|
||||
}, ms);
|
||||
const onAbort = () => {
|
||||
clearTimeout(timer);
|
||||
signal?.removeEventListener("abort", onAbort);
|
||||
reject(captureAbortError());
|
||||
};
|
||||
signal?.addEventListener("abort", onAbort, { once: true });
|
||||
});
|
||||
}
|
||||
|
||||
async function clearHover(cdp: CdpRunner, tabId: number): Promise<void> {
|
||||
await cdp
|
||||
.send(tabId, "Input.dispatchMouseEvent", {
|
||||
type: "mouseMoved",
|
||||
x: -10,
|
||||
y: -10,
|
||||
})
|
||||
.catch(() =>
|
||||
cdp.send(tabId, "Input.dispatchMouseEvent", { type: "mouseMoved", x: 0, y: 0 }).catch(() => {
|
||||
// best effort
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
function capturedText(node: CapturedNode): string | undefined {
|
||||
const value =
|
||||
node.attrs["aria-label"] ?? node.attrs.title ?? node.attrs.alt ?? node.textContent ?? "";
|
||||
@@ -657,7 +646,7 @@ async function probeHoverSurfaces(
|
||||
try {
|
||||
await clearHover(cdp, tabId);
|
||||
throwIfAborted(options.signal);
|
||||
await wait(HOVER_SETTLE_MS, options.signal);
|
||||
await waitForHover(HOVER_SETTLE_MS, options.signal);
|
||||
const baselineReply = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
|
||||
expression: hoverStateExpression(),
|
||||
returnByValue: true,
|
||||
@@ -672,7 +661,7 @@ async function probeHoverSurfaces(
|
||||
y: candidate.y,
|
||||
});
|
||||
throwIfAborted(options.signal);
|
||||
await wait(HOVER_SETTLE_MS, options.signal);
|
||||
await waitForHover(HOVER_SETTLE_MS, options.signal);
|
||||
const collected = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
|
||||
expression: hoverStateExpression(),
|
||||
returnByValue: true,
|
||||
@@ -805,6 +794,12 @@ function parseDocumentNodes(
|
||||
const posCol = STYLE_COL.position;
|
||||
const peCol = STYLE_COL["pointer-events"];
|
||||
const cursorCol = STYLE_COL.cursor;
|
||||
const visibilityCol = STYLE_COL.visibility;
|
||||
const opacityCol = STYLE_COL.opacity;
|
||||
const inputValues = sparseIndexMap(dn.inputValue);
|
||||
const textValues = sparseIndexMap(dn.textValue);
|
||||
const checkedInputs = new Set(dn.inputChecked?.index ?? []);
|
||||
const selectedOptions = new Set(dn.optionSelected?.index ?? []);
|
||||
|
||||
// Collect visible text from #text child nodes so element CapturedNodes
|
||||
// carry a textContent value usable as a button/link label fallback.
|
||||
@@ -845,6 +840,7 @@ function parseDocumentNodes(
|
||||
let position = "static";
|
||||
let pointerEvents = "auto";
|
||||
let cursor = "auto";
|
||||
let rendered = false;
|
||||
const li = layoutByNode.get(n);
|
||||
if (li !== undefined) {
|
||||
const b = dl?.bounds?.[li];
|
||||
@@ -855,13 +851,23 @@ function parseDocumentNodes(
|
||||
w: b[2] / dpr,
|
||||
h: b[3] / dpr,
|
||||
};
|
||||
rect = rectInTop(localRect, context);
|
||||
const bounds = context.projection
|
||||
? projectRectToViewport(localRect, context.projection)
|
||||
: null;
|
||||
rect = bounds ? { x: bounds.x, y: bounds.y, w: bounds.width, h: bounds.height } : null;
|
||||
}
|
||||
paintOrder = dl?.paintOrders?.[li] ?? 0;
|
||||
const styleRow = dl?.styles?.[li] ?? [];
|
||||
position = str(strings, styleRow[posCol]) || "static";
|
||||
pointerEvents = str(strings, styleRow[peCol]) || "auto";
|
||||
cursor = str(strings, styleRow[cursorCol]) || "auto";
|
||||
const visibility = str(strings, styleRow[visibilityCol]) || "visible";
|
||||
const opacity = str(strings, styleRow[opacityCol]) || "1";
|
||||
rendered =
|
||||
localRect !== null &&
|
||||
visibility !== "hidden" &&
|
||||
visibility !== "collapse" &&
|
||||
(Number.parseFloat(opacity) || 0) > 0;
|
||||
}
|
||||
|
||||
// Skip non-element nodes (#text, #cdata-section, etc.) — they carry no
|
||||
@@ -870,9 +876,24 @@ function parseDocumentNodes(
|
||||
|
||||
const textContent = nodeTextContent.get(n);
|
||||
|
||||
const rawFormValueIndex = tag === "textarea" ? textValues.get(n) : inputValues.get(n);
|
||||
const rawFormValue =
|
||||
rawFormValueIndex !== undefined ? str(strings, rawFormValueIndex) : undefined;
|
||||
const sensitive = isSensitiveFormControl({ tag, attrs });
|
||||
const formDefaultValue = attrs.value ?? "";
|
||||
const formValue = sensitive ? undefined : rawFormValue;
|
||||
const formState = snapshotFormState(
|
||||
rawFormValue,
|
||||
formDefaultValue,
|
||||
Object.prototype.hasOwnProperty.call(attrs, "value"),
|
||||
sensitive,
|
||||
);
|
||||
if (sensitive) delete attrs.value;
|
||||
|
||||
nodes.push({
|
||||
backendNodeId,
|
||||
parentBackendNodeId,
|
||||
...(context.frameId ? { frameId: context.frameId } : {}),
|
||||
ownerFrameBackendNodeId: context.ownerFrameBackendNodeId,
|
||||
tag,
|
||||
attrs,
|
||||
@@ -882,7 +903,18 @@ function parseDocumentNodes(
|
||||
position,
|
||||
pointerEvents,
|
||||
cursor,
|
||||
rendered,
|
||||
textContent,
|
||||
...(formValue !== undefined ? { formValue } : {}),
|
||||
...(tag === "input" || tag === "textarea"
|
||||
? {
|
||||
formPlaceholder: attrs.placeholder ?? "",
|
||||
...(!sensitive ? { formDefaultValue } : {}),
|
||||
...(formState ? { formState } : {}),
|
||||
}
|
||||
: {}),
|
||||
...(checkedInputs.has(n) ? { formValue: "true", formState: "filled" } : {}),
|
||||
...(selectedOptions.has(n) ? { formValue: attrs.value ?? textContent ?? "" } : {}),
|
||||
});
|
||||
}
|
||||
return { nodes, excludedBackendNodeIds };
|
||||
@@ -890,6 +922,8 @@ function parseDocumentNodes(
|
||||
|
||||
interface ParseFrameDocumentsResult {
|
||||
iframeNodes: CapturedIframeNodes;
|
||||
frameOwnerBackendNodeIds: Map<string, number>;
|
||||
frameParentIds: Map<string, string>;
|
||||
excludedBackendNodeIds: Set<number>;
|
||||
}
|
||||
|
||||
@@ -903,10 +937,14 @@ function parseChildFrameDocuments(
|
||||
visited = new Set<number>(),
|
||||
): ParseFrameDocumentsResult {
|
||||
const iframeNodes: CapturedIframeNodes = new Map();
|
||||
const frameOwnerBackendNodeIds = new Map<string, number>();
|
||||
const frameParentIds = new Map<string, string>();
|
||||
const excludedBackendNodeIds = new Set<number>();
|
||||
const parentDoc = documents[parentDocIndex];
|
||||
const cdi = sparseIndexMap(parentDoc?.nodes?.contentDocumentIndex);
|
||||
if (cdi.size === 0) return { iframeNodes, excludedBackendNodeIds };
|
||||
if (cdi.size === 0) {
|
||||
return { iframeNodes, frameOwnerBackendNodeIds, frameParentIds, excludedBackendNodeIds };
|
||||
}
|
||||
|
||||
const parentBackendIds = parentDoc?.nodes?.backendNodeId ?? [];
|
||||
const parentNodeByBackendId = new Map(parentNodes.map((node) => [node.backendNodeId, node]));
|
||||
@@ -918,19 +956,15 @@ function parseChildFrameDocuments(
|
||||
const iframeBackendId = parentBackendIds[nodeArrayIdx];
|
||||
if (iframeBackendId === undefined) continue;
|
||||
const iframeNode = parentNodeByBackendId.get(iframeBackendId);
|
||||
if (!iframeNode?.localRect) continue;
|
||||
|
||||
const iframeRectInTop = unclipRectInTop(iframeNode.localRect, parentContext);
|
||||
const childClip = intersectRects(iframeRectInTop, parentContext.clipRectInTop);
|
||||
if (!childClip) {
|
||||
iframeNodes.set(iframeBackendId, []);
|
||||
continue;
|
||||
}
|
||||
const projection =
|
||||
parentContext.projection && iframeNode?.localRect
|
||||
? childFrameProjection(parentContext.projection, iframeNode.localRect)
|
||||
: null;
|
||||
|
||||
const childContext: FrameContext = {
|
||||
frameId: snapshotFrameId(childDoc, strings),
|
||||
ownerFrameBackendNodeId: iframeBackendId,
|
||||
originInTop: { x: iframeRectInTop.x, y: iframeRectInTop.y },
|
||||
clipRectInTop: childClip,
|
||||
projection,
|
||||
scrollX: childDoc.scrollOffsetX ?? 0,
|
||||
scrollY: childDoc.scrollOffsetY ?? 0,
|
||||
};
|
||||
@@ -938,6 +972,10 @@ function parseChildFrameDocuments(
|
||||
nextVisited.add(childDocIndex);
|
||||
const parsed = parseDocumentNodes(childDoc, strings, dpr, childContext);
|
||||
iframeNodes.set(iframeBackendId, parsed.nodes);
|
||||
if (childContext.frameId) {
|
||||
frameOwnerBackendNodeIds.set(childContext.frameId, iframeBackendId);
|
||||
if (parentContext.frameId) frameParentIds.set(childContext.frameId, parentContext.frameId);
|
||||
}
|
||||
for (const id of parsed.excludedBackendNodeIds) excludedBackendNodeIds.add(id);
|
||||
|
||||
const nested = parseChildFrameDocuments(
|
||||
@@ -952,10 +990,16 @@ function parseChildFrameDocuments(
|
||||
for (const [nestedFrameId, nestedNodes] of nested.iframeNodes) {
|
||||
iframeNodes.set(nestedFrameId, nestedNodes);
|
||||
}
|
||||
for (const [frameId, ownerBackendNodeId] of nested.frameOwnerBackendNodeIds) {
|
||||
frameOwnerBackendNodeIds.set(frameId, ownerBackendNodeId);
|
||||
}
|
||||
for (const [frameId, parentFrameId] of nested.frameParentIds) {
|
||||
frameParentIds.set(frameId, parentFrameId);
|
||||
}
|
||||
for (const id of nested.excludedBackendNodeIds) excludedBackendNodeIds.add(id);
|
||||
}
|
||||
|
||||
return { iframeNodes, excludedBackendNodeIds };
|
||||
return { iframeNodes, frameOwnerBackendNodeIds, frameParentIds, excludedBackendNodeIds };
|
||||
}
|
||||
|
||||
export async function captureViewModel(
|
||||
@@ -964,7 +1008,13 @@ export async function captureViewModel(
|
||||
options: CaptureViewModelOptions = {},
|
||||
): Promise<CapturedViewModel> {
|
||||
throwIfAborted(options.signal);
|
||||
const metrics = await cdp.send<LayoutMetricsReply>(tabId, "Page.getLayoutMetrics", {});
|
||||
let metrics: LayoutMetricsReply = {};
|
||||
try {
|
||||
metrics = await cdp.send<LayoutMetricsReply>(tabId, "Page.getLayoutMetrics", {});
|
||||
} catch (error) {
|
||||
if (isAbortError(error)) throw error;
|
||||
console.debug("[bsk capture] layout metrics unavailable", error);
|
||||
}
|
||||
throwIfAborted(options.signal);
|
||||
const dpr = devicePixelRatio(metrics);
|
||||
const vpSrc = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
|
||||
@@ -998,9 +1048,13 @@ export async function captureViewModel(
|
||||
}
|
||||
|
||||
const topContext: FrameContext = {
|
||||
frameId: snapshotFrameId(doc0, strings),
|
||||
ownerFrameBackendNodeId: null,
|
||||
originInTop: { x: 0, y: 0 },
|
||||
clipRectInTop: viewportRect(viewport),
|
||||
projection: {
|
||||
sourceClips: [],
|
||||
edges: [],
|
||||
topViewport: viewport,
|
||||
},
|
||||
scrollX,
|
||||
scrollY,
|
||||
};
|
||||
@@ -1022,5 +1076,27 @@ export async function captureViewModel(
|
||||
: [];
|
||||
throwIfAborted(options.signal);
|
||||
|
||||
return { nodes, viewport, iframeNodes, surfaceProbes, excludedBackendNodeIds };
|
||||
const frameNodes = new Map<string, CapturedNode[]>();
|
||||
if (topContext.frameId) frameNodes.set(topContext.frameId, nodes);
|
||||
for (const iframeFrameNodes of iframeNodes.values()) {
|
||||
const frameId = iframeFrameNodes.find((node) => node.frameId)?.frameId;
|
||||
if (frameId) frameNodes.set(frameId, iframeFrameNodes);
|
||||
}
|
||||
for (const [frameId, ownerBackendNodeId] of frameParsed.frameOwnerBackendNodeIds) {
|
||||
if (!frameNodes.has(frameId)) {
|
||||
frameNodes.set(frameId, iframeNodes.get(ownerBackendNodeId) ?? []);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
nodes,
|
||||
viewport,
|
||||
iframeNodes,
|
||||
frameNodes,
|
||||
frameOwnerBackendNodeIds: frameParsed.frameOwnerBackendNodeIds,
|
||||
frameParentIds: frameParsed.frameParentIds,
|
||||
...(topContext.frameId ? { rootFrameId: topContext.frameId } : {}),
|
||||
surfaceProbes,
|
||||
excludedBackendNodeIds,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -0,0 +1,181 @@
|
||||
import type { CdpFrame, CdpFrameGraph } from "@/browser-driver/frame-graph";
|
||||
import { resolveFrameProjection } from "../frame-geometry";
|
||||
import { type GeometryProjection, projectRectToViewport } from "../geometry";
|
||||
import { type CdpRunner, cdpRunnerForTarget, sendToCdpTarget } from "../shared";
|
||||
import { type CapturedNode, type CapturedViewModel, captureViewModel } from "./capture";
|
||||
import {
|
||||
buildFrameDocuments,
|
||||
type FrameAxBatch,
|
||||
type FrameDocument,
|
||||
type FrameOwnedAxNode,
|
||||
} from "./frame-document";
|
||||
|
||||
export type FrameAxNode = FrameOwnedAxNode;
|
||||
export type CapturedFrameDocument<T extends FrameAxNode> = FrameDocument<T>;
|
||||
|
||||
function throwIfAborted(signal?: AbortSignal): void {
|
||||
if (signal?.aborted) throw new DOMException("observation aborted", "AbortError");
|
||||
}
|
||||
|
||||
function isAbortError(err: unknown): boolean {
|
||||
return err instanceof Error && err.name === "AbortError";
|
||||
}
|
||||
|
||||
async function discoverFrameGraph(cdp: CdpRunner, tabId: number): Promise<CdpFrameGraph | null> {
|
||||
if (!cdp.getFrameGraph) return null;
|
||||
try {
|
||||
return await cdp.getFrameGraph(tabId);
|
||||
} catch (err) {
|
||||
console.debug("[bsk observation] frame graph capture failed", err);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function transformFrameNodes(
|
||||
nodes: CapturedNode[],
|
||||
projection: GeometryProjection | null,
|
||||
): CapturedNode[] {
|
||||
return nodes.map((node) => ({
|
||||
...node,
|
||||
rect: projection
|
||||
? (() => {
|
||||
const bounds = projectRectToViewport(node.rect, projection);
|
||||
return bounds ? { x: bounds.x, y: bounds.y, w: bounds.width, h: bounds.height } : null;
|
||||
})()
|
||||
: null,
|
||||
}));
|
||||
}
|
||||
|
||||
async function captureMissingFrameDocuments(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
graph: CdpFrameGraph,
|
||||
captured: CapturedViewModel,
|
||||
signal?: AbortSignal,
|
||||
): Promise<void> {
|
||||
if (!captured.frameNodes) captured.frameNodes = new Map();
|
||||
if (!captured.frameOwnerBackendNodeIds) captured.frameOwnerBackendNodeIds = new Map();
|
||||
if (!captured.frameParentIds) captured.frameParentIds = new Map();
|
||||
|
||||
for (const frame of graph.frames) {
|
||||
if (!frame.target.sessionId || captured.frameNodes.has(frame.frameId)) continue;
|
||||
try {
|
||||
const child = await captureViewModel(cdpRunnerForTarget(cdp, frame.target), tabId, {
|
||||
signal,
|
||||
});
|
||||
let projection: GeometryProjection | null = null;
|
||||
try {
|
||||
projection = await resolveFrameProjection(cdp, graph, frame.frameId);
|
||||
} catch (err) {
|
||||
if (isAbortError(err)) throw err;
|
||||
console.debug("[bsk observation] child frame geometry projection failed", {
|
||||
frameId: frame.frameId,
|
||||
err,
|
||||
});
|
||||
}
|
||||
const rootFrameId = child.rootFrameId ?? frame.frameId;
|
||||
const childFrames = child.frameNodes ?? new Map<string, CapturedNode[]>();
|
||||
if (!childFrames.has(rootFrameId)) childFrames.set(rootFrameId, child.nodes);
|
||||
if (rootFrameId !== frame.frameId && !childFrames.has(frame.frameId)) {
|
||||
childFrames.set(
|
||||
frame.frameId,
|
||||
child.nodes.map((node) => ({ ...node, frameId: frame.frameId })),
|
||||
);
|
||||
}
|
||||
for (const [childFrameId, nodes] of childFrames) {
|
||||
captured.frameNodes.set(childFrameId, transformFrameNodes(nodes, projection));
|
||||
}
|
||||
if (frame.ownerBackendNodeId !== undefined) {
|
||||
captured.iframeNodes.set(
|
||||
frame.ownerBackendNodeId,
|
||||
captured.frameNodes.get(frame.frameId) ?? [],
|
||||
);
|
||||
captured.frameOwnerBackendNodeIds.set(frame.frameId, frame.ownerBackendNodeId);
|
||||
}
|
||||
if (frame.parentFrameId) captured.frameParentIds.set(frame.frameId, frame.parentFrameId);
|
||||
for (const [childFrameId, ownerBackendNodeId] of child.frameOwnerBackendNodeIds ?? []) {
|
||||
captured.frameOwnerBackendNodeIds.set(childFrameId, ownerBackendNodeId);
|
||||
}
|
||||
for (const [childFrameId, parentFrameId] of child.frameParentIds ?? []) {
|
||||
captured.frameParentIds.set(childFrameId, parentFrameId);
|
||||
}
|
||||
} catch (err) {
|
||||
if (isAbortError(err)) throw err;
|
||||
console.debug("[bsk observation] child frame DOM capture failed", {
|
||||
frameId: frame.frameId,
|
||||
err,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function captureAxTrees<T extends FrameAxNode>(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
discoveredFrames: CdpFrame[],
|
||||
captured: CapturedViewModel,
|
||||
signal?: AbortSignal,
|
||||
): Promise<FrameAxBatch<T>[]> {
|
||||
let frames = [...discoveredFrames];
|
||||
const capturedFrameIds = [...(captured.frameNodes?.keys() ?? [])];
|
||||
if (frames.length === 0) {
|
||||
const rootFrameId = captured.rootFrameId ?? capturedFrameIds[0] ?? "root";
|
||||
frames = [{ frameId: rootFrameId, target: { tabId } }];
|
||||
}
|
||||
for (const frameId of capturedFrameIds) {
|
||||
if (!frames.some((frame) => frame.frameId === frameId)) {
|
||||
frames.push({ frameId, target: { tabId } });
|
||||
}
|
||||
}
|
||||
|
||||
let firstFailure: unknown;
|
||||
const trees = await Promise.all(
|
||||
frames.map(async (frame): Promise<FrameAxBatch<T>> => {
|
||||
try {
|
||||
throwIfAborted(signal);
|
||||
await sendToCdpTarget(cdp, frame.target, "Accessibility.enable", {});
|
||||
const result = await sendToCdpTarget<{ nodes?: T[] }>(
|
||||
cdp,
|
||||
frame.target,
|
||||
"Accessibility.getFullAXTree",
|
||||
frame.frameId === "root" ? {} : { frameId: frame.frameId },
|
||||
);
|
||||
throwIfAborted(signal);
|
||||
return {
|
||||
frame,
|
||||
nodes: result.nodes ?? [],
|
||||
};
|
||||
} catch (err) {
|
||||
if (isAbortError(err)) throw err;
|
||||
firstFailure ??= err;
|
||||
console.debug("[bsk observation] frame AX capture failed", {
|
||||
frameId: frame.frameId,
|
||||
err,
|
||||
});
|
||||
return { frame, nodes: [] };
|
||||
}
|
||||
}),
|
||||
);
|
||||
const capturedNodeCount = [...captured.iframeNodes.values()].reduce(
|
||||
(count, nodes) => count + nodes.length,
|
||||
captured.nodes.length,
|
||||
);
|
||||
if (firstFailure && capturedNodeCount === 0 && trees.every((tree) => tree.nodes.length === 0)) {
|
||||
throw firstFailure;
|
||||
}
|
||||
return trees;
|
||||
}
|
||||
|
||||
export async function captureFrameData<T extends FrameAxNode>(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
captured: CapturedViewModel,
|
||||
signal?: AbortSignal,
|
||||
): Promise<CapturedFrameDocument<T>[]> {
|
||||
const graph = await discoverFrameGraph(cdp, tabId);
|
||||
const frames = graph?.frames ?? [];
|
||||
if (graph) await captureMissingFrameDocuments(cdp, tabId, graph, captured, signal);
|
||||
throwIfAborted(signal);
|
||||
const batches = await captureAxTrees<T>(cdp, tabId, frames, captured, signal);
|
||||
return buildFrameDocuments(graph, batches, captured);
|
||||
}
|
||||
@@ -0,0 +1,173 @@
|
||||
import {
|
||||
type CdpFrame,
|
||||
type CdpFrameGraph,
|
||||
type CdpTarget,
|
||||
cdpTargetKey,
|
||||
} from "@/browser-driver/frame-graph";
|
||||
import type { CapturedNode, CapturedViewModel } from "./capture";
|
||||
|
||||
export interface FrameOwnedAxNode {
|
||||
nodeId: string;
|
||||
frameId?: string;
|
||||
parentId?: string;
|
||||
childIds?: string[];
|
||||
backendDOMNodeId?: number;
|
||||
}
|
||||
|
||||
export interface FrameAxBatch<T extends FrameOwnedAxNode> {
|
||||
frame: CdpFrame;
|
||||
nodes: T[];
|
||||
}
|
||||
|
||||
export interface FrameDocument<T extends FrameOwnedAxNode> extends CdpFrame {
|
||||
contextScopeId: string;
|
||||
axNodes: T[];
|
||||
domNodes: CapturedNode[];
|
||||
}
|
||||
|
||||
interface Ownership {
|
||||
frameId: string;
|
||||
strength: number;
|
||||
}
|
||||
|
||||
interface OwnedCandidate<T> {
|
||||
node: T;
|
||||
target: CdpTarget;
|
||||
ownership: Ownership;
|
||||
}
|
||||
|
||||
function targetNodeKey(target: CdpTarget, nodeId: string): string {
|
||||
return `${cdpTargetKey(target)}:${nodeId}`;
|
||||
}
|
||||
|
||||
function targetBackendKey(target: CdpTarget, backendNodeId: number): string {
|
||||
return `${cdpTargetKey(target)}:${backendNodeId}`;
|
||||
}
|
||||
|
||||
function frameList<T extends FrameOwnedAxNode>(
|
||||
graph: CdpFrameGraph | null,
|
||||
batches: FrameAxBatch<T>[],
|
||||
captured: CapturedViewModel,
|
||||
): CdpFrame[] {
|
||||
const frames = new Map<string, CdpFrame>();
|
||||
for (const frame of graph?.frames ?? []) frames.set(frame.frameId, frame);
|
||||
for (const batch of batches) {
|
||||
if (!frames.has(batch.frame.frameId)) frames.set(batch.frame.frameId, batch.frame);
|
||||
}
|
||||
for (const frameId of captured.frameNodes?.keys() ?? []) {
|
||||
if (!frames.has(frameId))
|
||||
frames.set(frameId, { frameId, target: { tabId: batches[0]?.frame.target.tabId ?? 0 } });
|
||||
}
|
||||
return [...frames.values()].map((frame) => {
|
||||
const ownerBackendNodeId =
|
||||
frame.ownerBackendNodeId ?? captured.frameOwnerBackendNodeIds?.get(frame.frameId);
|
||||
const parentFrameId = frame.parentFrameId ?? captured.frameParentIds?.get(frame.frameId);
|
||||
return {
|
||||
...frame,
|
||||
...(ownerBackendNodeId !== undefined ? { ownerBackendNodeId } : {}),
|
||||
...(parentFrameId ? { parentFrameId } : {}),
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
export function buildFrameDocuments<T extends FrameOwnedAxNode>(
|
||||
graph: CdpFrameGraph | null,
|
||||
batches: FrameAxBatch<T>[],
|
||||
captured: CapturedViewModel,
|
||||
): FrameDocument<T>[] {
|
||||
const frames = frameList(graph, batches, captured);
|
||||
const frameById = new Map(frames.map((frame) => [frame.frameId, frame]));
|
||||
const rootFrameId = graph?.rootFrameId ?? captured.rootFrameId ?? frames[0]?.frameId;
|
||||
const domNodesForFrame = (frameId: string): CapturedNode[] =>
|
||||
captured.frameNodes?.get(frameId) ?? (frameId === rootFrameId ? captured.nodes : []);
|
||||
const backendOwner = new Map<string, string>();
|
||||
for (const frame of frames) {
|
||||
for (const node of domNodesForFrame(frame.frameId)) {
|
||||
backendOwner.set(targetBackendKey(frame.target, node.backendNodeId), frame.frameId);
|
||||
}
|
||||
}
|
||||
|
||||
const candidates = new Map<string, OwnedCandidate<T>>();
|
||||
for (const batch of batches) {
|
||||
const nodeById = new Map(batch.nodes.map((node) => [node.nodeId, node]));
|
||||
const ownershipByNodeId = new Map<string, Ownership>();
|
||||
const resolving = new Set<string>();
|
||||
const resolveOwnership = (node: T): Ownership => {
|
||||
const cached = ownershipByNodeId.get(node.nodeId);
|
||||
if (cached) return cached;
|
||||
if (node.frameId && frameById.has(node.frameId)) {
|
||||
const ownership = { frameId: node.frameId, strength: 4 };
|
||||
ownershipByNodeId.set(node.nodeId, ownership);
|
||||
return ownership;
|
||||
}
|
||||
if (typeof node.backendDOMNodeId === "number") {
|
||||
const frameId = backendOwner.get(
|
||||
targetBackendKey(batch.frame.target, node.backendDOMNodeId),
|
||||
);
|
||||
if (frameId) {
|
||||
const ownership = { frameId, strength: 3 };
|
||||
ownershipByNodeId.set(node.nodeId, ownership);
|
||||
return ownership;
|
||||
}
|
||||
}
|
||||
if (node.parentId && !resolving.has(node.nodeId)) {
|
||||
const parent = nodeById.get(node.parentId);
|
||||
if (parent) {
|
||||
resolving.add(node.nodeId);
|
||||
const inherited = resolveOwnership(parent);
|
||||
resolving.delete(node.nodeId);
|
||||
const ownership = {
|
||||
frameId: inherited.frameId,
|
||||
strength: Math.min(2, inherited.strength),
|
||||
};
|
||||
ownershipByNodeId.set(node.nodeId, ownership);
|
||||
return ownership;
|
||||
}
|
||||
}
|
||||
const ownership = { frameId: batch.frame.frameId, strength: 1 };
|
||||
ownershipByNodeId.set(node.nodeId, ownership);
|
||||
return ownership;
|
||||
};
|
||||
|
||||
for (const node of batch.nodes) {
|
||||
const ownership = resolveOwnership(node);
|
||||
const key = targetNodeKey(batch.frame.target, node.nodeId);
|
||||
const existing = candidates.get(key);
|
||||
if (!existing || ownership.strength > existing.ownership.strength) {
|
||||
candidates.set(key, { node, target: batch.frame.target, ownership });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const ownershipByTargetNode = new Map(
|
||||
[...candidates].map(([key, candidate]) => [key, candidate.ownership.frameId]),
|
||||
);
|
||||
const axNodesByFrame = new Map<string, T[]>();
|
||||
for (const candidate of candidates.values()) {
|
||||
const { node, target, ownership } = candidate;
|
||||
const parentFrameId = node.parentId
|
||||
? ownershipByTargetNode.get(targetNodeKey(target, node.parentId))
|
||||
: undefined;
|
||||
const childIds = node.childIds?.filter(
|
||||
(childId) => ownershipByTargetNode.get(targetNodeKey(target, childId)) === ownership.frameId,
|
||||
);
|
||||
const ownedNode = {
|
||||
...node,
|
||||
frameId: ownership.frameId,
|
||||
...(parentFrameId === ownership.frameId
|
||||
? { parentId: node.parentId }
|
||||
: { parentId: undefined }),
|
||||
...(childIds ? { childIds } : {}),
|
||||
};
|
||||
const nodes = axNodesByFrame.get(ownership.frameId) ?? [];
|
||||
nodes.push(ownedNode);
|
||||
axNodesByFrame.set(ownership.frameId, nodes);
|
||||
}
|
||||
|
||||
return frames.map((frame) => ({
|
||||
...frame,
|
||||
contextScopeId: frame.frameId,
|
||||
axNodes: axNodesByFrame.get(frame.frameId) ?? [],
|
||||
domNodes: domNodesForFrame(frame.frameId),
|
||||
}));
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user