mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
Compare commits
104
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6300eda0aa | ||
|
|
87cdcfe5ca | ||
|
|
21476c3884 | ||
|
|
c2a225377b | ||
|
|
e64f4cac9f | ||
|
|
368507b07d | ||
|
|
b750eed5d4 | ||
|
|
3d159c0254 | ||
|
|
524516cb56 | ||
|
|
f98198949a | ||
|
|
d05279f991 | ||
|
|
2719e76438 | ||
|
|
dd9fc9edba | ||
|
|
005a6fd534 | ||
|
|
20554508db | ||
|
|
8352c3c67a | ||
|
|
4eedd98c1d | ||
|
|
6aa931b60a | ||
|
|
3df2738280 | ||
|
|
f86e506d47 | ||
|
|
45a246c41b | ||
|
|
6b78211e79 | ||
|
|
eebf334b40 | ||
|
|
a284ffc5ae | ||
|
|
e720f1ffe5 | ||
|
|
2b7e10ca2b | ||
|
|
1ef93463dd | ||
|
|
c7252529f4 | ||
|
|
0b65b4b295 | ||
|
|
94ec7e73fc | ||
|
|
42403e55f2 | ||
|
|
22019cdf07 | ||
|
|
8d7c0aa66c | ||
|
|
a9489282fa | ||
|
|
d4cddd6224 | ||
|
|
8e4de6cf4b | ||
|
|
fea66ce404 | ||
|
|
8bae85127a | ||
|
|
edb149611b | ||
|
|
14c5884cbc | ||
|
|
e7aad9bc16 | ||
|
|
c8c5c4a7c4 | ||
|
|
3fb9e9f581 | ||
|
|
9854ee2f45 | ||
|
|
7035e46024 | ||
|
|
51085b960b | ||
|
|
51df868008 | ||
|
|
43ebfab7ba | ||
|
|
ce11325de2 | ||
|
|
4d1dcd122f | ||
|
|
a4e668542d | ||
|
|
52aa356153 | ||
|
|
d1001e14b2 | ||
|
|
eb71ddc1da | ||
|
|
b07b8fec65 | ||
|
|
47070fdc14 | ||
|
|
916229dc62 | ||
|
|
dde707b558 | ||
|
|
e2d07574d7 | ||
|
|
461d204ca3 | ||
|
|
bc5d9610a3 | ||
|
|
2955260d11 | ||
|
|
ce89e95211 | ||
|
|
24faa10659 | ||
|
|
a7bc8bacf6 | ||
|
|
3d050e0f26 | ||
|
|
afdc398730 | ||
|
|
6b96fcca32 | ||
|
|
0771292336 | ||
|
|
54869cabd0 | ||
|
|
2de7c5c6ff | ||
|
|
703fc64399 | ||
|
|
4549ed0892 | ||
|
|
37fbaacc94 | ||
|
|
f81a1e7796 | ||
|
|
78c2719718 | ||
|
|
f443cfae74 | ||
|
|
0ce497f017 | ||
|
|
08212497f1 | ||
|
|
f98c0606a1 | ||
|
|
1371c92721 | ||
|
|
c75f97119d | ||
|
|
ac806ae8f4 | ||
|
|
89200975f3 | ||
|
|
238a98476e | ||
|
|
1da2d54ddc | ||
|
|
6e35e62199 | ||
|
|
610782698b | ||
|
|
9504072914 | ||
|
|
566b6c17fd | ||
|
|
2ccc0aafb9 | ||
|
|
eeab97bcdd | ||
|
|
6c610faed9 | ||
|
|
d685c75d25 | ||
|
|
73724617f9 | ||
|
|
4ba6fb45e1 | ||
|
|
9d4ac90592 | ||
|
|
cde6f22ecf | ||
|
|
d24bb6aa93 | ||
|
|
dbc4845c0d | ||
|
|
91225e5fdb | ||
|
|
2dad162a87 | ||
|
|
063b0bbc91 | ||
|
|
0de67c537a |
No files matched your search
@@ -0,0 +1,105 @@
|
|||||||
|
name: Release dsh plugin
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
tags:
|
||||||
|
- dsh-plugin-v*
|
||||||
|
workflow_dispatch:
|
||||||
|
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
|
||||||
|
concurrency:
|
||||||
|
group: release-dsh-plugin-${{ github.ref }}
|
||||||
|
cancel-in-progress: false
|
||||||
|
|
||||||
|
env:
|
||||||
|
PACKAGE_NAME: "@wxg-prc-cpg/browser-skill-dsh-plugin"
|
||||||
|
PACKAGE_DIR: packages/dsh-plugin-browserskill
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
resolve:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
outputs:
|
||||||
|
version: ${{ steps.version.outputs.version }}
|
||||||
|
tag: ${{ steps.version.outputs.tag }}
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v6
|
||||||
|
|
||||||
|
- name: Resolve plugin version
|
||||||
|
id: version
|
||||||
|
shell: bash
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
||||||
|
VERSION="$(node -p "require('./${{ env.PACKAGE_DIR }}/package.json').version")"
|
||||||
|
else
|
||||||
|
VERSION="${GITHUB_REF_NAME#dsh-plugin-v}"
|
||||||
|
fi
|
||||||
|
VERSION="${VERSION#v}"
|
||||||
|
echo "version=${VERSION}" >> "$GITHUB_OUTPUT"
|
||||||
|
echo "tag=dsh-plugin-v${VERSION}" >> "$GITHUB_OUTPUT"
|
||||||
|
|
||||||
|
guard:
|
||||||
|
needs: resolve
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v6
|
||||||
|
|
||||||
|
- name: Verify tag matches package.json name and version
|
||||||
|
shell: bash
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
PKG_NAME="$(node -p "require('./${{ env.PACKAGE_DIR }}/package.json').name")"
|
||||||
|
PKG_VERSION="$(node -p "require('./${{ env.PACKAGE_DIR }}/package.json').version")"
|
||||||
|
RELEASE_VERSION="${{ needs.resolve.outputs.version }}"
|
||||||
|
if [ "$PKG_NAME" != "$PACKAGE_NAME" ]; then
|
||||||
|
echo "package.json name (${PKG_NAME}) does not match expected ${PACKAGE_NAME}"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
if [ "$PKG_VERSION" != "$RELEASE_VERSION" ]; then
|
||||||
|
echo "${PACKAGE_DIR}/package.json version (${PKG_VERSION}) does not match release version (${RELEASE_VERSION})"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
echo "Version guard passed: ${PKG_NAME}@${RELEASE_VERSION}"
|
||||||
|
|
||||||
|
publish:
|
||||||
|
needs: [resolve, guard]
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
# Secret NPM_TOKEN is stored on the "npm-publish" GitHub Environment.
|
||||||
|
environment: npm-publish
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v6
|
||||||
|
|
||||||
|
- uses: pnpm/action-setup@v6
|
||||||
|
|
||||||
|
- uses: actions/setup-node@v6
|
||||||
|
with:
|
||||||
|
node-version: 22
|
||||||
|
cache: pnpm
|
||||||
|
registry-url: https://registry.npmjs.org
|
||||||
|
scope: "@wxg-prc-cpg"
|
||||||
|
|
||||||
|
- name: Install dependencies
|
||||||
|
run: pnpm install --frozen-lockfile
|
||||||
|
|
||||||
|
- name: Typecheck plugin
|
||||||
|
run: pnpm --filter "${{ env.PACKAGE_NAME }}" typecheck
|
||||||
|
|
||||||
|
- name: Test plugin
|
||||||
|
run: pnpm --filter "${{ env.PACKAGE_NAME }}" test
|
||||||
|
|
||||||
|
- name: Publish to npm
|
||||||
|
working-directory: ${{ env.PACKAGE_DIR }}
|
||||||
|
env:
|
||||||
|
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
if [ -z "${NODE_AUTH_TOKEN:-}" ]; then
|
||||||
|
echo "::error::Environment secret NPM_TOKEN is not configured"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
pnpm publish --access public --no-git-checks
|
||||||
|
echo "Published ${PACKAGE_NAME}@${{ needs.resolve.outputs.version }}"
|
||||||
@@ -3,3 +3,6 @@
|
|||||||
**/.output/**
|
**/.output/**
|
||||||
**/target/**
|
**/target/**
|
||||||
**/node_modules/**
|
**/node_modules/**
|
||||||
|
# Generated client CSS (tailwind compile output); built by
|
||||||
|
# packages/dsh-plugin-browserskill/scripts/build-client-css.mjs.
|
||||||
|
**/dsh-plugin-browserskill/src/client/bsk-ui.nomodule.css
|
||||||
+9
-5
@@ -34,13 +34,17 @@ where only `extension connected` fails is expected; go to Step 3.
|
|||||||
## 3. Open the extension install page
|
## 3. Open the extension install page
|
||||||
|
|
||||||
If `extension connected` is `FAIL` (`0 browsers connected`), the user likely
|
If `extension connected` is `FAIL` (`0 browsers connected`), the user likely
|
||||||
has not installed or enabled the browser extension yet. Open the
|
has not installed or enabled the browser extension yet. Open the install page
|
||||||
[Chrome Web Store install page](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi)
|
matching the user's browser —
|
||||||
in the user's browser, then tell the user:
|
[Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi)
|
||||||
|
for Chrome and other Chromium browsers,
|
||||||
|
[Edge Add-ons](https://microsoftedge.microsoft.com/addons/detail/browserskill/emacgiaaaiojkkpkddmmdfhmokgmnikg)
|
||||||
|
for Microsoft Edge — then tell the user:
|
||||||
|
|
||||||
> Install **BrowserSkill** from the
|
> Install **BrowserSkill** from the
|
||||||
> [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi),
|
> [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi)
|
||||||
> open the popup, and wait until it turns green. Reply when done.
|
> (or [Edge Add-ons](https://microsoftedge.microsoft.com/addons/detail/browserskill/emacgiaaaiojkkpkddmmdfhmokgmnikg)
|
||||||
|
> on Edge), open the popup, and wait until it turns green. Reply when done.
|
||||||
|
|
||||||
If opening the page fails, give the user the same link. Then run `bsk doctor`
|
If opening the page fails, give the user the same link. Then run `bsk doctor`
|
||||||
once more. All `ok`/`na` → tell the user it's ready.
|
once more. All `ok`/`na` → tell the user it's ready.
|
||||||
@@ -13,7 +13,7 @@
|
|||||||
</p>
|
</p>
|
||||||
|
|
||||||
**BrowserSkill** connects Cursor, Claude Code, Codex, OpenClaw, CodeBuddy,
|
**BrowserSkill** connects Cursor, Claude Code, Codex, OpenClaw, CodeBuddy,
|
||||||
WorkBuddy, Pi, Hermes Agent, and other shell-capable AI agents to your already logged-in
|
WorkBuddy, Pi, Hermes Agent, DeepSeek Harness, and other AI agents to your already logged-in
|
||||||
browser.
|
browser.
|
||||||
|
|
||||||
Need the agent to touch a tab you already have open? It must borrow that tab
|
Need the agent to touch a tab you already have open? It must borrow that tab
|
||||||
@@ -67,7 +67,8 @@ Set up browser-skill on this machine by following https://raw.githubusercontent.
|
|||||||
|
|
||||||
<br>
|
<br>
|
||||||
|
|
||||||
Install the CLI, then install the extension from the [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi).
|
Install the CLI, then install the extension from the [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi)
|
||||||
|
or [Edge Add-ons](https://microsoftedge.microsoft.com/addons/detail/browserskill/emacgiaaaiojkkpkddmmdfhmokgmnikg).
|
||||||
|
|
||||||
#### 1. Install the `bsk` CLI
|
#### 1. Install the `bsk` CLI
|
||||||
|
|
||||||
@@ -91,7 +92,14 @@ bsk --version
|
|||||||
|
|
||||||
#### 2. Install the browser extension
|
#### 2. Install the browser extension
|
||||||
|
|
||||||
Install BrowserSkill from the [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi).
|
Install BrowserSkill from your browser's store:
|
||||||
|
|
||||||
|
| Browser | Store listing |
|
||||||
|
| --- | --- |
|
||||||
|
| Chrome | [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi) |
|
||||||
|
| Microsoft Edge | [Edge Add-ons](https://microsoftedge.microsoft.com/addons/detail/browserskill/emacgiaaaiojkkpkddmmdfhmokgmnikg) |
|
||||||
|
|
||||||
|
On other Chromium-based browsers, install the Chrome Web Store build.
|
||||||
|
|
||||||
#### 3. Install the skill
|
#### 3. Install the skill
|
||||||
|
|
||||||
@@ -123,7 +131,8 @@ internal variants and install paths.
|
|||||||
|
|
||||||
Other shell-capable agent harnesses are supported too. Copy
|
Other shell-capable agent harnesses are supported too. Copy
|
||||||
[`skill/SKILL.md`](skill/SKILL.md) into your harness's skills directory as
|
[`skill/SKILL.md`](skill/SKILL.md) into your harness's skills directory as
|
||||||
`browser-skill/SKILL.md` to install the skill manually.
|
`browser-skill/SKILL.md` to install the skill manually. DeepSeek Harness uses a
|
||||||
|
dedicated plugin instead — see [DeepSeek Harness plugin](#deepseek-harness-plugin).
|
||||||
|
|
||||||
</details>
|
</details>
|
||||||
|
|
||||||
@@ -133,6 +142,26 @@ Start a new Agent session and write a prompt that needs the browser, for example
|
|||||||
/browser-skill open example.com and summarize what is on the page.
|
/browser-skill open example.com and summarize what is on the page.
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## DeepSeek Harness plugin
|
||||||
|
|
||||||
|
Using [DeepSeek Harness](https://github.com/deepseek-ai/deepseek-harness) (`dsh`)?
|
||||||
|
BrowserSkill ships a first-class dsh plugin on npm as
|
||||||
|
[`@wxg-prc-cpg/browser-skill-dsh-plugin`](https://www.npmjs.com/package/@wxg-prc-cpg/browser-skill-dsh-plugin).
|
||||||
|
It injects native `browser_*` tools (no shelling out to `bsk`) and a live Web UI
|
||||||
|
overlay of each Agent Window.
|
||||||
|
|
||||||
|
Add it to a dsh profile, then start that profile:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
dsh plugin --profile web add @wxg-prc-cpg/browser-skill-dsh-plugin
|
||||||
|
dsh --profile web
|
||||||
|
```
|
||||||
|
|
||||||
|
The plugin carries its own copy of the skill, so `bsk install-skill` is not needed
|
||||||
|
for dsh — but the `bsk` CLI and the browser extension are still prerequisites. See
|
||||||
|
the [plugin README](packages/dsh-plugin-browserskill/README.md) for the tool list,
|
||||||
|
configuration, and the observation overlay.
|
||||||
|
|
||||||
## How It Works
|
## How It Works
|
||||||
|
|
||||||
BrowserSkill is a local bridge between your agent harness and your browser.
|
BrowserSkill is a local bridge between your agent harness and your browser.
|
||||||
@@ -166,7 +195,9 @@ flowchart TB
|
|||||||
|
|
||||||
The agent never talks to the browser directly. It asks the `bsk` CLI to perform a
|
The agent never talks to the browser directly. It asks the `bsk` CLI to perform a
|
||||||
browser task; the local daemon routes that request to the extension; the
|
browser task; the local daemon routes that request to the extension; the
|
||||||
extension runs it in an Agent Window.
|
extension runs it in an Agent Window. DeepSeek Harness takes the same path
|
||||||
|
through the [plugin](#deepseek-harness-plugin): the agent calls injected
|
||||||
|
`browser_*` tools, and the plugin invokes `bsk` on its behalf.
|
||||||
|
|
||||||
## For Developers
|
## For Developers
|
||||||
|
|
||||||
@@ -176,6 +207,7 @@ The repository is a Cargo + pnpm workspace:
|
|||||||
- `crates/bsk-protocol` — shared wire types and JSON schemas
|
- `crates/bsk-protocol` — shared wire types and JSON schemas
|
||||||
- `apps/extension` — browser extension
|
- `apps/extension` — browser extension
|
||||||
- `packages/ui` and `packages/i18n` — shared extension UI support
|
- `packages/ui` and `packages/i18n` — shared extension UI support
|
||||||
|
- `packages/dsh-plugin-browserskill` — DeepSeek Harness plugin (`@wxg-prc-cpg/browser-skill-dsh-plugin`)
|
||||||
|
|
||||||
## License
|
## License
|
||||||
|
|
||||||
|
|||||||
+32
-7
@@ -12,7 +12,7 @@
|
|||||||
<a href="README.md">English</a> · 中文
|
<a href="README.md">English</a> · 中文
|
||||||
</p>
|
</p>
|
||||||
|
|
||||||
**BrowserSkill** 把 Cursor、Claude Code、Codex、OpenClaw、CodeBuddy、WorkBuddy、Pi、Hermes Agent 等支持 Shell 的 AI Agent 连接到你已登录的浏览器。
|
**BrowserSkill** 把 Cursor、Claude Code、Codex、OpenClaw、CodeBuddy、WorkBuddy、Pi、Hermes Agent、DeepSeek Harness 等 AI Agent 连接到你已登录的浏览器。
|
||||||
|
|
||||||
需要 Agent 操作你已打开的标签页?必须显式借用该标签,任务结束后归还,其余浏览器窗口不受影响。
|
需要 Agent 操作你已打开的标签页?必须显式借用该标签,任务结束后归还,其余浏览器窗口不受影响。
|
||||||
|
|
||||||
@@ -54,7 +54,8 @@ BrowserSkill 由两个本地运行组件组成:`bsk` CLI/daemon 和浏览器
|
|||||||
|
|
||||||
<br>
|
<br>
|
||||||
|
|
||||||
先安装 CLI,再从 [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi) 安装浏览器扩展。
|
先安装 CLI,再从 [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi)
|
||||||
|
或 [Edge 加载项商店](https://microsoftedge.microsoft.com/addons/detail/browserskill/emacgiaaaiojkkpkddmmdfhmokgmnikg) 安装浏览器扩展。
|
||||||
|
|
||||||
#### 1. 安装 `bsk` CLI
|
#### 1. 安装 `bsk` CLI
|
||||||
|
|
||||||
@@ -64,8 +65,11 @@ BrowserSkill 由两个本地运行组件组成:`bsk` CLI/daemon 和浏览器
|
|||||||
curl -fsSL https://raw.githubusercontent.com/Tencent/BrowserSkill/main/install.sh | sh
|
curl -fsSL https://raw.githubusercontent.com/Tencent/BrowserSkill/main/install.sh | sh
|
||||||
```
|
```
|
||||||
|
|
||||||
**Windows**:从 [最新 CLI release](https://github.com/Tencent/BrowserSkill/releases/latest)
|
**Windows**(PowerShell,安装到 `~/.local/bin`):
|
||||||
下载 `bsk-v<version>-x86_64-pc-windows-msvc.zip`,解压后将 `bsk.exe` 加入 `PATH`。
|
|
||||||
|
```powershell
|
||||||
|
irm https://raw.githubusercontent.com/Tencent/BrowserSkill/main/install.ps1 | iex
|
||||||
|
```
|
||||||
|
|
||||||
验证二进制:
|
验证二进制:
|
||||||
|
|
||||||
@@ -75,7 +79,14 @@ bsk --version
|
|||||||
|
|
||||||
#### 2. 安装浏览器扩展
|
#### 2. 安装浏览器扩展
|
||||||
|
|
||||||
从 [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi) 安装 BrowserSkill。
|
在对应浏览器的商店安装 BrowserSkill:
|
||||||
|
|
||||||
|
| 浏览器 | 商店页面 |
|
||||||
|
| --- | --- |
|
||||||
|
| Chrome | [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi) |
|
||||||
|
| Microsoft Edge | [Edge 加载项商店](https://microsoftedge.microsoft.com/addons/detail/browserskill/emacgiaaaiojkkpkddmmdfhmokgmnikg) |
|
||||||
|
|
||||||
|
其他基于 Chromium 的浏览器,安装 Chrome Web Store 版本即可。
|
||||||
|
|
||||||
#### 3. 安装 skill
|
#### 3. 安装 skill
|
||||||
|
|
||||||
@@ -102,7 +113,7 @@ bsk install-skill
|
|||||||
|
|
||||||
用 <kbd>Space</kbd> 选择需要安装的 Agent harness,然后按 <kbd>Enter</kbd> 安装 skill。运行 `bsk install-skill --list` 可查看 internal 变体及安装路径。
|
用 <kbd>Space</kbd> 选择需要安装的 Agent harness,然后按 <kbd>Enter</kbd> 安装 skill。运行 `bsk install-skill --list` 可查看 internal 变体及安装路径。
|
||||||
|
|
||||||
其他支持 Shell 的 Agent harness 也可使用 BrowserSkill,但需手动将 [`skill/SKILL.md`](skill/SKILL.md) 复制到对应 skills 目录下的 `browser-skill/SKILL.md`。
|
其他支持 Shell 的 Agent harness 也可使用 BrowserSkill,但需手动将 [`skill/SKILL.md`](skill/SKILL.md) 复制到对应 skills 目录下的 `browser-skill/SKILL.md`。DeepSeek Harness 走独立插件,见 [DeepSeek Harness 插件](#deepseek-harness-插件)。
|
||||||
|
|
||||||
</details>
|
</details>
|
||||||
|
|
||||||
@@ -112,6 +123,19 @@ bsk install-skill
|
|||||||
/browser-skill open example.com and summarize what is on the page.
|
/browser-skill open example.com and summarize what is on the page.
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## DeepSeek Harness 插件
|
||||||
|
|
||||||
|
在用 [DeepSeek Harness](https://github.com/deepseek-ai/deepseek-harness)(`dsh`)?BrowserSkill 提供了官方 dsh 插件,已发布到 npm:[`@wxg-prc-cpg/browser-skill-dsh-plugin`](https://www.npmjs.com/package/@wxg-prc-cpg/browser-skill-dsh-plugin)。它会注入原生 `browser_*` 工具(无需再通过 Shell 调用 `bsk`),并在 Web UI 中实时观察每个 Agent Window。
|
||||||
|
|
||||||
|
把它装进某个 dsh profile,然后启动该 profile:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
dsh plugin --profile web add @wxg-prc-cpg/browser-skill-dsh-plugin
|
||||||
|
dsh --profile web
|
||||||
|
```
|
||||||
|
|
||||||
|
插件自带 skill,所以在 dsh 下无需执行 `bsk install-skill`;但 `bsk` CLI 和浏览器扩展仍是前置条件。工具清单、配置项与观察浮层见[插件 README](packages/dsh-plugin-browserskill/README.md)。
|
||||||
|
|
||||||
## 工作原理
|
## 工作原理
|
||||||
|
|
||||||
BrowserSkill 是 Agent 运行时与浏览器之间的本地桥接层。
|
BrowserSkill 是 Agent 运行时与浏览器之间的本地桥接层。
|
||||||
@@ -143,7 +167,7 @@ flowchart TB
|
|||||||
style UserWindows fill:#f8fafc,stroke:#cbd5e1,color:#334155
|
style UserWindows fill:#f8fafc,stroke:#cbd5e1,color:#334155
|
||||||
```
|
```
|
||||||
|
|
||||||
Agent 不直接与浏览器通信。它通过 `bsk` CLI 下发浏览器任务;本地 daemon 把请求路由到扩展;扩展在 Agent Window 中执行。
|
Agent 不直接与浏览器通信。它通过 `bsk` CLI 下发浏览器任务;本地 daemon 把请求路由到扩展;扩展在 Agent Window 中执行。DeepSeek Harness 走同一条链路,只是经由 [插件](#deepseek-harness-插件):Agent 调用注入的 `browser_*` 工具,由插件代为执行 `bsk`。
|
||||||
|
|
||||||
## 面向开发者
|
## 面向开发者
|
||||||
|
|
||||||
@@ -153,6 +177,7 @@ Agent 不直接与浏览器通信。它通过 `bsk` CLI 下发浏览器任务;
|
|||||||
- `crates/bsk-protocol` — 共享协议类型与 JSON Schema
|
- `crates/bsk-protocol` — 共享协议类型与 JSON Schema
|
||||||
- `apps/extension` — 浏览器扩展
|
- `apps/extension` — 浏览器扩展
|
||||||
- `packages/ui` 和 `packages/i18n` — 扩展 UI 共享支持
|
- `packages/ui` 和 `packages/i18n` — 扩展 UI 共享支持
|
||||||
|
- `packages/dsh-plugin-browserskill` — DeepSeek Harness 插件(`@wxg-prc-cpg/browser-skill-dsh-plugin`)
|
||||||
|
|
||||||
## 许可证
|
## 许可证
|
||||||
|
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@browser-skill/extension",
|
"name": "@browser-skill/extension",
|
||||||
"version": "0.1.5",
|
"version": "0.1.6",
|
||||||
"private": true,
|
"private": true,
|
||||||
"type": "module",
|
"type": "module",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
import { describe, expect, it, vi } from "vitest";
|
import { describe, expect, it, vi } from "vitest";
|
||||||
import { type CdpDebuggerApi, ChromiumCdp } from "../chromium-cdp";
|
import { type CdpDebuggee, type CdpDebuggerApi, ChromiumCdp } from "../chromium-cdp";
|
||||||
|
|
||||||
function fakeChromeEvent<TArgs extends unknown[]>() {
|
function fakeChromeEvent<TArgs extends unknown[]>() {
|
||||||
const listeners = new Set<(...args: TArgs) => void>();
|
const listeners = new Set<(...args: TArgs) => void>();
|
||||||
@@ -14,7 +14,7 @@ function fakeChromeEvent<TArgs extends unknown[]>() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function fakeApi() {
|
function fakeApi() {
|
||||||
const onEvent = fakeChromeEvent<[chrome.debugger.Debuggee, string, unknown]>();
|
const onEvent = fakeChromeEvent<[CdpDebuggee, string, unknown]>();
|
||||||
const onDetach = fakeChromeEvent<[chrome.debugger.Debuggee, string]>();
|
const onDetach = fakeChromeEvent<[chrome.debugger.Debuggee, string]>();
|
||||||
const api: CdpDebuggerApi = {
|
const api: CdpDebuggerApi = {
|
||||||
attach: vi.fn(async () => {}),
|
attach: vi.fn(async () => {}),
|
||||||
@@ -29,6 +29,135 @@ function fakeApi() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
describe("ChromiumCdp", () => {
|
describe("ChromiumCdp", () => {
|
||||||
|
it("discovers multiple iframe targets and recursively routes nested OOPIF commands", async () => {
|
||||||
|
const { api, onEvent } = fakeApi();
|
||||||
|
(api.sendCommand as ReturnType<typeof vi.fn>).mockImplementation(
|
||||||
|
async (
|
||||||
|
target: chrome.debugger.Debuggee & { sessionId?: string },
|
||||||
|
method: string,
|
||||||
|
params?: { frameId?: string },
|
||||||
|
) => {
|
||||||
|
if (method === "DOM.getFrameOwner") {
|
||||||
|
return { backendNodeId: params?.frameId === "nested" ? 300 : 200 };
|
||||||
|
}
|
||||||
|
if (method !== "Page.getFrameTree") return {};
|
||||||
|
if (target.sessionId === "right-session") {
|
||||||
|
return {
|
||||||
|
frameTree: {
|
||||||
|
frame: { id: "right", url: "https://right.test" },
|
||||||
|
childFrames: [
|
||||||
|
{ frame: { id: "nested", parentId: "right", url: "https://nested.test" } },
|
||||||
|
],
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if (target.sessionId === "nested-session") {
|
||||||
|
return { frameTree: { frame: { id: "nested", url: "https://nested.test" } } };
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
frameTree: {
|
||||||
|
frame: { id: "main", url: "https://app.test" },
|
||||||
|
childFrames: [
|
||||||
|
{ frame: { id: "left", parentId: "main", url: "https://left.test" } },
|
||||||
|
{
|
||||||
|
frame: { id: "right", parentId: "main", url: "https://right.test" },
|
||||||
|
childFrames: [
|
||||||
|
{ frame: { id: "nested", parentId: "right", url: "https://nested.test" } },
|
||||||
|
],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
};
|
||||||
|
},
|
||||||
|
);
|
||||||
|
const cdp = new ChromiumCdp(api);
|
||||||
|
await cdp.ensureAttached(4);
|
||||||
|
onEvent.fire({ tabId: 4 }, "Target.attachedToTarget", { sessionId: "right-session" });
|
||||||
|
onEvent.fire({ tabId: 4 } as chrome.debugger.Debuggee, "Target.attachedToTarget", {
|
||||||
|
sessionId: "nested-session",
|
||||||
|
});
|
||||||
|
|
||||||
|
const graph = await cdp.getFrameGraph(4);
|
||||||
|
|
||||||
|
expect(graph.frames).toHaveLength(4);
|
||||||
|
expect(graph.frames.find((frame) => frame.frameId === "left")?.target).toEqual({ tabId: 4 });
|
||||||
|
expect(graph.frames.find((frame) => frame.frameId === "right")?.target.sessionId).toBe(
|
||||||
|
"right-session",
|
||||||
|
);
|
||||||
|
expect(graph.frames.find((frame) => frame.frameId === "nested")?.target.sessionId).toBe(
|
||||||
|
"nested-session",
|
||||||
|
);
|
||||||
|
expect(graph.frames.find((frame) => frame.frameId === "right")?.ownerBackendNodeId).toBe(200);
|
||||||
|
expect(graph.frames.find((frame) => frame.frameId === "nested")?.ownerBackendNodeId).toBe(300);
|
||||||
|
expect(api.sendCommand).toHaveBeenCalledWith(
|
||||||
|
{ tabId: 4, sessionId: "right-session" },
|
||||||
|
"DOM.getFrameOwner",
|
||||||
|
{ frameId: "nested" },
|
||||||
|
);
|
||||||
|
expect(api.sendCommand).toHaveBeenCalledWith(
|
||||||
|
{ tabId: 4, sessionId: "nested-session" },
|
||||||
|
"Target.setAutoAttach",
|
||||||
|
expect.objectContaining({ flatten: true }),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("waits for recursively attached iframe targets without a depth limit", async () => {
|
||||||
|
const { api, onEvent } = fakeApi();
|
||||||
|
const depth = 6;
|
||||||
|
const scheduled = new Set<string>();
|
||||||
|
const nestedTree = (index: number): Record<string, unknown> => ({
|
||||||
|
frame: {
|
||||||
|
id: index === 0 ? "main" : `frame-${index}`,
|
||||||
|
...(index > 0 ? { parentId: index === 1 ? "main" : `frame-${index - 1}` } : {}),
|
||||||
|
},
|
||||||
|
...(index < depth ? { childFrames: [nestedTree(index + 1)] } : {}),
|
||||||
|
});
|
||||||
|
(api.sendCommand as ReturnType<typeof vi.fn>).mockImplementation(
|
||||||
|
async (target: CdpDebuggee, method: string, params?: { frameId?: string }) => {
|
||||||
|
if (method === "Target.setAutoAttach") {
|
||||||
|
const parentIndex = target.sessionId
|
||||||
|
? Number(target.sessionId.replace("session-", ""))
|
||||||
|
: 0;
|
||||||
|
const nextIndex = parentIndex + 1;
|
||||||
|
const key = `${target.sessionId ?? "root"}:${nextIndex}`;
|
||||||
|
if (nextIndex <= depth && !scheduled.has(key)) {
|
||||||
|
scheduled.add(key);
|
||||||
|
setTimeout(() => {
|
||||||
|
onEvent.fire(
|
||||||
|
{ tabId: 4, ...(target.sessionId ? { sessionId: target.sessionId } : {}) },
|
||||||
|
"Target.attachedToTarget",
|
||||||
|
{
|
||||||
|
sessionId: `session-${nextIndex}`,
|
||||||
|
targetInfo: { type: "iframe" },
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}, 5);
|
||||||
|
}
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
if (method === "Page.getFrameTree") {
|
||||||
|
if (!target.sessionId) return { frameTree: nestedTree(0) };
|
||||||
|
const index = Number(target.sessionId.replace("session-", ""));
|
||||||
|
return { frameTree: nestedTree(index) };
|
||||||
|
}
|
||||||
|
if (method === "DOM.getFrameOwner") {
|
||||||
|
return { backendNodeId: Number(params?.frameId?.replace("frame-", "")) + 100 };
|
||||||
|
}
|
||||||
|
return {};
|
||||||
|
},
|
||||||
|
);
|
||||||
|
const cdp = new ChromiumCdp(api);
|
||||||
|
|
||||||
|
const graph = await cdp.getFrameGraph(4);
|
||||||
|
|
||||||
|
expect(graph.frames).toHaveLength(depth + 1);
|
||||||
|
for (let index = 1; index <= depth; index += 1) {
|
||||||
|
expect(
|
||||||
|
graph.frames.find((frame) => frame.frameId === `frame-${index}`)?.target.sessionId,
|
||||||
|
).toBe(`session-${index}`);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
it("coalesces concurrent attach calls for the same tab", async () => {
|
it("coalesces concurrent attach calls for the same tab", async () => {
|
||||||
const { api } = fakeApi();
|
const { api } = fakeApi();
|
||||||
let releaseAttach!: () => void;
|
let releaseAttach!: () => void;
|
||||||
@@ -67,6 +196,30 @@ describe("ChromiumCdp", () => {
|
|||||||
expect(cdp.isAttached(5)).toBe(false);
|
expect(cdp.isAttached(5)).toBe(false);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("rolls back the raw attach when a domain enable fails so the tab is not left stuck", async () => {
|
||||||
|
const { api } = fakeApi();
|
||||||
|
(api.sendCommand as ReturnType<typeof vi.fn>).mockImplementation(
|
||||||
|
async (_target, method: string) => {
|
||||||
|
if (method === "Page.enable") throw new Error("Page.enable rejected");
|
||||||
|
return {};
|
||||||
|
},
|
||||||
|
);
|
||||||
|
const cdp = new ChromiumCdp(api);
|
||||||
|
|
||||||
|
// Raw attach succeeds, but Page.enable fails. Without the rollback
|
||||||
|
// the debugger would stay attached while `attachedTabs` omits the id,
|
||||||
|
// leaving the tab stuck on "Another debugger is already attached" for
|
||||||
|
// every later call until the extension is reloaded.
|
||||||
|
await expect(cdp.ensureAttached(42)).rejects.toThrow(/Page\.enable rejected/);
|
||||||
|
expect(cdp.isAttached(42)).toBe(false);
|
||||||
|
expect(api.detach).toHaveBeenCalledWith({ tabId: 42 });
|
||||||
|
|
||||||
|
// The rollback detached, so a fresh attach can succeed afterwards.
|
||||||
|
(api.sendCommand as ReturnType<typeof vi.fn>).mockResolvedValue({});
|
||||||
|
await expect(cdp.ensureAttached(42)).resolves.toBeUndefined();
|
||||||
|
expect(cdp.isAttached(42)).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
it("send() rejects on chrome.runtime.lastError-style failures", async () => {
|
it("send() rejects on chrome.runtime.lastError-style failures", async () => {
|
||||||
const { api } = fakeApi();
|
const { api } = fakeApi();
|
||||||
(api.sendCommand as ReturnType<typeof vi.fn>).mockImplementation(
|
(api.sendCommand as ReturnType<typeof vi.fn>).mockImplementation(
|
||||||
@@ -132,11 +285,33 @@ describe("ChromiumCdp", () => {
|
|||||||
expect(cdp.isAttached(12)).toBe(true);
|
expect(cdp.isAttached(12)).toBe(true);
|
||||||
expect(api.sendCommand).toHaveBeenCalledWith({ tabId: 12 }, "Runtime.enable", {});
|
expect(api.sendCommand).toHaveBeenCalledWith({ tabId: 12 }, "Runtime.enable", {});
|
||||||
expect(api.sendCommand).toHaveBeenCalledWith({ tabId: 12 }, "Log.enable", {});
|
expect(api.sendCommand).toHaveBeenCalledWith({ tabId: 12 }, "Log.enable", {});
|
||||||
await cdp.ensureConsoleCapture(12);
|
});
|
||||||
const enableCalls = (api.sendCommand as ReturnType<typeof vi.fn>).mock.calls.filter(
|
|
||||||
|
it("retries console capture after a domain enable fails during attach", async () => {
|
||||||
|
const { api } = fakeApi();
|
||||||
|
(api.sendCommand as ReturnType<typeof vi.fn>).mockImplementation(
|
||||||
|
async (_target, method: string) => {
|
||||||
|
if (method === "Log.enable") throw new Error("restricted");
|
||||||
|
return {};
|
||||||
|
},
|
||||||
|
);
|
||||||
|
const cdp = new ChromiumCdp(api);
|
||||||
|
// Attach is best-effort: Log.enable fails but attach still succeeds,
|
||||||
|
// leaving the tab unmarked so capture can be retried.
|
||||||
|
await cdp.ensureAttached(12);
|
||||||
|
const afterAttach = (api.sendCommand as ReturnType<typeof vi.fn>).mock.calls.filter(
|
||||||
([, method]) => method === "Runtime.enable" || method === "Log.enable",
|
([, method]) => method === "Runtime.enable" || method === "Log.enable",
|
||||||
);
|
);
|
||||||
expect(enableCalls).toHaveLength(2);
|
expect(afterAttach).toHaveLength(2);
|
||||||
|
|
||||||
|
// Before the fix the "attempted" flag was set before success, so this
|
||||||
|
// was a no-op and the tab silently returned no console output forever.
|
||||||
|
// Now it retries both domains.
|
||||||
|
await cdp.ensureConsoleCapture(12);
|
||||||
|
const afterRetry = (api.sendCommand as ReturnType<typeof vi.fn>).mock.calls.filter(
|
||||||
|
([, method]) => method === "Runtime.enable" || method === "Log.enable",
|
||||||
|
);
|
||||||
|
expect(afterRetry).toHaveLength(4);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("retries and surfaces Network.enable failures for explicit capture", async () => {
|
it("retries and surfaces Network.enable failures for explicit capture", async () => {
|
||||||
|
|||||||
@@ -0,0 +1,136 @@
|
|||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import { buildFrameGraph, type CdpFrameTreeNode, type CdpFrameTreeSource } from "../frame-graph";
|
||||||
|
|
||||||
|
describe("buildFrameGraph", () => {
|
||||||
|
it("keeps sibling frames distinct and routes nested OOPIFs to their child sessions", () => {
|
||||||
|
const graph = buildFrameGraph([
|
||||||
|
{
|
||||||
|
target: { tabId: 4 },
|
||||||
|
tree: {
|
||||||
|
frame: { id: "main", url: "https://app.test" },
|
||||||
|
childFrames: [
|
||||||
|
{ frame: { id: "left", parentId: "main", url: "https://left.test" } },
|
||||||
|
{
|
||||||
|
frame: { id: "right", parentId: "main", url: "https://right.test" },
|
||||||
|
childFrames: [
|
||||||
|
{ frame: { id: "nested", parentId: "right", url: "https://nested.test" } },
|
||||||
|
],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
target: { tabId: 4, sessionId: "right-session" },
|
||||||
|
tree: {
|
||||||
|
frame: { id: "right", url: "https://right.test" },
|
||||||
|
childFrames: [{ frame: { id: "nested", parentId: "right", url: "https://nested.test" } }],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
target: { tabId: 4, sessionId: "nested-session" },
|
||||||
|
tree: { frame: { id: "nested", url: "https://nested.test" } },
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
|
||||||
|
expect(graph?.frames).toHaveLength(4);
|
||||||
|
expect(graph?.frames.find((frame) => frame.frameId === "left")?.target).toEqual({ tabId: 4 });
|
||||||
|
expect(graph?.frames.find((frame) => frame.frameId === "right")).toMatchObject({
|
||||||
|
parentFrameId: "main",
|
||||||
|
target: { tabId: 4, sessionId: "right-session" },
|
||||||
|
});
|
||||||
|
expect(graph?.frames.find((frame) => frame.frameId === "nested")).toMatchObject({
|
||||||
|
parentFrameId: "right",
|
||||||
|
target: { tabId: 4, sessionId: "nested-session" },
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("propagates each target boundary to its same-process descendants", () => {
|
||||||
|
const sources: CdpFrameTreeSource[] = [
|
||||||
|
{
|
||||||
|
target: { tabId: 4 },
|
||||||
|
tree: {
|
||||||
|
frame: { id: "main" },
|
||||||
|
childFrames: [
|
||||||
|
{
|
||||||
|
frame: { id: "oopif-a", parentId: "main" },
|
||||||
|
childFrames: [
|
||||||
|
{
|
||||||
|
frame: { id: "same-process-b", parentId: "oopif-a" },
|
||||||
|
childFrames: [
|
||||||
|
{
|
||||||
|
frame: { id: "oopif-c", parentId: "same-process-b" },
|
||||||
|
childFrames: [{ frame: { id: "same-process-d", parentId: "oopif-c" } }],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
target: { tabId: 4, sessionId: "session-a" },
|
||||||
|
tree: {
|
||||||
|
frame: { id: "oopif-a" },
|
||||||
|
childFrames: [
|
||||||
|
{
|
||||||
|
frame: { id: "same-process-b", parentId: "oopif-a" },
|
||||||
|
childFrames: [{ frame: { id: "oopif-c", parentId: "same-process-b" } }],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
target: { tabId: 4, sessionId: "session-c" },
|
||||||
|
tree: {
|
||||||
|
frame: { id: "oopif-c" },
|
||||||
|
childFrames: [{ frame: { id: "same-process-d", parentId: "oopif-c" } }],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
const graph = buildFrameGraph(sources);
|
||||||
|
|
||||||
|
expect(graph?.frames.map((frame) => [frame.frameId, frame.target.sessionId])).toEqual([
|
||||||
|
["main", undefined],
|
||||||
|
["oopif-a", "session-a"],
|
||||||
|
["same-process-b", "session-a"],
|
||||||
|
["oopif-c", "session-c"],
|
||||||
|
["same-process-d", "session-c"],
|
||||||
|
]);
|
||||||
|
expect(graph?.frames.find((frame) => frame.frameId === "oopif-a")?.parentFrameId).toBe("main");
|
||||||
|
expect(graph?.frames.find((frame) => frame.frameId === "oopif-c")?.parentFrameId).toBe(
|
||||||
|
"same-process-b",
|
||||||
|
);
|
||||||
|
|
||||||
|
const reordered = buildFrameGraph([sources[2], sources[1], sources[0]]);
|
||||||
|
expect(reordered).toEqual(graph);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("walks deeply nested frame trees without consuming the JavaScript call stack", () => {
|
||||||
|
const depth = 5_000;
|
||||||
|
const root: CdpFrameTreeNode = { frame: { id: "frame-0" } };
|
||||||
|
let parent = root;
|
||||||
|
for (let index = 1; index <= depth; index += 1) {
|
||||||
|
const child: CdpFrameTreeNode = {
|
||||||
|
frame: { id: `frame-${index}`, parentId: `frame-${index - 1}` },
|
||||||
|
};
|
||||||
|
parent.childFrames = [child];
|
||||||
|
parent = child;
|
||||||
|
}
|
||||||
|
|
||||||
|
const graph = buildFrameGraph([{ target: { tabId: 4 }, tree: root }]);
|
||||||
|
|
||||||
|
expect(graph?.frames).toHaveLength(depth + 1);
|
||||||
|
expect(graph?.frames.at(-1)?.frameId).toBe(`frame-${depth}`);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not loop when an in-memory frame tree contains an object cycle", () => {
|
||||||
|
const root: CdpFrameTreeNode = { frame: { id: "main" } };
|
||||||
|
root.childFrames = [root];
|
||||||
|
|
||||||
|
expect(buildFrameGraph([{ target: { tabId: 4 }, tree: root }])?.frames).toEqual([
|
||||||
|
{ frameId: "main", target: { tabId: 4 } },
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -31,6 +31,15 @@ import type {
|
|||||||
NetworkEntryKind,
|
NetworkEntryKind,
|
||||||
NetworkResult,
|
NetworkResult,
|
||||||
} from "@/transport/types";
|
} from "@/transport/types";
|
||||||
|
import {
|
||||||
|
buildFrameGraph,
|
||||||
|
type CdpFrameGraph,
|
||||||
|
type CdpFrameTreeNode,
|
||||||
|
type CdpFrameTreeSource,
|
||||||
|
type CdpTarget,
|
||||||
|
} from "./frame-graph";
|
||||||
|
|
||||||
|
export type CdpDebuggee = chrome.debugger.Debuggee & { sessionId?: string };
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Minimal slice of `chrome.debugger` the rest of the extension
|
* Minimal slice of `chrome.debugger` the rest of the extension
|
||||||
@@ -38,21 +47,15 @@ import type {
|
|||||||
* fake without monkey-patching the real `chrome` global.
|
* fake without monkey-patching the real `chrome` global.
|
||||||
*/
|
*/
|
||||||
export interface CdpDebuggerApi {
|
export interface CdpDebuggerApi {
|
||||||
attach(target: chrome.debugger.Debuggee, requiredVersion: string): Promise<void>;
|
attach(target: CdpDebuggee, requiredVersion: string): Promise<void>;
|
||||||
detach(target: chrome.debugger.Debuggee): Promise<void>;
|
detach(target: CdpDebuggee): Promise<void>;
|
||||||
sendCommand(
|
sendCommand(target: CdpDebuggee, method: string, commandParams?: object): Promise<unknown>;
|
||||||
target: chrome.debugger.Debuggee,
|
|
||||||
method: string,
|
|
||||||
commandParams?: object,
|
|
||||||
): Promise<unknown>;
|
|
||||||
/**
|
/**
|
||||||
* Fires for every CDP event (`Page.lifecycleEvent`, `DOM.documentUpdated`,
|
* Fires for every CDP event (`Page.lifecycleEvent`, `DOM.documentUpdated`,
|
||||||
* …). The first callback argument is the source debuggee; the second
|
* …). The first callback argument is the source debuggee; the second
|
||||||
* is the CDP method name; the third is the payload.
|
* is the CDP method name; the third is the payload.
|
||||||
*/
|
*/
|
||||||
onEvent: chrome.events.Event<
|
onEvent: chrome.events.Event<(source: CdpDebuggee, method: string, params: unknown) => void>;
|
||||||
(source: chrome.debugger.Debuggee, method: string, params: unknown) => void
|
|
||||||
>;
|
|
||||||
/**
|
/**
|
||||||
* Fires when Chrome unilaterally detaches us — most commonly because
|
* Fires when Chrome unilaterally detaches us — most commonly because
|
||||||
* the tab navigated to a chrome:// URL or the user clicked
|
* the tab navigated to a chrome:// URL or the user clicked
|
||||||
@@ -110,6 +113,8 @@ const MAX_CONSOLE_STACK_FRAMES = 20;
|
|||||||
const MAX_NETWORK_BUFFER = 200;
|
const MAX_NETWORK_BUFFER = 200;
|
||||||
const MAX_NETWORK_FIELD_LENGTH = 4096;
|
const MAX_NETWORK_FIELD_LENGTH = 4096;
|
||||||
const MAX_NETWORK_REQUEST_META = 1024;
|
const MAX_NETWORK_REQUEST_META = 1024;
|
||||||
|
const FRAME_DISCOVERY_TIMEOUT_MS = 1000;
|
||||||
|
const FRAME_DISCOVERY_QUIET_MS = 20;
|
||||||
|
|
||||||
interface ParsedDialogOpening {
|
interface ParsedDialogOpening {
|
||||||
type: JavaScriptDialogType;
|
type: JavaScriptDialogType;
|
||||||
@@ -131,6 +136,27 @@ interface NetworkRequestMeta {
|
|||||||
truncated: boolean;
|
truncated: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
interface FrameDiscoveryState {
|
||||||
|
sessions: Set<string>;
|
||||||
|
pending: Set<Promise<void>>;
|
||||||
|
generation: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function settleBeforeDeadline(promises: Promise<void>[], deadline: number): Promise<boolean> {
|
||||||
|
if (promises.length === 0) return true;
|
||||||
|
const remaining = deadline - Date.now();
|
||||||
|
if (remaining <= 0) return false;
|
||||||
|
let timer: ReturnType<typeof setTimeout> | undefined;
|
||||||
|
const timedOut = await Promise.race([
|
||||||
|
Promise.allSettled(promises).then(() => false),
|
||||||
|
new Promise<boolean>((resolve) => {
|
||||||
|
timer = setTimeout(() => resolve(true), remaining);
|
||||||
|
}),
|
||||||
|
]);
|
||||||
|
if (timer) clearTimeout(timer);
|
||||||
|
return !timedOut;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Wrapper around `chrome.debugger` that owns the "attach once per
|
* Wrapper around `chrome.debugger` that owns the "attach once per
|
||||||
* tabId" cache and exposes typed `send<T>()`.
|
* tabId" cache and exposes typed `send<T>()`.
|
||||||
@@ -144,15 +170,17 @@ export class ChromiumCdp {
|
|||||||
private readonly dialogSequences = new Map<number, number>();
|
private readonly dialogSequences = new Map<number, number>();
|
||||||
private readonly consoleBuffers = new Map<number, ConsoleEntry[]>();
|
private readonly consoleBuffers = new Map<number, ConsoleEntry[]>();
|
||||||
private readonly consoleSequences = new Map<number, number>();
|
private readonly consoleSequences = new Map<number, number>();
|
||||||
private readonly consoleDomainsAttemptedTabs = new Set<number>();
|
private readonly consoleDomainsEnabledTabs = new Set<number>();
|
||||||
private readonly networkBuffers = new Map<number, NetworkEntry[]>();
|
private readonly networkBuffers = new Map<number, NetworkEntry[]>();
|
||||||
private readonly networkSequences = new Map<number, number>();
|
private readonly networkSequences = new Map<number, number>();
|
||||||
private readonly networkDomainsEnabledTabs = new Set<number>();
|
private readonly networkDomainsEnabledTabs = new Set<number>();
|
||||||
private readonly networkRequestMeta = new Map<number, Map<string, NetworkRequestMeta>>();
|
private readonly networkRequestMeta = new Map<number, Map<string, NetworkRequestMeta>>();
|
||||||
|
private readonly frameDiscovery = new Map<number, FrameDiscoveryState>();
|
||||||
private detachSubscription: { dispose(): void } | null = null;
|
private detachSubscription: { dispose(): void } | null = null;
|
||||||
private dialogSubscription: { dispose(): void } | null = null;
|
private dialogSubscription: { dispose(): void } | null = null;
|
||||||
private consoleSubscription: { dispose(): void } | null = null;
|
private consoleSubscription: { dispose(): void } | null = null;
|
||||||
private networkSubscription: { dispose(): void } | null = null;
|
private networkSubscription: { dispose(): void } | null = null;
|
||||||
|
private frameTargetSubscription: { dispose(): void } | null = null;
|
||||||
|
|
||||||
constructor(api: CdpDebuggerApi = chromeDebuggerApi) {
|
constructor(api: CdpDebuggerApi = chromeDebuggerApi) {
|
||||||
this.api = api;
|
this.api = api;
|
||||||
@@ -160,6 +188,7 @@ export class ChromiumCdp {
|
|||||||
this.bindDialogHandler();
|
this.bindDialogHandler();
|
||||||
this.bindConsoleHandler();
|
this.bindConsoleHandler();
|
||||||
this.bindNetworkHandler();
|
this.bindNetworkHandler();
|
||||||
|
this.bindFrameTargetHandler();
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Attach to `tabId` if we haven't already in this driver. */
|
/** Attach to `tabId` if we haven't already in this driver. */
|
||||||
@@ -172,10 +201,29 @@ export class ChromiumCdp {
|
|||||||
}
|
}
|
||||||
const attach = (async () => {
|
const attach = (async () => {
|
||||||
await this.api.attach({ tabId }, CDP_PROTOCOL_VERSION);
|
await this.api.attach({ tabId }, CDP_PROTOCOL_VERSION);
|
||||||
await this.enablePageDomain(tabId);
|
try {
|
||||||
await this.enableConsoleDomains(tabId);
|
await this.enablePageDomain(tabId);
|
||||||
await this.enableNetworkDomainBestEffort(tabId);
|
await this.enableConsoleDomains(tabId);
|
||||||
this.attachedTabs.add(tabId);
|
await this.enableNetworkDomainBestEffort(tabId);
|
||||||
|
this.attachedTabs.add(tabId);
|
||||||
|
await this.enableFrameDiscovery({ tabId }).catch((err) => {
|
||||||
|
console.debug("[bsk cdp] frame discovery unavailable", { tabId, err });
|
||||||
|
});
|
||||||
|
} catch (err) {
|
||||||
|
// A CDP domain enable failed after the raw attach succeeded
|
||||||
|
// (e.g. `Page.enable` rejects because the tab just navigated to
|
||||||
|
// a chrome:// URL or the Web Store). `attachedTabs` does not yet
|
||||||
|
// hold `tabId`, so without rolling back the debugger would stay
|
||||||
|
// attached and the next `ensureAttached` would re-attach and hit
|
||||||
|
// "Another debugger is already attached" forever — the tab is
|
||||||
|
// unusable until the extension is reloaded. Detach directly
|
||||||
|
// (`this.detach()` is a no-op here because `attachedTabs` lacks
|
||||||
|
// the id) so the next attempt starts from a clean slate.
|
||||||
|
await this.api.detach({ tabId }).catch((detachErr) => {
|
||||||
|
console.debug("[bsk cdp] rollback detach failed", { tabId, detachErr });
|
||||||
|
});
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
})()
|
})()
|
||||||
.catch((err) => {
|
.catch((err) => {
|
||||||
// Chrome surfaces "Another debugger is already attached" when
|
// Chrome surfaces "Another debugger is already attached" when
|
||||||
@@ -206,6 +254,73 @@ export class ChromiumCdp {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async sendToTarget<T = unknown>(target: CdpTarget, method: string, params?: object): Promise<T> {
|
||||||
|
if (!this.attachedTabs.has(target.tabId)) {
|
||||||
|
await this.ensureAttached(target.tabId);
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
return (await this.api.sendCommand(target, method, params ?? {})) as T;
|
||||||
|
} catch (err) {
|
||||||
|
throw normalizeError(err);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async getFrameGraph(tabId: number): Promise<CdpFrameGraph> {
|
||||||
|
await this.ensureAttached(tabId);
|
||||||
|
await this.enableFrameDiscovery({ tabId }).catch(() => {});
|
||||||
|
await this.drainFrameAttachTasks(tabId);
|
||||||
|
|
||||||
|
const sources: CdpFrameTreeSource[] = [];
|
||||||
|
const root = await this.sendToTarget<{ frameTree?: CdpFrameTreeNode }>(
|
||||||
|
{ tabId },
|
||||||
|
"Page.getFrameTree",
|
||||||
|
{},
|
||||||
|
);
|
||||||
|
if (root.frameTree) sources.push({ target: { tabId }, tree: root.frameTree });
|
||||||
|
|
||||||
|
const sessions = [...(this.frameDiscovery.get(tabId)?.sessions ?? [])];
|
||||||
|
const childTrees = await Promise.all(
|
||||||
|
sessions.map(async (sessionId): Promise<CdpFrameTreeSource | null> => {
|
||||||
|
const target = { tabId, sessionId };
|
||||||
|
try {
|
||||||
|
const reply = await this.sendToTarget<{ frameTree?: CdpFrameTreeNode }>(
|
||||||
|
target,
|
||||||
|
"Page.getFrameTree",
|
||||||
|
{},
|
||||||
|
);
|
||||||
|
return reply.frameTree ? { target, tree: reply.frameTree } : null;
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
for (const source of childTrees) {
|
||||||
|
if (source) sources.push(source);
|
||||||
|
}
|
||||||
|
|
||||||
|
const graph = buildFrameGraph(sources);
|
||||||
|
if (!graph) throw new Error("Page.getFrameTree returned no root frame");
|
||||||
|
const frameById = new Map(graph.frames.map((frame) => [frame.frameId, frame]));
|
||||||
|
await Promise.all(
|
||||||
|
graph.frames.map(async (frame) => {
|
||||||
|
if (!frame.parentFrameId) return;
|
||||||
|
const parent = frameById.get(frame.parentFrameId);
|
||||||
|
if (!parent) return;
|
||||||
|
try {
|
||||||
|
const owner = await this.sendToTarget<{ backendNodeId?: number }>(
|
||||||
|
parent.target,
|
||||||
|
"DOM.getFrameOwner",
|
||||||
|
{ frameId: frame.frameId },
|
||||||
|
);
|
||||||
|
if (owner.backendNodeId !== undefined) frame.ownerBackendNodeId = owner.backendNodeId;
|
||||||
|
} catch {
|
||||||
|
// The frame may have navigated between tree capture and owner lookup.
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
return graph;
|
||||||
|
}
|
||||||
|
|
||||||
/** Return a cursor marking the current dialog sequence for `tabId`. */
|
/** Return a cursor marking the current dialog sequence for `tabId`. */
|
||||||
dialogCursor(tabId: number): DialogCursor {
|
dialogCursor(tabId: number): DialogCursor {
|
||||||
return this.dialogSequences.get(tabId) ?? 0;
|
return this.dialogSequences.get(tabId) ?? 0;
|
||||||
@@ -316,6 +431,7 @@ export class ChromiumCdp {
|
|||||||
this.clearDialogState(tabId);
|
this.clearDialogState(tabId);
|
||||||
this.clearConsoleState(tabId);
|
this.clearConsoleState(tabId);
|
||||||
this.clearNetworkState(tabId);
|
this.clearNetworkState(tabId);
|
||||||
|
this.clearFrameState(tabId);
|
||||||
try {
|
try {
|
||||||
await this.api.detach({ tabId });
|
await this.api.detach({ tabId });
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
@@ -338,7 +454,7 @@ export class ChromiumCdp {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/** Subscribe to all CDP events. Returned disposable removes the listener. */
|
/** Subscribe to all CDP events. Returned disposable removes the listener. */
|
||||||
onEvent(handler: (source: chrome.debugger.Debuggee, method: string, params: unknown) => void): {
|
onEvent(handler: (source: CdpDebuggee, method: string, params: unknown) => void): {
|
||||||
dispose(): void;
|
dispose(): void;
|
||||||
} {
|
} {
|
||||||
this.api.onEvent.addListener(handler);
|
this.api.onEvent.addListener(handler);
|
||||||
@@ -357,11 +473,12 @@ export class ChromiumCdp {
|
|||||||
this.dialogSequences.clear();
|
this.dialogSequences.clear();
|
||||||
this.consoleBuffers.clear();
|
this.consoleBuffers.clear();
|
||||||
this.consoleSequences.clear();
|
this.consoleSequences.clear();
|
||||||
this.consoleDomainsAttemptedTabs.clear();
|
this.consoleDomainsEnabledTabs.clear();
|
||||||
this.networkBuffers.clear();
|
this.networkBuffers.clear();
|
||||||
this.networkSequences.clear();
|
this.networkSequences.clear();
|
||||||
this.networkDomainsEnabledTabs.clear();
|
this.networkDomainsEnabledTabs.clear();
|
||||||
this.networkRequestMeta.clear();
|
this.networkRequestMeta.clear();
|
||||||
|
this.frameDiscovery.clear();
|
||||||
await Promise.all(
|
await Promise.all(
|
||||||
tabs.map(async (tabId) => {
|
tabs.map(async (tabId) => {
|
||||||
try {
|
try {
|
||||||
@@ -378,15 +495,24 @@ export class ChromiumCdp {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private async enableConsoleDomains(tabId: number): Promise<void> {
|
private async enableConsoleDomains(tabId: number): Promise<void> {
|
||||||
if (this.consoleDomainsAttemptedTabs.has(tabId)) return;
|
if (this.consoleDomainsEnabledTabs.has(tabId)) return;
|
||||||
this.consoleDomainsAttemptedTabs.add(tabId);
|
// Mark the tab only after both domains enable successfully — a
|
||||||
|
// transient failure (e.g. restricted page during attach) must leave
|
||||||
|
// the tab unmarked so a later `ensureConsoleCapture` can retry,
|
||||||
|
// instead of silently returning no console output forever. Mirrors
|
||||||
|
// `enableNetworkDomain`, which records the tab after success only.
|
||||||
|
let failed = false;
|
||||||
for (const method of ["Runtime.enable", "Log.enable"]) {
|
for (const method of ["Runtime.enable", "Log.enable"]) {
|
||||||
try {
|
try {
|
||||||
await this.api.sendCommand({ tabId }, method, {});
|
await this.api.sendCommand({ tabId }, method, {});
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
|
failed = true;
|
||||||
console.debug("[bsk cdp] console domain enable failed", { tabId, method, err });
|
console.debug("[bsk cdp] console domain enable failed", { tabId, method, err });
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if (!failed) {
|
||||||
|
this.consoleDomainsEnabledTabs.add(tabId);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private async enableNetworkDomain(tabId: number): Promise<void> {
|
private async enableNetworkDomain(tabId: number): Promise<void> {
|
||||||
@@ -403,9 +529,104 @@ export class ChromiumCdp {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private async enableFrameDiscovery(target: CdpTarget): Promise<void> {
|
||||||
|
await this.api.sendCommand(target, "Target.setAutoAttach", {
|
||||||
|
autoAttach: true,
|
||||||
|
waitForDebuggerOnStart: false,
|
||||||
|
flatten: true,
|
||||||
|
filter: [{ type: "iframe", exclude: false }],
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
private bindFrameTargetHandler(): void {
|
||||||
|
if (this.frameTargetSubscription) return;
|
||||||
|
const listener = (source: CdpDebuggee, method: string, params: unknown) => {
|
||||||
|
const tabId = source.tabId;
|
||||||
|
if (typeof tabId !== "number") return;
|
||||||
|
const raw = (params ?? {}) as Record<string, unknown>;
|
||||||
|
const sessionId = typeof raw.sessionId === "string" ? raw.sessionId : undefined;
|
||||||
|
if (!sessionId) return;
|
||||||
|
|
||||||
|
if (method === "Target.detachedFromTarget") {
|
||||||
|
const state = this.frameDiscovery.get(tabId);
|
||||||
|
if (state?.sessions.delete(sessionId)) state.generation += 1;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (method !== "Target.attachedToTarget") return;
|
||||||
|
const targetInfo = raw.targetInfo as { type?: string } | undefined;
|
||||||
|
if (targetInfo?.type && targetInfo.type !== "iframe") return;
|
||||||
|
|
||||||
|
const state = this.frameDiscoveryState(tabId);
|
||||||
|
if (state.sessions.has(sessionId)) return;
|
||||||
|
state.sessions.add(sessionId);
|
||||||
|
state.generation += 1;
|
||||||
|
|
||||||
|
const task = this.initializeFrameTarget({ tabId, sessionId });
|
||||||
|
state.pending.add(task);
|
||||||
|
void task.finally(() => {
|
||||||
|
state.pending.delete(task);
|
||||||
|
});
|
||||||
|
};
|
||||||
|
this.api.onEvent.addListener(listener);
|
||||||
|
this.frameTargetSubscription = {
|
||||||
|
dispose: () => this.api.onEvent.removeListener(listener),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private async initializeFrameTarget(target: CdpTarget): Promise<void> {
|
||||||
|
try {
|
||||||
|
await this.enableFrameDiscovery(target);
|
||||||
|
} catch (err) {
|
||||||
|
const state = this.frameDiscovery.get(target.tabId);
|
||||||
|
if (state?.sessions.delete(target.sessionId as string)) state.generation += 1;
|
||||||
|
console.debug("[bsk cdp] child frame target initialization failed", { target, err });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async drainFrameAttachTasks(tabId: number): Promise<void> {
|
||||||
|
const deadline = Date.now() + FRAME_DISCOVERY_TIMEOUT_MS;
|
||||||
|
while (Date.now() < deadline) {
|
||||||
|
const state = this.frameDiscovery.get(tabId);
|
||||||
|
const generation = state?.generation ?? 0;
|
||||||
|
if (state?.pending.size && !(await settleBeforeDeadline([...state.pending], deadline))) break;
|
||||||
|
|
||||||
|
// Target.setAutoAttach may enqueue the next attachedToTarget event after
|
||||||
|
// its command promise settles. Wait for a short quiet window, then finish
|
||||||
|
// only if neither the state object nor its generation changed.
|
||||||
|
const quietTime = Math.min(FRAME_DISCOVERY_QUIET_MS, deadline - Date.now());
|
||||||
|
if (quietTime <= 0) break;
|
||||||
|
await new Promise<void>((resolve) => setTimeout(resolve, quietTime));
|
||||||
|
const current = this.frameDiscovery.get(tabId);
|
||||||
|
if (
|
||||||
|
current === state &&
|
||||||
|
(current?.generation ?? 0) === generation &&
|
||||||
|
!current?.pending.size
|
||||||
|
) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
console.debug("[bsk cdp] frame discovery did not reach quiescence before timeout", { tabId });
|
||||||
|
}
|
||||||
|
|
||||||
|
private frameDiscoveryState(tabId: number): FrameDiscoveryState {
|
||||||
|
const existing = this.frameDiscovery.get(tabId);
|
||||||
|
if (existing) return existing;
|
||||||
|
const created: FrameDiscoveryState = {
|
||||||
|
sessions: new Set(),
|
||||||
|
pending: new Set(),
|
||||||
|
generation: 0,
|
||||||
|
};
|
||||||
|
this.frameDiscovery.set(tabId, created);
|
||||||
|
return created;
|
||||||
|
}
|
||||||
|
|
||||||
|
private clearFrameState(tabId: number): void {
|
||||||
|
this.frameDiscovery.delete(tabId);
|
||||||
|
}
|
||||||
|
|
||||||
private bindDialogHandler(): void {
|
private bindDialogHandler(): void {
|
||||||
if (this.dialogSubscription) return;
|
if (this.dialogSubscription) return;
|
||||||
const listener = (source: chrome.debugger.Debuggee, method: string, params: unknown) => {
|
const listener = (source: CdpDebuggee, method: string, params: unknown) => {
|
||||||
if (method !== "Page.javascriptDialogOpening") return;
|
if (method !== "Page.javascriptDialogOpening") return;
|
||||||
const tabId = source.tabId;
|
const tabId = source.tabId;
|
||||||
if (typeof tabId !== "number") return;
|
if (typeof tabId !== "number") return;
|
||||||
@@ -489,7 +710,7 @@ export class ChromiumCdp {
|
|||||||
private clearConsoleState(tabId: number): void {
|
private clearConsoleState(tabId: number): void {
|
||||||
this.consoleBuffers.delete(tabId);
|
this.consoleBuffers.delete(tabId);
|
||||||
this.consoleSequences.delete(tabId);
|
this.consoleSequences.delete(tabId);
|
||||||
this.consoleDomainsAttemptedTabs.delete(tabId);
|
this.consoleDomainsEnabledTabs.delete(tabId);
|
||||||
}
|
}
|
||||||
|
|
||||||
private bindNetworkHandler(): void {
|
private bindNetworkHandler(): void {
|
||||||
@@ -589,6 +810,7 @@ export class ChromiumCdp {
|
|||||||
this.clearDialogState(source.tabId);
|
this.clearDialogState(source.tabId);
|
||||||
this.clearConsoleState(source.tabId);
|
this.clearConsoleState(source.tabId);
|
||||||
this.clearNetworkState(source.tabId);
|
this.clearNetworkState(source.tabId);
|
||||||
|
this.clearFrameState(source.tabId);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
this.api.onDetach.addListener(listener);
|
this.api.onDetach.addListener(listener);
|
||||||
@@ -607,6 +829,8 @@ export class ChromiumCdp {
|
|||||||
this.consoleSubscription = null;
|
this.consoleSubscription = null;
|
||||||
this.networkSubscription?.dispose();
|
this.networkSubscription?.dispose();
|
||||||
this.networkSubscription = null;
|
this.networkSubscription = null;
|
||||||
|
this.frameTargetSubscription?.dispose();
|
||||||
|
this.frameTargetSubscription = null;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Detach tabs only when no other live session has claimed them. */
|
/** Detach tabs only when no other live session has claimed them. */
|
||||||
|
|||||||
@@ -0,0 +1,107 @@
|
|||||||
|
export interface CdpTarget {
|
||||||
|
tabId: number;
|
||||||
|
sessionId?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface CdpFrame {
|
||||||
|
frameId: string;
|
||||||
|
parentFrameId?: string;
|
||||||
|
ownerBackendNodeId?: number;
|
||||||
|
url?: string;
|
||||||
|
target: CdpTarget;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface CdpFrameGraph {
|
||||||
|
rootFrameId: string;
|
||||||
|
frames: CdpFrame[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface CdpFrameTreeNode {
|
||||||
|
frame: {
|
||||||
|
id: string;
|
||||||
|
parentId?: string;
|
||||||
|
url?: string;
|
||||||
|
};
|
||||||
|
childFrames?: CdpFrameTreeNode[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface CdpFrameTreeSource {
|
||||||
|
target: CdpTarget;
|
||||||
|
tree: CdpFrameTreeNode;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function cdpTargetKey(target: CdpTarget): string {
|
||||||
|
return `${target.tabId}:${target.sessionId ?? "root"}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function mergeFrame(
|
||||||
|
frames: Map<string, CdpFrame>,
|
||||||
|
order: string[],
|
||||||
|
node: CdpFrameTreeNode,
|
||||||
|
target: CdpTarget,
|
||||||
|
): void {
|
||||||
|
const existing = frames.get(node.frame.id);
|
||||||
|
if (!existing) order.push(node.frame.id);
|
||||||
|
const parentFrameId = node.frame.parentId || existing?.parentFrameId;
|
||||||
|
const url = node.frame.url || existing?.url;
|
||||||
|
frames.set(node.frame.id, {
|
||||||
|
frameId: node.frame.id,
|
||||||
|
...(parentFrameId ? { parentFrameId } : {}),
|
||||||
|
...(url ? { url } : {}),
|
||||||
|
target,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function walkFrameTree(
|
||||||
|
source: CdpFrameTreeSource,
|
||||||
|
targetByRootFrameId: ReadonlyMap<string, CdpTarget>,
|
||||||
|
frames: Map<string, CdpFrame>,
|
||||||
|
order: string[],
|
||||||
|
): void {
|
||||||
|
const stack: Array<{ node: CdpFrameTreeNode; inheritedTarget: CdpTarget }> = [
|
||||||
|
{ node: source.tree, inheritedTarget: source.target },
|
||||||
|
];
|
||||||
|
const expanded = new WeakSet<CdpFrameTreeNode>();
|
||||||
|
|
||||||
|
while (stack.length > 0) {
|
||||||
|
const current = stack.pop();
|
||||||
|
if (!current || expanded.has(current.node)) continue;
|
||||||
|
expanded.add(current.node);
|
||||||
|
|
||||||
|
const target = targetByRootFrameId.get(current.node.frame.id) ?? current.inheritedTarget;
|
||||||
|
mergeFrame(frames, order, current.node, target);
|
||||||
|
|
||||||
|
const children = current.node.childFrames ?? [];
|
||||||
|
for (let index = children.length - 1; index >= 0; index -= 1) {
|
||||||
|
stack.push({ node: children[index], inheritedTarget: target });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function buildFrameGraph(sources: CdpFrameTreeSource[]): CdpFrameGraph | null {
|
||||||
|
const rootSource = sources.find((source) => source.target.sessionId === undefined) ?? sources[0];
|
||||||
|
if (!rootSource) return null;
|
||||||
|
|
||||||
|
const frames = new Map<string, CdpFrame>();
|
||||||
|
const order: string[] = [];
|
||||||
|
const targetByRootFrameId = new Map(
|
||||||
|
sources.map((source) => [source.tree.frame.id, source.target] as const),
|
||||||
|
);
|
||||||
|
// The top-level source is authoritative if malformed input reports the same
|
||||||
|
// frame as both a root target and a child target.
|
||||||
|
targetByRootFrameId.set(rootSource.tree.frame.id, rootSource.target);
|
||||||
|
|
||||||
|
walkFrameTree(rootSource, targetByRootFrameId, frames, order);
|
||||||
|
for (const source of sources) {
|
||||||
|
if (source === rootSource) continue;
|
||||||
|
walkFrameTree(source, targetByRootFrameId, frames, order);
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
rootFrameId: rootSource.tree.frame.id,
|
||||||
|
frames: order.flatMap((frameId) => {
|
||||||
|
const frame = frames.get(frameId);
|
||||||
|
return frame ? [frame] : [];
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -8,6 +8,7 @@ import {
|
|||||||
useRef,
|
useRef,
|
||||||
useState,
|
useState,
|
||||||
} from "react";
|
} from "react";
|
||||||
|
import type { HelpHighlightRect } from "@/lib/help-bridge";
|
||||||
import logoUrl from "../../assets/logo.png";
|
import logoUrl from "../../assets/logo.png";
|
||||||
|
|
||||||
export interface HelpRequestData {
|
export interface HelpRequestData {
|
||||||
@@ -19,6 +20,10 @@ export interface HelpRequestData {
|
|||||||
displayMode?: "full" | "compact";
|
displayMode?: "full" | "compact";
|
||||||
/** CSS selectors to scroll to + flash-highlight. */
|
/** CSS selectors to scroll to + flash-highlight. */
|
||||||
selectors: string[];
|
selectors: string[];
|
||||||
|
/** Pre-resolved top-viewport rectangles, used for cross-frame targets. */
|
||||||
|
rects?: HelpHighlightRect[];
|
||||||
|
/** Re-resolve cross-frame rectangles after scroll, resize, or layout changes. */
|
||||||
|
refreshRects?: () => Promise<HelpHighlightRect[] | undefined>;
|
||||||
onContinue: (note: string) => void;
|
onContinue: (note: string) => void;
|
||||||
onCancel: () => void;
|
onCancel: () => void;
|
||||||
}
|
}
|
||||||
@@ -49,6 +54,7 @@ const PANEL_WIDTH = 420;
|
|||||||
const FALLBACK_PANEL_H = 180;
|
const FALLBACK_PANEL_H = 180;
|
||||||
const FALLBACK_PANEL_H_COLLAPSED = 44;
|
const FALLBACK_PANEL_H_COLLAPSED = 44;
|
||||||
const EMPTY_SELECTORS: string[] = [];
|
const EMPTY_SELECTORS: string[] = [];
|
||||||
|
const EMPTY_RECTS: HelpHighlightRect[] = [];
|
||||||
|
|
||||||
function clamp(value: number, min: number, max: number): number {
|
function clamp(value: number, min: number, max: number): number {
|
||||||
return Math.max(min, Math.min(max, value));
|
return Math.max(min, Math.min(max, value));
|
||||||
@@ -179,6 +185,8 @@ export function HelpRequestOverlay({ request }: Props) {
|
|||||||
const { t } = useTranslation("extension");
|
const { t } = useTranslation("extension");
|
||||||
const isCompact = request?.displayMode === "compact";
|
const isCompact = request?.displayMode === "compact";
|
||||||
const effectiveSelectors = isCompact ? EMPTY_SELECTORS : (request?.selectors ?? EMPTY_SELECTORS);
|
const effectiveSelectors = isCompact ? EMPTY_SELECTORS : (request?.selectors ?? EMPTY_SELECTORS);
|
||||||
|
const [liveRects, setLiveRects] = useState<HelpHighlightRect[] | null>(null);
|
||||||
|
const effectiveRects = isCompact ? EMPTY_RECTS : (liveRects ?? request?.rects ?? EMPTY_RECTS);
|
||||||
const [note, setNote] = useState("");
|
const [note, setNote] = useState("");
|
||||||
const [collapsed, setCollapsed] = useState(false);
|
const [collapsed, setCollapsed] = useState(false);
|
||||||
const [boxes, setBoxes] = useState<Box[]>([]);
|
const [boxes, setBoxes] = useState<Box[]>([]);
|
||||||
@@ -201,6 +209,7 @@ export function HelpRequestOverlay({ request }: Props) {
|
|||||||
// Reset the note and collapse state whenever a new request appears.
|
// Reset the note and collapse state whenever a new request appears.
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
setNote("");
|
setNote("");
|
||||||
|
setLiveRects(null);
|
||||||
setCollapsed(false);
|
setCollapsed(false);
|
||||||
placementRef.current = null;
|
placementRef.current = null;
|
||||||
setDragPos(null);
|
setDragPos(null);
|
||||||
@@ -209,6 +218,32 @@ export function HelpRequestOverlay({ request }: Props) {
|
|||||||
dragStartRef.current = null;
|
dragStartRef.current = null;
|
||||||
}, [request?.id]);
|
}, [request?.id]);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (!request || isCompact || !request.refreshRects) return;
|
||||||
|
let disposed = false;
|
||||||
|
let resolving = false;
|
||||||
|
const refresh = async () => {
|
||||||
|
if (resolving) return;
|
||||||
|
resolving = true;
|
||||||
|
try {
|
||||||
|
const rects = await request.refreshRects?.();
|
||||||
|
if (!disposed && rects) setLiveRects(rects);
|
||||||
|
} finally {
|
||||||
|
resolving = false;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
const onViewportChange = () => void refresh();
|
||||||
|
window.addEventListener("scroll", onViewportChange, true);
|
||||||
|
window.addEventListener("resize", onViewportChange);
|
||||||
|
const interval = window.setInterval(onViewportChange, 500);
|
||||||
|
return () => {
|
||||||
|
disposed = true;
|
||||||
|
window.removeEventListener("scroll", onViewportChange, true);
|
||||||
|
window.removeEventListener("resize", onViewportChange);
|
||||||
|
window.clearInterval(interval);
|
||||||
|
};
|
||||||
|
}, [request, isCompact]);
|
||||||
|
|
||||||
// Scroll the first matched target into view once per distinct request id.
|
// Scroll the first matched target into view once per distinct request id.
|
||||||
useLayoutEffect(() => {
|
useLayoutEffect(() => {
|
||||||
if (!request) return;
|
if (!request) return;
|
||||||
@@ -231,7 +266,7 @@ export function HelpRequestOverlay({ request }: Props) {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
const update = () => {
|
const update = () => {
|
||||||
const nextBoxes = measure(effectiveSelectors);
|
const nextBoxes = [...measure(effectiveSelectors), ...effectiveRects];
|
||||||
setBoxes((prev) => (boxesEqual(prev, nextBoxes) ? prev : nextBoxes));
|
setBoxes((prev) => (boxesEqual(prev, nextBoxes) ? prev : nextBoxes));
|
||||||
};
|
};
|
||||||
update();
|
update();
|
||||||
@@ -243,7 +278,7 @@ export function HelpRequestOverlay({ request }: Props) {
|
|||||||
window.removeEventListener("resize", update);
|
window.removeEventListener("resize", update);
|
||||||
window.clearInterval(interval);
|
window.clearInterval(interval);
|
||||||
};
|
};
|
||||||
}, [request, effectiveSelectors]);
|
}, [request, effectiveSelectors, effectiveRects]);
|
||||||
|
|
||||||
const onHeaderPointerDown = useCallback(
|
const onHeaderPointerDown = useCallback(
|
||||||
(e: ReactPointerEvent<HTMLDivElement>) => {
|
(e: ReactPointerEvent<HTMLDivElement>) => {
|
||||||
@@ -305,7 +340,7 @@ export function HelpRequestOverlay({ request }: Props) {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if (userMovedRef.current) return;
|
if (userMovedRef.current) return;
|
||||||
const measured = boxes.length > 0 ? boxes : measure(effectiveSelectors);
|
const measured = boxes.length > 0 ? boxes : [...measure(effectiveSelectors), ...effectiveRects];
|
||||||
const anchor = unionBox(measured);
|
const anchor = unionBox(measured);
|
||||||
const banner = bannerRef.current;
|
const banner = bannerRef.current;
|
||||||
if (!anchor || !banner) {
|
if (!anchor || !banner) {
|
||||||
@@ -329,7 +364,7 @@ export function HelpRequestOverlay({ request }: Props) {
|
|||||||
}
|
}
|
||||||
const nextPos = placePanel(anchor, panelW, panelH, placementRef.current.placement);
|
const nextPos = placePanel(anchor, panelW, panelH, placementRef.current.placement);
|
||||||
setPanelPos((prev) => (panelPosEqual(prev, nextPos) ? prev : nextPos));
|
setPanelPos((prev) => (panelPosEqual(prev, nextPos) ? prev : nextPos));
|
||||||
}, [request, boxes, collapsed, isCompact, effectiveSelectors]);
|
}, [request, boxes, collapsed, isCompact, effectiveSelectors, effectiveRects]);
|
||||||
|
|
||||||
if (!request) return null;
|
if (!request) return null;
|
||||||
|
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
import { i18n } from "@browser-skill/i18n";
|
import { i18n } from "@browser-skill/i18n";
|
||||||
import { I18nextProvider } from "@browser-skill/i18n/react";
|
import { I18nextProvider } from "@browser-skill/i18n/react";
|
||||||
import { cleanup, fireEvent, render, screen } from "@testing-library/react";
|
import { cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react";
|
||||||
import { createElement } from "react";
|
import { createElement } from "react";
|
||||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||||
import { type HelpRequestData, HelpRequestOverlay } from "../HelpRequestOverlay";
|
import { type HelpRequestData, HelpRequestOverlay } from "../HelpRequestOverlay";
|
||||||
@@ -38,6 +38,36 @@ describe("HelpRequestOverlay", () => {
|
|||||||
expect(screen.getByText("Please complete the captcha")).toBeTruthy();
|
expect(screen.getByText("Please complete the captcha")).toBeTruthy();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("renders explicit viewport rectangles for cross-frame targets", () => {
|
||||||
|
const { container } = renderOverlay(
|
||||||
|
baseRequest({ rects: [{ top: 30, left: 40, width: 120, height: 50 }] }),
|
||||||
|
);
|
||||||
|
const highlight = container.querySelector<HTMLElement>("[data-slot='help-highlight']");
|
||||||
|
expect(highlight?.style.top).toBe("30px");
|
||||||
|
expect(highlight?.style.left).toBe("40px");
|
||||||
|
expect(highlight?.style.width).toBe("120px");
|
||||||
|
expect(highlight?.style.height).toBe("50px");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("re-resolves cross-frame rectangles after the viewport changes", async () => {
|
||||||
|
const refreshRects = vi.fn(async () => [{ top: 80, left: 90, width: 140, height: 60 }]);
|
||||||
|
const { container } = renderOverlay(
|
||||||
|
baseRequest({
|
||||||
|
rects: [{ top: 30, left: 40, width: 120, height: 50 }],
|
||||||
|
refreshRects,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
window.dispatchEvent(new Event("scroll"));
|
||||||
|
|
||||||
|
await waitFor(() => {
|
||||||
|
const highlight = container.querySelector<HTMLElement>("[data-slot='help-highlight']");
|
||||||
|
expect(highlight?.style.top).toBe("80px");
|
||||||
|
expect(highlight?.style.left).toBe("90px");
|
||||||
|
});
|
||||||
|
expect(refreshRects).toHaveBeenCalledTimes(1);
|
||||||
|
});
|
||||||
|
|
||||||
it("keeps the inactive render stable", () => {
|
it("keeps the inactive render stable", () => {
|
||||||
const { container, rerender } = renderOverlay(null);
|
const { container, rerender } = renderOverlay(null);
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,64 @@
|
|||||||
|
import { describe, expect, it, vi } from "vitest";
|
||||||
|
import { createHelpRequestData } from "../help-request";
|
||||||
|
|
||||||
|
describe("createHelpRequestData", () => {
|
||||||
|
it("gives live delivery and recovery the same rectangle refresh mechanism", async () => {
|
||||||
|
const finish = vi.fn();
|
||||||
|
const query = vi.fn(async () => ({
|
||||||
|
active: true,
|
||||||
|
request: {
|
||||||
|
requestId: "help-1",
|
||||||
|
prompt: "complete the challenge",
|
||||||
|
selectors: [],
|
||||||
|
rects: [{ top: 80, left: 90, width: 140, height: 60 }],
|
||||||
|
timeoutMs: 1_000,
|
||||||
|
},
|
||||||
|
}));
|
||||||
|
const request = createHelpRequestData(
|
||||||
|
{
|
||||||
|
requestId: "help-1",
|
||||||
|
prompt: "complete the challenge",
|
||||||
|
selectors: [],
|
||||||
|
rects: [{ top: 10, left: 20, width: 30, height: 40 }],
|
||||||
|
timeoutMs: 1_000,
|
||||||
|
},
|
||||||
|
{ finish, query },
|
||||||
|
);
|
||||||
|
|
||||||
|
await expect(request.refreshRects?.()).resolves.toEqual([
|
||||||
|
{ top: 80, left: 90, width: 140, height: 60 },
|
||||||
|
]);
|
||||||
|
request.onContinue(" done ");
|
||||||
|
request.onCancel();
|
||||||
|
|
||||||
|
expect(query).toHaveBeenCalledOnce();
|
||||||
|
expect(finish).toHaveBeenNthCalledWith(1, "help-1", "continued", " done ");
|
||||||
|
expect(finish).toHaveBeenNthCalledWith(2, "help-1", "cancelled");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("ignores rectangle replies for a replaced request", async () => {
|
||||||
|
const request = createHelpRequestData(
|
||||||
|
{
|
||||||
|
requestId: "help-1",
|
||||||
|
prompt: "complete the challenge",
|
||||||
|
selectors: [],
|
||||||
|
timeoutMs: 1_000,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
finish: vi.fn(),
|
||||||
|
query: vi.fn(async () => ({
|
||||||
|
active: true,
|
||||||
|
request: {
|
||||||
|
requestId: "help-2",
|
||||||
|
prompt: "new challenge",
|
||||||
|
selectors: [],
|
||||||
|
rects: [{ top: 1, left: 2, width: 3, height: 4 }],
|
||||||
|
timeoutMs: 1_000,
|
||||||
|
},
|
||||||
|
})),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
await expect(request.refreshRects?.()).resolves.toBeUndefined();
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -130,4 +130,31 @@ describe("OverlayController", () => {
|
|||||||
expect(controller.snapshot().controlVisible).toBe(true);
|
expect(controller.snapshot().controlVisible).toBe(true);
|
||||||
expect(shouldShowAgentControlOverlay(controller.snapshot())).toBe(true);
|
expect(shouldShowAgentControlOverlay(controller.snapshot())).toBe(true);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("hides the control overlay when the user hides control hints", () => {
|
||||||
|
const controller = new OverlayController();
|
||||||
|
controller.activateAgentSession("sess-1");
|
||||||
|
expect(shouldShowAgentControlOverlay(controller.snapshot())).toBe(true);
|
||||||
|
|
||||||
|
controller.setControlHintsHidden(true);
|
||||||
|
expect(controller.snapshot().controlHintsHidden).toBe(true);
|
||||||
|
// The session still owns the tab — only the chrome is hidden.
|
||||||
|
expect(controller.snapshot().controlVisible).toBe(true);
|
||||||
|
expect(shouldShowAgentControlOverlay(controller.snapshot())).toBe(false);
|
||||||
|
|
||||||
|
controller.setControlHintsHidden(false);
|
||||||
|
expect(shouldShowAgentControlOverlay(controller.snapshot())).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps the control-hints preference across session overlay resets", () => {
|
||||||
|
const controller = new OverlayController();
|
||||||
|
controller.activateAgentSession("sess-1");
|
||||||
|
controller.setControlHintsHidden(true);
|
||||||
|
|
||||||
|
controller.resetAgentOverlays("sess-1");
|
||||||
|
expect(controller.snapshot().controlHintsHidden).toBe(true);
|
||||||
|
|
||||||
|
controller.activateAgentSession("sess-2");
|
||||||
|
expect(shouldShowAgentControlOverlay(controller.snapshot())).toBe(false);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,109 @@
|
|||||||
|
import { describe, expect, it, vi } from "vitest";
|
||||||
|
import { isHoverSurfaceCandidateElement, isLikelyHoverSurfaceOwner } from "../record-hover-surface";
|
||||||
|
|
||||||
|
function mockRect(
|
||||||
|
el: Element,
|
||||||
|
rect: { left: number; top: number; width: number; height: number },
|
||||||
|
): void {
|
||||||
|
vi.spyOn(el, "getBoundingClientRect").mockReturnValue({
|
||||||
|
x: rect.left,
|
||||||
|
y: rect.top,
|
||||||
|
top: rect.top,
|
||||||
|
left: rect.left,
|
||||||
|
right: rect.left + rect.width,
|
||||||
|
bottom: rect.top + rect.height,
|
||||||
|
width: rect.width,
|
||||||
|
height: rect.height,
|
||||||
|
toJSON: () => ({}),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("record-hover-surface", () => {
|
||||||
|
it("recognizes popover containers without treating topbar wrappers or menu items as surfaces", () => {
|
||||||
|
document.body.innerHTML = `
|
||||||
|
<ul class="header-menu tw-flex navbar-app__authentication-menu">
|
||||||
|
<li class="tg-popover">
|
||||||
|
<div class="tg-avatar tg-avatar--img"></div>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<div class="tg-popover__popper">
|
||||||
|
<div class="tg-popover__content">
|
||||||
|
<div class="tg-popover__content-body">
|
||||||
|
<li class="header-menu__item">
|
||||||
|
<a class="header-dropdown-menu__link" href="/u/me">My profile</a>
|
||||||
|
</li>
|
||||||
|
<li class="header-menu__item">
|
||||||
|
<a class="header-dropdown-menu__link" href="/dashboard/groups">My groups</a>
|
||||||
|
</li>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
`;
|
||||||
|
|
||||||
|
expect(isHoverSurfaceCandidateElement(document.querySelector(".header-menu")!)).toBe(false);
|
||||||
|
expect(isHoverSurfaceCandidateElement(document.querySelector(".tg-popover")!)).toBe(false);
|
||||||
|
expect(isHoverSurfaceCandidateElement(document.querySelector(".header-menu__item")!)).toBe(
|
||||||
|
false,
|
||||||
|
);
|
||||||
|
expect(
|
||||||
|
isHoverSurfaceCandidateElement(document.querySelector(".header-dropdown-menu__link")!),
|
||||||
|
).toBe(false);
|
||||||
|
expect(isHoverSurfaceCandidateElement(document.querySelector(".tg-popover__popper")!)).toBe(
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
expect(isHoverSurfaceCandidateElement(document.querySelector(".tg-popover__content")!)).toBe(
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
expect(
|
||||||
|
isHoverSurfaceCandidateElement(document.querySelector(".tg-popover__content-body")!),
|
||||||
|
).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not treat elements covered by an open surface as the surface owner", () => {
|
||||||
|
document.body.innerHTML = `
|
||||||
|
<button class="avatar">image</button>
|
||||||
|
<a class="fork" href="/fork">Fork</a>
|
||||||
|
<button class="clone">Clone & Download</button>
|
||||||
|
<div class="tg-popover__content-body"></div>
|
||||||
|
`;
|
||||||
|
const avatar = document.querySelector(".avatar")!;
|
||||||
|
const fork = document.querySelector(".fork")!;
|
||||||
|
const clone = document.querySelector(".clone")!;
|
||||||
|
const surface = document.querySelector(".tg-popover__content-body")!;
|
||||||
|
mockRect(avatar, { left: 1340, top: 11, width: 26, height: 26 });
|
||||||
|
mockRect(fork, { left: 1275, top: 56, width: 69, height: 28 });
|
||||||
|
mockRect(clone, { left: 1159, top: 111, width: 180, height: 30 });
|
||||||
|
mockRect(surface, { left: 1111, top: 44, width: 240, height: 496 });
|
||||||
|
|
||||||
|
expect(isLikelyHoverSurfaceOwner(avatar, surface)).toBe(true);
|
||||||
|
expect(isLikelyHoverSurfaceOwner(fork, surface)).toBe(false);
|
||||||
|
expect(isLikelyHoverSurfaceOwner(clone, surface)).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not treat fixed navigation containers as plain floating surfaces", () => {
|
||||||
|
document.body.innerHTML = `
|
||||||
|
<header class="navbar-app">
|
||||||
|
<a href="/fork">Fork</a>
|
||||||
|
<button>image</button>
|
||||||
|
</header>
|
||||||
|
<div class="plain-floating"></div>
|
||||||
|
`;
|
||||||
|
const navbar = document.querySelector(".navbar-app")!;
|
||||||
|
const floating = document.querySelector(".plain-floating")!;
|
||||||
|
mockRect(navbar, { left: 0, top: 0, width: 1386, height: 51 });
|
||||||
|
mockRect(floating, { left: 1111, top: 44, width: 240, height: 496 });
|
||||||
|
vi.spyOn(window, "getComputedStyle").mockImplementation((el) => {
|
||||||
|
const style = {
|
||||||
|
cursor: "",
|
||||||
|
display: "block",
|
||||||
|
pointerEvents: "auto",
|
||||||
|
position: el === navbar ? "fixed" : "absolute",
|
||||||
|
visibility: "visible",
|
||||||
|
} as CSSStyleDeclaration;
|
||||||
|
return style;
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(isHoverSurfaceCandidateElement(navbar)).toBe(false);
|
||||||
|
expect(isHoverSurfaceCandidateElement(floating)).toBe(true);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,31 @@
|
|||||||
|
import type { HelpQueryResponse, HelpRequestMessage } from "@/lib/help-bridge";
|
||||||
|
import type { HelpRequestData } from "./HelpRequestOverlay";
|
||||||
|
|
||||||
|
export interface HelpRequestDataDeps {
|
||||||
|
finish(requestId: string, outcome: "continued" | "cancelled", note?: string): void;
|
||||||
|
query(): Promise<HelpQueryResponse | undefined>;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Build the content-side representation shared by live delivery and recovery. */
|
||||||
|
export function createHelpRequestData(
|
||||||
|
message: Omit<HelpRequestMessage, "type">,
|
||||||
|
deps: HelpRequestDataDeps,
|
||||||
|
): HelpRequestData {
|
||||||
|
return {
|
||||||
|
id: message.requestId,
|
||||||
|
prompt: message.prompt,
|
||||||
|
...(message.title ? { title: message.title } : {}),
|
||||||
|
...(message.displayMode ? { displayMode: message.displayMode } : {}),
|
||||||
|
selectors: message.selectors,
|
||||||
|
rects: message.rects,
|
||||||
|
refreshRects: async () => {
|
||||||
|
const response = await deps.query();
|
||||||
|
return response?.active && response.request?.requestId === message.requestId
|
||||||
|
? response.request.rects
|
||||||
|
: undefined;
|
||||||
|
},
|
||||||
|
onContinue: (note: string) =>
|
||||||
|
deps.finish(message.requestId, "continued", note.trim() ? note : undefined),
|
||||||
|
onCancel: () => deps.finish(message.requestId, "cancelled"),
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -18,6 +18,12 @@ export interface OverlayState {
|
|||||||
* so 「Agent 正在控制」does not flash between RecordOverlay and teardown.
|
* so 「Agent 正在控制」does not flash between RecordOverlay and teardown.
|
||||||
*/
|
*/
|
||||||
suppressControlAfterRecord: boolean;
|
suppressControlAfterRecord: boolean;
|
||||||
|
/**
|
||||||
|
* User preference (chrome.storage, toggled from the popup): hide the
|
||||||
|
* control hints — status pill, orange glow, and the input blocker that
|
||||||
|
* comes with them. Not session state; survives overlay resets.
|
||||||
|
*/
|
||||||
|
controlHintsHidden: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
type MutableOverlayState = Omit<OverlayState, "controlVisible">;
|
type MutableOverlayState = Omit<OverlayState, "controlVisible">;
|
||||||
@@ -36,6 +42,7 @@ export class OverlayController {
|
|||||||
controlMode: "hidden",
|
controlMode: "hidden",
|
||||||
automationBypassCount: 0,
|
automationBypassCount: 0,
|
||||||
suppressControlAfterRecord: false,
|
suppressControlAfterRecord: false,
|
||||||
|
controlHintsHidden: false,
|
||||||
};
|
};
|
||||||
|
|
||||||
snapshot(): OverlayState {
|
snapshot(): OverlayState {
|
||||||
@@ -129,6 +136,10 @@ export class OverlayController {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
setControlHintsHidden(hidden: boolean): void {
|
||||||
|
this.state.controlHintsHidden = hidden;
|
||||||
|
}
|
||||||
|
|
||||||
resetAgentOverlays(sessionId: string): HelpRequestData | null {
|
resetAgentOverlays(sessionId: string): HelpRequestData | null {
|
||||||
if (this.state.activeSessionId && this.state.activeSessionId !== sessionId) {
|
if (this.state.activeSessionId && this.state.activeSessionId !== sessionId) {
|
||||||
return null;
|
return null;
|
||||||
@@ -152,6 +163,7 @@ export class OverlayController {
|
|||||||
export function shouldShowAgentControlOverlay(state: OverlayState): boolean {
|
export function shouldShowAgentControlOverlay(state: OverlayState): boolean {
|
||||||
return (
|
return (
|
||||||
state.controlVisible &&
|
state.controlVisible &&
|
||||||
|
!state.controlHintsHidden &&
|
||||||
!state.suppressControlAfterRecord &&
|
!state.suppressControlAfterRecord &&
|
||||||
state.activeHelp === null &&
|
state.activeHelp === null &&
|
||||||
state.activeRecord === null
|
state.activeRecord === null
|
||||||
|
|||||||
@@ -1,4 +1,17 @@
|
|||||||
import { describeEventTarget, describeTarget, type TargetDescriptor } from "@/lib/describe-target";
|
import {
|
||||||
|
type CaptureTargetDescriptor,
|
||||||
|
describeEventTarget,
|
||||||
|
describeTarget,
|
||||||
|
resolveClickableElement,
|
||||||
|
resolveHoverElement,
|
||||||
|
} from "@/lib/describe-target";
|
||||||
|
import {
|
||||||
|
evaluateHoverTrigger,
|
||||||
|
type HoverTriggerDecision,
|
||||||
|
type HoverTriggerRect,
|
||||||
|
hasDirectHoverInteractiveSignal,
|
||||||
|
hasStrongHoverExpansionSignal,
|
||||||
|
} from "@/lib/hover-trigger-policy";
|
||||||
import {
|
import {
|
||||||
isRecordCancelMessage,
|
isRecordCancelMessage,
|
||||||
isRecordStartMessage,
|
isRecordStartMessage,
|
||||||
@@ -14,7 +27,15 @@ import {
|
|||||||
type RecordStopAck,
|
type RecordStopAck,
|
||||||
type RecordStopMessage,
|
type RecordStopMessage,
|
||||||
} from "@/lib/record-bridge";
|
} from "@/lib/record-bridge";
|
||||||
import { shouldRecordPress } from "@/lib/trace-reducer";
|
import { shouldRecordPress } from "@/lib/recording/draft-policy";
|
||||||
|
import {
|
||||||
|
closestHoverSurfaceCandidate,
|
||||||
|
collectHoverSurfaceStates,
|
||||||
|
decideHoverSurfaceRelation,
|
||||||
|
type HoverSurfaceState,
|
||||||
|
isHoverSurfaceCandidateElement,
|
||||||
|
isLikelyHoverSurfaceOwner,
|
||||||
|
} from "./record-hover-surface";
|
||||||
|
|
||||||
const pendingStepSends = new Map<string, Set<Promise<boolean>>>();
|
const pendingStepSends = new Map<string, Set<Promise<boolean>>>();
|
||||||
const failedStepDeliveries = new Set<string>();
|
const failedStepDeliveries = new Set<string>();
|
||||||
@@ -44,13 +65,34 @@ export interface RecordCaptureController {
|
|||||||
|
|
||||||
interface FillSession {
|
interface FillSession {
|
||||||
element: FillableElement;
|
element: FillableElement;
|
||||||
target: TargetDescriptor;
|
target: CaptureTargetDescriptor;
|
||||||
baselineValue: string;
|
baselineValue: string;
|
||||||
lastValue: string;
|
lastValue: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
interface HoverCandidate {
|
||||||
|
element: Element;
|
||||||
|
target: CaptureTargetDescriptor;
|
||||||
|
recordedAt: number;
|
||||||
|
score: number;
|
||||||
|
eligible: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface HoverSurfaceNode {
|
||||||
|
element: Element;
|
||||||
|
signature: string;
|
||||||
|
owner: HoverCandidate;
|
||||||
|
parent?: HoverSurfaceNode;
|
||||||
|
}
|
||||||
|
|
||||||
type FillableElement = HTMLInputElement | HTMLTextAreaElement | HTMLElement;
|
type FillableElement = HTMLInputElement | HTMLTextAreaElement | HTMLElement;
|
||||||
|
|
||||||
|
const HOVER_BEFORE_CLICK_MAX_MS = 10_000;
|
||||||
|
const HOVER_SURFACE_CONTEXT_MAX_MS = 30_000;
|
||||||
|
const HOVER_REPLACE_SCORE_MARGIN = 50;
|
||||||
|
const HOVER_CANDIDATE_LIMIT = 24;
|
||||||
|
const HOVER_TRIGGER_LABEL_MAX = 48;
|
||||||
|
|
||||||
function eventTarget(event: Event): EventTarget | null {
|
function eventTarget(event: Event): EventTarget | null {
|
||||||
return event.composedPath()[0] ?? event.target;
|
return event.composedPath()[0] ?? event.target;
|
||||||
}
|
}
|
||||||
@@ -122,6 +164,176 @@ function fillableValue(el: FillableElement): string {
|
|||||||
return el.textContent ?? "";
|
return el.textContent ?? "";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function hoverTriggerAttrs(el: Element): Record<string, string | undefined> {
|
||||||
|
return {
|
||||||
|
id: el.id || undefined,
|
||||||
|
class: typeof el.className === "string" ? el.className || undefined : undefined,
|
||||||
|
"data-testid": el.getAttribute("data-testid") ?? undefined,
|
||||||
|
"data-test": el.getAttribute("data-test") ?? undefined,
|
||||||
|
"data-cy": el.getAttribute("data-cy") ?? undefined,
|
||||||
|
"aria-label": el.getAttribute("aria-label") ?? undefined,
|
||||||
|
"aria-haspopup": el.getAttribute("aria-haspopup") ?? undefined,
|
||||||
|
"aria-controls": el.getAttribute("aria-controls") ?? undefined,
|
||||||
|
"aria-expanded": el.getAttribute("aria-expanded") ?? undefined,
|
||||||
|
"aria-hidden": el.getAttribute("aria-hidden") ?? undefined,
|
||||||
|
"aria-disabled": el.getAttribute("aria-disabled") ?? undefined,
|
||||||
|
contenteditable: el.getAttribute("contenteditable") ?? undefined,
|
||||||
|
disabled: el.hasAttribute("disabled") ? "" : undefined,
|
||||||
|
hidden: el.hasAttribute("hidden") ? "" : undefined,
|
||||||
|
inert: el.hasAttribute("inert") ? "" : undefined,
|
||||||
|
onclick: el.getAttribute("onclick") ?? undefined,
|
||||||
|
onmouseenter: el.getAttribute("onmouseenter") ?? undefined,
|
||||||
|
onmouseover: el.getAttribute("onmouseover") ?? undefined,
|
||||||
|
role: el.getAttribute("role") ?? undefined,
|
||||||
|
tabindex: el.getAttribute("tabindex") ?? undefined,
|
||||||
|
title: el.getAttribute("title") ?? undefined,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function hoverTriggerRect(el: Element): HoverTriggerRect {
|
||||||
|
const rect = el.getBoundingClientRect();
|
||||||
|
return { x: rect.left, y: rect.top, w: rect.width, h: rect.height };
|
||||||
|
}
|
||||||
|
|
||||||
|
function hoverTriggerStyle(el: Element): { cursor?: string; pointerEvents?: string } {
|
||||||
|
if (!(el instanceof HTMLElement)) return {};
|
||||||
|
const style = getComputedStyle(el);
|
||||||
|
return { cursor: style.cursor, pointerEvents: style.pointerEvents };
|
||||||
|
}
|
||||||
|
|
||||||
|
function hasHoverGraphicDescendant(el: Element): boolean {
|
||||||
|
return el.querySelector("img,svg,use,path,i") !== null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizeLabelText(value: string): string {
|
||||||
|
return value.replace(/\s+/g, " ").trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
function truncateLabel(value: string, max = HOVER_TRIGGER_LABEL_MAX): string {
|
||||||
|
const normalized = normalizeLabelText(value);
|
||||||
|
if (normalized.length <= max) return normalized;
|
||||||
|
return `${normalized.slice(0, max - 1)}…`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function collectHoverTriggerLabelText(root: Element): string {
|
||||||
|
let text = "";
|
||||||
|
const visit = (node: Node) => {
|
||||||
|
if (node.nodeType === Node.TEXT_NODE) {
|
||||||
|
text += ` ${node.textContent ?? ""}`;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (!(node instanceof Element)) return;
|
||||||
|
if (node !== root && isHoverSurfaceCandidateElement(node)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
for (const child of node.childNodes) visit(child);
|
||||||
|
};
|
||||||
|
for (const child of root.childNodes) visit(child);
|
||||||
|
return normalizeLabelText(text);
|
||||||
|
}
|
||||||
|
|
||||||
|
function compactHoverTargetName(
|
||||||
|
el: Element,
|
||||||
|
desc: CaptureTargetDescriptor,
|
||||||
|
): CaptureTargetDescriptor {
|
||||||
|
if (!desc.name) return desc;
|
||||||
|
const fullText = normalizeLabelText(el.textContent ?? "");
|
||||||
|
const compactName = collectHoverTriggerLabelText(el);
|
||||||
|
if (!compactName || compactName === desc.name) return desc;
|
||||||
|
const compactComparable = compactName.replace(/\s+/g, "");
|
||||||
|
const descComparable = desc.name.replace(/…$/, "").replace(/\s+/g, "");
|
||||||
|
const fullTextComparable = fullText.replace(/\s+/g, "");
|
||||||
|
if (
|
||||||
|
descComparable.startsWith(compactComparable) &&
|
||||||
|
fullTextComparable.startsWith(descComparable) &&
|
||||||
|
compactName.length < desc.name.length
|
||||||
|
) {
|
||||||
|
return { ...desc, name: truncateLabel(compactName) };
|
||||||
|
}
|
||||||
|
return desc;
|
||||||
|
}
|
||||||
|
|
||||||
|
function isWeakHoverTarget(target: CaptureTargetDescriptor): boolean {
|
||||||
|
return !target.role && !target.name && target.tag === "div";
|
||||||
|
}
|
||||||
|
|
||||||
|
function looksLikeAvatarElement(el: Element): boolean {
|
||||||
|
const className = typeof el.className === "string" ? el.className : "";
|
||||||
|
return /\b(avatar|user-avatar)\b/i.test(className);
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizeHoverTarget(
|
||||||
|
el: Element,
|
||||||
|
desc: CaptureTargetDescriptor,
|
||||||
|
decision: HoverTriggerDecision,
|
||||||
|
): CaptureTargetDescriptor {
|
||||||
|
if (desc.role === "img" && !desc.name && looksLikeAvatarElement(el)) {
|
||||||
|
return { ...desc, name: "image" };
|
||||||
|
}
|
||||||
|
if (
|
||||||
|
isWeakHoverTarget(desc) &&
|
||||||
|
(looksLikeAvatarElement(el) ||
|
||||||
|
(hasHoverGraphicDescendant(el) && decision.reasons.includes("icon-only")))
|
||||||
|
) {
|
||||||
|
return { tag: "img", role: "img", name: "image" };
|
||||||
|
}
|
||||||
|
return desc;
|
||||||
|
}
|
||||||
|
|
||||||
|
function hoverTriggerSignals(el: Element, desc: CaptureTargetDescriptor) {
|
||||||
|
if (!(el instanceof HTMLElement)) return null;
|
||||||
|
const style = hoverTriggerStyle(el);
|
||||||
|
return {
|
||||||
|
tag: el.tagName.toLowerCase(),
|
||||||
|
role: desc.role,
|
||||||
|
label: desc.name,
|
||||||
|
attrs: hoverTriggerAttrs(el),
|
||||||
|
rect: hoverTriggerRect(el),
|
||||||
|
cursor: style.cursor,
|
||||||
|
pointerEvents: style.pointerEvents,
|
||||||
|
hasGraphicDescendant: hasHoverGraphicDescendant(el),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function hoverCandidateFromEvent(target: EventTarget | null): HoverCandidate | null {
|
||||||
|
if (!(target instanceof Element)) return null;
|
||||||
|
const element = resolveHoverElement(target);
|
||||||
|
if (!element) return null;
|
||||||
|
const desc = describeTarget(element);
|
||||||
|
const signals = hoverTriggerSignals(element, desc);
|
||||||
|
if (!signals) return null;
|
||||||
|
const decision = evaluateHoverTrigger(signals);
|
||||||
|
const hasExpansionSignal = hasStrongHoverExpansionSignal(signals);
|
||||||
|
const hasDirectSignal = hasDirectHoverInteractiveSignal(signals);
|
||||||
|
if (!decision.eligible && !hasExpansionSignal && !hasDirectSignal) return null;
|
||||||
|
if (!decision.eligible && !desc.name && !desc.role) return null;
|
||||||
|
const normalizedTarget = normalizeHoverTarget(
|
||||||
|
element,
|
||||||
|
compactHoverTargetName(element, desc),
|
||||||
|
decision,
|
||||||
|
);
|
||||||
|
const now = Date.now();
|
||||||
|
return {
|
||||||
|
element,
|
||||||
|
target: normalizedTarget,
|
||||||
|
recordedAt: now,
|
||||||
|
score: decision.score,
|
||||||
|
eligible: decision.eligible,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function shouldReplaceHoverCandidate(
|
||||||
|
current: HoverCandidate,
|
||||||
|
next: HoverCandidate,
|
||||||
|
now: number,
|
||||||
|
): boolean {
|
||||||
|
if (next.element === current.element) return false;
|
||||||
|
if (now - current.recordedAt > HOVER_BEFORE_CLICK_MAX_MS) return true;
|
||||||
|
if (current.element.contains(next.element)) return false;
|
||||||
|
if (next.element.contains(current.element)) return true;
|
||||||
|
return next.score >= current.score + HOVER_REPLACE_SCORE_MARGIN;
|
||||||
|
}
|
||||||
|
|
||||||
/** Clicks that only pick an autocomplete/suggestion value — not a semantic submit action. */
|
/** Clicks that only pick an autocomplete/suggestion value — not a semantic submit action. */
|
||||||
function isInputCompletionClick(target: EventTarget | null, session: FillSession | null): boolean {
|
function isInputCompletionClick(target: EventTarget | null, session: FillSession | null): boolean {
|
||||||
if (!session || !(target instanceof Element)) return false;
|
if (!session || !(target instanceof Element)) return false;
|
||||||
@@ -168,10 +380,17 @@ export function startRecordCapture(
|
|||||||
const emitStep = (step: RecordStepPayload) => {
|
const emitStep = (step: RecordStepPayload) => {
|
||||||
sendStep({ page_url: location.href, ...step });
|
sendStep({ page_url: location.href, ...step });
|
||||||
};
|
};
|
||||||
|
const hoverSurfaceStateMap = (states: HoverSurfaceState[]): Map<Element, string> =>
|
||||||
|
new Map(states.map((state) => [state.element, state.signature]));
|
||||||
let fillSession: FillSession | null = null;
|
let fillSession: FillSession | null = null;
|
||||||
let composing = false;
|
let composing = false;
|
||||||
let lastUrl = location.href;
|
let lastUrl = location.href;
|
||||||
let keyboardActivation: { target: EventTarget | null; recordedAt: number } | undefined;
|
let keyboardActivation: { target: EventTarget | null; recordedAt: number } | undefined;
|
||||||
|
let pendingHover: HoverCandidate | null = null;
|
||||||
|
const recentHoverCandidates: HoverCandidate[] = [];
|
||||||
|
const emittedHoverElements = new WeakSet<Element>();
|
||||||
|
let hoverSurfaceStates = hoverSurfaceStateMap(collectHoverSurfaceStates());
|
||||||
|
const hoverSurfaceNodes = new Map<Element, HoverSurfaceNode>();
|
||||||
let generatedControlClick: Element | null = null;
|
let generatedControlClick: Element | null = null;
|
||||||
let navigationActionPending = false;
|
let navigationActionPending = false;
|
||||||
let navigationActionVersion = 0;
|
let navigationActionVersion = 0;
|
||||||
@@ -237,6 +456,223 @@ export function startRecordCapture(
|
|||||||
});
|
});
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const rememberHoverCandidate = (hover: HoverCandidate) => {
|
||||||
|
const duplicate = recentHoverCandidates.findIndex(
|
||||||
|
(candidate) => candidate.element === hover.element,
|
||||||
|
);
|
||||||
|
if (duplicate >= 0) recentHoverCandidates.splice(duplicate, 1);
|
||||||
|
recentHoverCandidates.push(hover);
|
||||||
|
if (recentHoverCandidates.length > HOVER_CANDIDATE_LIMIT) {
|
||||||
|
recentHoverCandidates.splice(0, recentHoverCandidates.length - HOVER_CANDIDATE_LIMIT);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const surfaceArea = (el: Element): number => {
|
||||||
|
const rect = el.getBoundingClientRect();
|
||||||
|
return rect.width * rect.height;
|
||||||
|
};
|
||||||
|
|
||||||
|
const smallestContaining = <T>(
|
||||||
|
target: Element,
|
||||||
|
items: Iterable<T>,
|
||||||
|
elementFor: (item: T) => Element,
|
||||||
|
): T | undefined => {
|
||||||
|
let best: T | undefined;
|
||||||
|
for (const item of items) {
|
||||||
|
const element = elementFor(item);
|
||||||
|
if (!element.contains(target)) continue;
|
||||||
|
if (!best || surfaceArea(element) < surfaceArea(elementFor(best))) {
|
||||||
|
best = item;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return best;
|
||||||
|
};
|
||||||
|
|
||||||
|
const ownedSurfaceContaining = (target: Element): HoverSurfaceNode | undefined =>
|
||||||
|
smallestContaining(target, hoverSurfaceNodes.values(), (node) => node.element);
|
||||||
|
|
||||||
|
const surfaceStateContaining = (
|
||||||
|
target: Element,
|
||||||
|
states: HoverSurfaceState[],
|
||||||
|
): HoverSurfaceState | undefined => smallestContaining(target, states, (state) => state.element);
|
||||||
|
|
||||||
|
const inferOwnerForSurface = (
|
||||||
|
surface: Element,
|
||||||
|
now: number,
|
||||||
|
before?: HoverCandidate,
|
||||||
|
): HoverCandidate | undefined => {
|
||||||
|
for (let index = recentHoverCandidates.length - 1; index >= 0; index -= 1) {
|
||||||
|
const candidate = recentHoverCandidates[index];
|
||||||
|
if (!candidate) continue;
|
||||||
|
if (before && candidate.element === before.element) continue;
|
||||||
|
if (before && candidate.recordedAt > before.recordedAt) continue;
|
||||||
|
if (surface.contains(candidate.element)) continue;
|
||||||
|
if (now - candidate.recordedAt > HOVER_SURFACE_CONTEXT_MAX_MS) continue;
|
||||||
|
if (isLikelyHoverSurfaceOwner(candidate.element, surface)) return candidate;
|
||||||
|
}
|
||||||
|
return undefined;
|
||||||
|
};
|
||||||
|
|
||||||
|
const nodeForUnownedSurface = (
|
||||||
|
surfaceState: HoverSurfaceState,
|
||||||
|
currentStates: HoverSurfaceState[],
|
||||||
|
now: number,
|
||||||
|
): HoverSurfaceNode | undefined => {
|
||||||
|
const owner = inferOwnerForSurface(surfaceState.element, now);
|
||||||
|
if (!owner) return undefined;
|
||||||
|
const parent = parentSurfaceNodeForOwner(owner, currentStates, now);
|
||||||
|
const node: HoverSurfaceNode = {
|
||||||
|
element: surfaceState.element,
|
||||||
|
signature: surfaceState.signature,
|
||||||
|
owner,
|
||||||
|
...(parent ? { parent } : {}),
|
||||||
|
};
|
||||||
|
hoverSurfaceNodes.set(surfaceState.element, node);
|
||||||
|
return node;
|
||||||
|
};
|
||||||
|
|
||||||
|
const ownedSurfaceForAction = (target: Element): HoverSurfaceNode | undefined => {
|
||||||
|
const closestSurface = closestHoverSurfaceCandidate(target);
|
||||||
|
if (closestSurface) {
|
||||||
|
const owned = hoverSurfaceNodes.get(closestSurface);
|
||||||
|
if (owned) return owned;
|
||||||
|
}
|
||||||
|
const ownedContaining = ownedSurfaceContaining(target);
|
||||||
|
if (ownedContaining) return ownedContaining;
|
||||||
|
const now = Date.now();
|
||||||
|
const currentStates = collectHoverSurfaceStates();
|
||||||
|
pruneGoneHoverSurfaces(currentStates);
|
||||||
|
const surfaceState = closestSurface
|
||||||
|
? currentStates.find((state) => state.element === closestSurface)
|
||||||
|
: surfaceStateContaining(target, currentStates);
|
||||||
|
hoverSurfaceStates = hoverSurfaceStateMap(currentStates);
|
||||||
|
if (!surfaceState) return undefined;
|
||||||
|
return nodeForUnownedSurface(surfaceState, currentStates, now);
|
||||||
|
};
|
||||||
|
|
||||||
|
const pruneGoneHoverSurfaces = (currentStates: HoverSurfaceState[]) => {
|
||||||
|
const current = new Set(currentStates.map((state) => state.element));
|
||||||
|
for (const element of hoverSurfaceNodes.keys()) {
|
||||||
|
if (!current.has(element)) hoverSurfaceNodes.delete(element);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const parentSurfaceNodeForOwner = (
|
||||||
|
owner: HoverCandidate,
|
||||||
|
currentStates: HoverSurfaceState[],
|
||||||
|
now: number,
|
||||||
|
): HoverSurfaceNode | undefined => {
|
||||||
|
const ownedParent = ownedSurfaceContaining(owner.element);
|
||||||
|
if (ownedParent) return ownedParent;
|
||||||
|
const parentState = surfaceStateContaining(owner.element, currentStates);
|
||||||
|
if (!parentState) return undefined;
|
||||||
|
const parentOwner = inferOwnerForSurface(parentState.element, now, owner);
|
||||||
|
if (!parentOwner) return undefined;
|
||||||
|
const parentNode: HoverSurfaceNode = {
|
||||||
|
element: parentState.element,
|
||||||
|
signature: parentState.signature,
|
||||||
|
owner: parentOwner,
|
||||||
|
};
|
||||||
|
hoverSurfaceNodes.set(parentState.element, parentNode);
|
||||||
|
return parentNode;
|
||||||
|
};
|
||||||
|
|
||||||
|
const createSurfaceNode = (
|
||||||
|
state: HoverSurfaceState,
|
||||||
|
owner: HoverCandidate,
|
||||||
|
currentStates: HoverSurfaceState[],
|
||||||
|
now: number,
|
||||||
|
): HoverSurfaceNode | undefined => {
|
||||||
|
const parent = parentSurfaceNodeForOwner(owner, currentStates, now);
|
||||||
|
if (parent?.element === state.element) return undefined;
|
||||||
|
if (now - owner.recordedAt > HOVER_SURFACE_CONTEXT_MAX_MS) return undefined;
|
||||||
|
if (!isLikelyHoverSurfaceOwner(owner.element, state.element)) return undefined;
|
||||||
|
const node: HoverSurfaceNode = {
|
||||||
|
element: state.element,
|
||||||
|
signature: state.signature,
|
||||||
|
owner,
|
||||||
|
...(parent ? { parent } : {}),
|
||||||
|
};
|
||||||
|
hoverSurfaceNodes.set(state.element, node);
|
||||||
|
return node;
|
||||||
|
};
|
||||||
|
|
||||||
|
const bindChangedHoverSurfaces = (
|
||||||
|
previousStates: Map<Element, string>,
|
||||||
|
currentStates: HoverSurfaceState[],
|
||||||
|
now: number,
|
||||||
|
) => {
|
||||||
|
for (const state of currentStates) {
|
||||||
|
if (previousStates.get(state.element) === state.signature) continue;
|
||||||
|
const owner = inferOwnerForSurface(state.element, now);
|
||||||
|
if (!owner) continue;
|
||||||
|
createSurfaceNode(state, owner, currentStates, now);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const refreshHoverSurfaces = (previousStates = hoverSurfaceStates) => {
|
||||||
|
const now = Date.now();
|
||||||
|
const currentStates = collectHoverSurfaceStates();
|
||||||
|
pruneGoneHoverSurfaces(currentStates);
|
||||||
|
bindChangedHoverSurfaces(previousStates, currentStates, now);
|
||||||
|
hoverSurfaceStates = hoverSurfaceStateMap(currentStates);
|
||||||
|
};
|
||||||
|
|
||||||
|
const scheduleHoverSurfaceRefresh = (previousStates: Map<Element, string>) => {
|
||||||
|
setTimeout(() => refreshHoverSurfaces(previousStates), 0);
|
||||||
|
};
|
||||||
|
|
||||||
|
const emitHoverStep = (hover: HoverCandidate) => {
|
||||||
|
if (emittedHoverElements.has(hover.element)) return;
|
||||||
|
emitStep({
|
||||||
|
op: "hover",
|
||||||
|
target: hover.target,
|
||||||
|
});
|
||||||
|
emittedHoverElements.add(hover.element);
|
||||||
|
};
|
||||||
|
|
||||||
|
const surfaceOwnerChain = (
|
||||||
|
surface: HoverSurfaceNode,
|
||||||
|
actionElement: Element,
|
||||||
|
now: number,
|
||||||
|
): HoverCandidate[] => {
|
||||||
|
const chain: HoverCandidate[] = [];
|
||||||
|
const selected = new WeakSet<Element>();
|
||||||
|
const surfaces: HoverSurfaceNode[] = [];
|
||||||
|
let node: HoverSurfaceNode | undefined = surface;
|
||||||
|
while (node && surfaces.length < 4) {
|
||||||
|
surfaces.unshift(node);
|
||||||
|
node = node.parent;
|
||||||
|
}
|
||||||
|
for (const owned of surfaces) {
|
||||||
|
const owner = owned.owner;
|
||||||
|
if (owner.element === actionElement || actionElement.contains(owner.element)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (selected.has(owner.element)) continue;
|
||||||
|
if (emittedHoverElements.has(owner.element)) continue;
|
||||||
|
if (now - owner.recordedAt > HOVER_SURFACE_CONTEXT_MAX_MS) continue;
|
||||||
|
selected.add(owner.element);
|
||||||
|
chain.push(owner);
|
||||||
|
}
|
||||||
|
return chain;
|
||||||
|
};
|
||||||
|
|
||||||
|
const containedHoverOwnerForAction = (
|
||||||
|
actionElement: Element,
|
||||||
|
now: number,
|
||||||
|
): HoverCandidate | undefined => {
|
||||||
|
for (let index = recentHoverCandidates.length - 1; index >= 0; index -= 1) {
|
||||||
|
const candidate = recentHoverCandidates[index];
|
||||||
|
if (!candidate) continue;
|
||||||
|
if (candidate.element === actionElement) continue;
|
||||||
|
if (!candidate.element.contains(actionElement)) continue;
|
||||||
|
if (now - candidate.recordedAt > HOVER_SURFACE_CONTEXT_MAX_MS) continue;
|
||||||
|
return candidate;
|
||||||
|
}
|
||||||
|
return undefined;
|
||||||
|
};
|
||||||
|
|
||||||
const emitClick = (event: MouseEvent) => {
|
const emitClick = (event: MouseEvent) => {
|
||||||
// Only record clicks an LLM can re-identify (named interactive controls).
|
// Only record clicks an LLM can re-identify (named interactive controls).
|
||||||
const target = describeEventTarget(eventTarget(event));
|
const target = describeEventTarget(eventTarget(event));
|
||||||
@@ -249,6 +685,54 @@ export function startRecordCapture(
|
|||||||
});
|
});
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const emitHoverCandidateBeforeAction = (actionTarget: EventTarget | null) => {
|
||||||
|
if (!(actionTarget instanceof Element)) return;
|
||||||
|
const actionElement = resolveClickableElement(actionTarget) ?? actionTarget;
|
||||||
|
const surface = ownedSurfaceForAction(actionElement);
|
||||||
|
const now = Date.now();
|
||||||
|
const containedOwner = surface ? undefined : containedHoverOwnerForAction(actionElement, now);
|
||||||
|
const hoverChain = surface
|
||||||
|
? surfaceOwnerChain(surface, actionElement, now)
|
||||||
|
: containedOwner
|
||||||
|
? [containedOwner]
|
||||||
|
: [];
|
||||||
|
if (hoverChain.length === 0) {
|
||||||
|
pendingHover = null;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
for (const hover of hoverChain) emitHoverStep(hover);
|
||||||
|
pendingHover = null;
|
||||||
|
};
|
||||||
|
|
||||||
|
const onMouseOver = (event: MouseEvent) => {
|
||||||
|
const target = eventTarget(event);
|
||||||
|
if (isOverlayTarget(target)) return;
|
||||||
|
const now = Date.now();
|
||||||
|
if (pendingHover && target instanceof Element && pendingHover.element.contains(target)) {
|
||||||
|
if (now - pendingHover.recordedAt <= HOVER_BEFORE_CLICK_MAX_MS) return;
|
||||||
|
pendingHover = null;
|
||||||
|
}
|
||||||
|
const candidate = hoverCandidateFromEvent(target);
|
||||||
|
if (candidate) {
|
||||||
|
const previousSurfaceStates = new Map(hoverSurfaceStates);
|
||||||
|
rememberHoverCandidate(candidate);
|
||||||
|
refreshHoverSurfaces(previousSurfaceStates);
|
||||||
|
scheduleHoverSurfaceRefresh(previousSurfaceStates);
|
||||||
|
}
|
||||||
|
const hover = candidate?.eligible ? candidate : null;
|
||||||
|
if (pendingHover && target instanceof Element) {
|
||||||
|
const relation = decideHoverSurfaceRelation({ triggerElement: pendingHover.element }, target);
|
||||||
|
if (relation.related && now - pendingHover.recordedAt <= HOVER_BEFORE_CLICK_MAX_MS) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!hover) return;
|
||||||
|
if (pendingHover && !shouldReplaceHoverCandidate(pendingHover, hover, now)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
pendingHover = hover;
|
||||||
|
};
|
||||||
|
|
||||||
const onClick = (event: MouseEvent) => {
|
const onClick = (event: MouseEvent) => {
|
||||||
if (event.button !== 0) return;
|
if (event.button !== 0) return;
|
||||||
const target = eventTarget(event);
|
const target = eventTarget(event);
|
||||||
@@ -278,6 +762,7 @@ export function startRecordCapture(
|
|||||||
|
|
||||||
const fillable = fillableFromTarget(target);
|
const fillable = fillableFromTarget(target);
|
||||||
if (fillable) {
|
if (fillable) {
|
||||||
|
emitHoverCandidateBeforeAction(fillable);
|
||||||
ensureFillSession(fillable);
|
ensureFillSession(fillable);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@@ -285,6 +770,7 @@ export function startRecordCapture(
|
|||||||
if (target instanceof Element) {
|
if (target instanceof Element) {
|
||||||
const nearbyFillable = nearbyFillableFromSearchChrome(target);
|
const nearbyFillable = nearbyFillableFromSearchChrome(target);
|
||||||
if (nearbyFillable) {
|
if (nearbyFillable) {
|
||||||
|
emitHoverCandidateBeforeAction(nearbyFillable);
|
||||||
ensureFillSession(nearbyFillable);
|
ensureFillSession(nearbyFillable);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@@ -304,6 +790,7 @@ export function startRecordCapture(
|
|||||||
|
|
||||||
commitFillSession();
|
commitFillSession();
|
||||||
if (target instanceof Element && target.closest("select")) return;
|
if (target instanceof Element && target.closest("select")) return;
|
||||||
|
emitHoverCandidateBeforeAction(target);
|
||||||
emitClick(event);
|
emitClick(event);
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -345,6 +832,7 @@ export function startRecordCapture(
|
|||||||
commitFillSession();
|
commitFillSession();
|
||||||
const target = eventTarget(event);
|
const target = eventTarget(event);
|
||||||
if (target instanceof HTMLSelectElement) {
|
if (target instanceof HTMLSelectElement) {
|
||||||
|
emitHoverCandidateBeforeAction(target);
|
||||||
const values = Array.from(target.selectedOptions).map((opt) => opt.value);
|
const values = Array.from(target.selectedOptions).map((opt) => opt.value);
|
||||||
const labels = Array.from(target.selectedOptions).map((opt) =>
|
const labels = Array.from(target.selectedOptions).map((opt) =>
|
||||||
(opt.label || opt.textContent || opt.value).trim(),
|
(opt.label || opt.textContent || opt.value).trim(),
|
||||||
@@ -398,6 +886,7 @@ export function startRecordCapture(
|
|||||||
}
|
}
|
||||||
const desc = describeEventTarget(target);
|
const desc = describeEventTarget(target);
|
||||||
if (!desc && !event.key) return;
|
if (!desc && !event.key) return;
|
||||||
|
emitHoverCandidateBeforeAction(target);
|
||||||
emitStep({
|
emitStep({
|
||||||
op: "press",
|
op: "press",
|
||||||
key: event.key,
|
key: event.key,
|
||||||
@@ -408,6 +897,7 @@ export function startRecordCapture(
|
|||||||
};
|
};
|
||||||
|
|
||||||
document.addEventListener("click", onClick, true);
|
document.addEventListener("click", onClick, true);
|
||||||
|
document.addEventListener("mouseover", onMouseOver, true);
|
||||||
document.addEventListener("focusin", onFocusIn, true);
|
document.addEventListener("focusin", onFocusIn, true);
|
||||||
document.addEventListener("focusout", onFocusOut, true);
|
document.addEventListener("focusout", onFocusOut, true);
|
||||||
document.addEventListener("input", onInput, true);
|
document.addEventListener("input", onInput, true);
|
||||||
@@ -438,6 +928,7 @@ export function startRecordCapture(
|
|||||||
dispose() {
|
dispose() {
|
||||||
commitFillSession();
|
commitFillSession();
|
||||||
document.removeEventListener("click", onClick, true);
|
document.removeEventListener("click", onClick, true);
|
||||||
|
document.removeEventListener("mouseover", onMouseOver, true);
|
||||||
document.removeEventListener("focusin", onFocusIn, true);
|
document.removeEventListener("focusin", onFocusIn, true);
|
||||||
document.removeEventListener("focusout", onFocusOut, true);
|
document.removeEventListener("focusout", onFocusOut, true);
|
||||||
document.removeEventListener("input", onInput, true);
|
document.removeEventListener("input", onInput, true);
|
||||||
|
|||||||
@@ -0,0 +1,237 @@
|
|||||||
|
export interface HoverSurfaceDecision {
|
||||||
|
related: boolean;
|
||||||
|
reason?: string;
|
||||||
|
surface?: Element;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface HoverSurfaceContext {
|
||||||
|
triggerElement: Element;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface HoverSurfaceState {
|
||||||
|
element: Element;
|
||||||
|
signature: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
const HOVER_SURFACE_SELECTOR = [
|
||||||
|
'[role="menu"]',
|
||||||
|
'[role="menubar"]',
|
||||||
|
'[role="listbox"]',
|
||||||
|
'[role="dialog"]',
|
||||||
|
'[role="tooltip"]',
|
||||||
|
'[aria-modal="true"]',
|
||||||
|
"[data-popper-placement]",
|
||||||
|
"[data-floating-ui-placement]",
|
||||||
|
"[data-headlessui-state]",
|
||||||
|
"[data-radix-popper-content-wrapper]",
|
||||||
|
].join(",");
|
||||||
|
|
||||||
|
const HOVER_SURFACE_WORD_RE =
|
||||||
|
/(^|[^a-z0-9])(dropdown-menu|drop-down-menu|menu-list|popover__(popper|content|content-body)|popover-content|popper|popup|flyout|submenu|submenu-wrapper|context-menu|account-menu|user-menu|avatar-menu)(?=[^a-z0-9]|$)/i;
|
||||||
|
const HOVER_SURFACE_ITEM_RE =
|
||||||
|
/(^|[^a-z0-9])((dropdown|menu|submenu|context-menu)(-|__?)?(item|link)|item-(text|label|content)|item__?(text|label|content))(?=[^a-z0-9]|$)/i;
|
||||||
|
const HOVER_SURFACE_MAX_AXIS_GAP = 96;
|
||||||
|
const HOVER_SURFACE_MAX_DIAGONAL_GAP = 16;
|
||||||
|
const HOVER_SURFACE_MAX_VIEWPORT_AREA_RATIO = 0.5;
|
||||||
|
|
||||||
|
function elementTokenText(el: Element): string {
|
||||||
|
return [
|
||||||
|
el.id,
|
||||||
|
typeof el.className === "string" ? el.className : "",
|
||||||
|
el.getAttribute("data-testid"),
|
||||||
|
el.getAttribute("data-test"),
|
||||||
|
el.getAttribute("data-cy"),
|
||||||
|
el.getAttribute("aria-label"),
|
||||||
|
el.getAttribute("role"),
|
||||||
|
]
|
||||||
|
.filter((value): value is string => typeof value === "string" && value.length > 0)
|
||||||
|
.join(" ");
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isRecognizedHoverSurfaceElement(el: Element): boolean {
|
||||||
|
if (el.matches(HOVER_SURFACE_SELECTOR)) return true;
|
||||||
|
const tokenText = elementTokenText(el);
|
||||||
|
if (HOVER_SURFACE_WORD_RE.test(tokenText) && !HOVER_SURFACE_ITEM_RE.test(tokenText)) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
function isVisibleSurfaceElement(el: Element): boolean {
|
||||||
|
if (!(el instanceof HTMLElement)) return false;
|
||||||
|
const style = getComputedStyle(el);
|
||||||
|
if (style.pointerEvents === "none" || style.visibility === "hidden" || style.display === "none") {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return rectFor(el) !== null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function closestRecognizedHoverSurface(element: Element): Element | null {
|
||||||
|
let node: Element | null = element;
|
||||||
|
let depth = 0;
|
||||||
|
while (node && node !== document.body && node !== document.documentElement && depth < 10) {
|
||||||
|
if (isRecognizedHoverSurfaceElement(node)) return node;
|
||||||
|
node = node.parentElement;
|
||||||
|
depth += 1;
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isPositionedFloatingElement(el: Element): boolean {
|
||||||
|
if (!(el instanceof HTMLElement)) return false;
|
||||||
|
const style = getComputedStyle(el);
|
||||||
|
if (style.position !== "absolute") return false;
|
||||||
|
if (style.pointerEvents === "none" || style.visibility === "hidden" || style.display === "none") {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
const rect = rectFor(el);
|
||||||
|
if (!rect) return false;
|
||||||
|
const viewportArea = window.innerWidth * window.innerHeight;
|
||||||
|
const area = rect.width * rect.height;
|
||||||
|
return (
|
||||||
|
area > 0 && (!viewportArea || area <= viewportArea * HOVER_SURFACE_MAX_VIEWPORT_AREA_RATIO)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isHoverSurfaceCandidateElement(el: Element): boolean {
|
||||||
|
return isRecognizedHoverSurfaceElement(el) || isPositionedFloatingElement(el);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function closestHoverSurfaceCandidate(element: Element): Element | null {
|
||||||
|
let node: Element | null = element;
|
||||||
|
let depth = 0;
|
||||||
|
while (node && node !== document.body && node !== document.documentElement && depth < 10) {
|
||||||
|
if (isHoverSurfaceCandidateElement(node)) return node;
|
||||||
|
node = node.parentElement;
|
||||||
|
depth += 1;
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function closestFloatingSurface(clickElement: Element): Element | null {
|
||||||
|
let node: Element | null = clickElement;
|
||||||
|
let depth = 0;
|
||||||
|
while (node && node !== document.body && node !== document.documentElement && depth < 10) {
|
||||||
|
if (isPositionedFloatingElement(node)) return node;
|
||||||
|
node = node.parentElement;
|
||||||
|
depth += 1;
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function hoverSurfaceSignature(el: Element): string {
|
||||||
|
const rect = rectFor(el);
|
||||||
|
return [
|
||||||
|
rect
|
||||||
|
? `${Math.round(rect.left)},${Math.round(rect.top)},${Math.round(rect.width)},${Math.round(rect.height)}`
|
||||||
|
: "",
|
||||||
|
(el.textContent ?? "").trim().slice(0, 512),
|
||||||
|
].join("|");
|
||||||
|
}
|
||||||
|
|
||||||
|
export function collectHoverSurfaceStates(): HoverSurfaceState[] {
|
||||||
|
const states: HoverSurfaceState[] = [];
|
||||||
|
for (const el of document.querySelectorAll("*")) {
|
||||||
|
if (isHoverSurfaceCandidateElement(el) && isVisibleSurfaceElement(el)) {
|
||||||
|
states.push({ element: el, signature: hoverSurfaceSignature(el) });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return states;
|
||||||
|
}
|
||||||
|
|
||||||
|
function rectFor(el: Element): DOMRect | null {
|
||||||
|
const rect = el.getBoundingClientRect();
|
||||||
|
if (rect.width <= 0 || rect.height <= 0) return null;
|
||||||
|
return rect;
|
||||||
|
}
|
||||||
|
|
||||||
|
function surfaceRelationMetrics(surface: Element, trigger: Element) {
|
||||||
|
const surfaceRect = rectFor(surface);
|
||||||
|
const triggerRect = rectFor(trigger);
|
||||||
|
if (!surfaceRect || !triggerRect) return null;
|
||||||
|
const horizontalOverlap =
|
||||||
|
Math.min(triggerRect.right, surfaceRect.right) - Math.max(triggerRect.left, surfaceRect.left);
|
||||||
|
const verticalOverlap =
|
||||||
|
Math.min(triggerRect.bottom, surfaceRect.bottom) - Math.max(triggerRect.top, surfaceRect.top);
|
||||||
|
const horizontalGap = Math.max(
|
||||||
|
0,
|
||||||
|
triggerRect.left - surfaceRect.right,
|
||||||
|
surfaceRect.left - triggerRect.right,
|
||||||
|
);
|
||||||
|
const verticalGap = Math.max(
|
||||||
|
0,
|
||||||
|
triggerRect.top - surfaceRect.bottom,
|
||||||
|
surfaceRect.top - triggerRect.bottom,
|
||||||
|
);
|
||||||
|
return {
|
||||||
|
surfaceRect,
|
||||||
|
triggerRect,
|
||||||
|
horizontalOverlap,
|
||||||
|
verticalOverlap,
|
||||||
|
horizontalGap,
|
||||||
|
verticalGap,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function isNearTrigger(surface: Element, trigger: Element): boolean {
|
||||||
|
const metrics = surfaceRelationMetrics(surface, trigger);
|
||||||
|
if (!metrics) return true;
|
||||||
|
const { horizontalOverlap, verticalOverlap, horizontalGap, verticalGap } = metrics;
|
||||||
|
if (horizontalOverlap > 0 && verticalGap <= HOVER_SURFACE_MAX_AXIS_GAP) return true;
|
||||||
|
if (verticalOverlap > 0 && horizontalGap <= HOVER_SURFACE_MAX_AXIS_GAP) return true;
|
||||||
|
return (
|
||||||
|
horizontalGap <= HOVER_SURFACE_MAX_DIAGONAL_GAP && verticalGap <= HOVER_SURFACE_MAX_DIAGONAL_GAP
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isLikelyHoverSurfaceOwner(trigger: Element, surface: Element): boolean {
|
||||||
|
const metrics = surfaceRelationMetrics(surface, trigger);
|
||||||
|
if (!metrics) return true;
|
||||||
|
const {
|
||||||
|
surfaceRect,
|
||||||
|
triggerRect,
|
||||||
|
horizontalOverlap,
|
||||||
|
verticalOverlap,
|
||||||
|
horizontalGap,
|
||||||
|
verticalGap,
|
||||||
|
} = metrics;
|
||||||
|
const opensVertically =
|
||||||
|
horizontalOverlap > 0 &&
|
||||||
|
verticalGap <= HOVER_SURFACE_MAX_AXIS_GAP &&
|
||||||
|
(triggerRect.bottom <= surfaceRect.top + 8 || surfaceRect.bottom <= triggerRect.top + 8);
|
||||||
|
const opensSideways =
|
||||||
|
verticalOverlap > 0 &&
|
||||||
|
horizontalGap <= HOVER_SURFACE_MAX_AXIS_GAP &&
|
||||||
|
(triggerRect.right <= surfaceRect.left + 8 || surfaceRect.right <= triggerRect.left + 8);
|
||||||
|
const opensFromCorner =
|
||||||
|
horizontalGap > 0 &&
|
||||||
|
verticalGap > 0 &&
|
||||||
|
horizontalGap <= HOVER_SURFACE_MAX_DIAGONAL_GAP &&
|
||||||
|
verticalGap <= HOVER_SURFACE_MAX_DIAGONAL_GAP;
|
||||||
|
return opensVertically || opensSideways || opensFromCorner;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function decideHoverSurfaceRelation(
|
||||||
|
context: HoverSurfaceContext,
|
||||||
|
clickElement: Element,
|
||||||
|
): HoverSurfaceDecision {
|
||||||
|
const clickSurface = closestRecognizedHoverSurface(clickElement);
|
||||||
|
if (clickSurface && isNearTrigger(clickSurface, context.triggerElement)) {
|
||||||
|
return { related: true, reason: "click-inside-hover-surface", surface: clickSurface };
|
||||||
|
}
|
||||||
|
|
||||||
|
const floatingSurface = closestFloatingSurface(clickElement);
|
||||||
|
if (floatingSurface && isNearTrigger(floatingSurface, context.triggerElement)) {
|
||||||
|
return {
|
||||||
|
related: true,
|
||||||
|
reason: "click-inside-near-floating-surface",
|
||||||
|
surface: floatingSurface,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
if (context.triggerElement.contains(clickElement) && clickElement !== context.triggerElement) {
|
||||||
|
return { related: true, reason: "click-inside-trigger-contained-surface" };
|
||||||
|
}
|
||||||
|
|
||||||
|
return { related: false, reason: "click-outside-hover-surface" };
|
||||||
|
}
|
||||||
@@ -22,6 +22,7 @@ import {
|
|||||||
} from "@/lib/overlay-bridge";
|
} from "@/lib/overlay-bridge";
|
||||||
import { POPUP_PORT_NAME, type PopupInbound, type PopupOutbound } from "@/lib/popup-bridge";
|
import { POPUP_PORT_NAME, type PopupInbound, type PopupOutbound } from "@/lib/popup-bridge";
|
||||||
import { attachSessionsLiveFlag } from "@/lib/sessions-live-flag";
|
import { attachSessionsLiveFlag } from "@/lib/sessions-live-flag";
|
||||||
|
import { createDisconnectCleanup } from "@/session-manager/disconnect-cleanup";
|
||||||
import { attachSessionEventHandler } from "@/session-manager/event-handler";
|
import { attachSessionEventHandler } from "@/session-manager/event-handler";
|
||||||
import { SessionManager } from "@/session-manager/manager";
|
import { SessionManager } from "@/session-manager/manager";
|
||||||
import {
|
import {
|
||||||
@@ -137,6 +138,13 @@ export default defineBackground(() => {
|
|||||||
// stale `true` does not keep waking us on every page load until the
|
// stale `true` does not keep waking us on every page load until the
|
||||||
// first mutation (review M4/M5 round 3 m-R3-1).
|
// first mutation (review M4/M5 round 3 m-R3-1).
|
||||||
void sessionsLive.refresh();
|
void sessionsLive.refresh();
|
||||||
|
const cleanupAfterDisconnect = createDisconnectCleanup({
|
||||||
|
manager: sessions,
|
||||||
|
sessionStopDeps: { cdp },
|
||||||
|
onSessionsChanged: () => {
|
||||||
|
void sessionsLive.syncFromManager();
|
||||||
|
},
|
||||||
|
});
|
||||||
const dispatcher = new ToolDispatcher({
|
const dispatcher = new ToolDispatcher({
|
||||||
transport,
|
transport,
|
||||||
sessions,
|
sessions,
|
||||||
@@ -267,7 +275,20 @@ export default defineBackground(() => {
|
|||||||
|
|
||||||
void (async () => {
|
void (async () => {
|
||||||
const connectionEnabled = await getConnectionEnabled();
|
const connectionEnabled = await getConnectionEnabled();
|
||||||
await controller.attach(transport, detectBrowserMeta(), connectionEnabled);
|
await controller.attach(transport, detectBrowserMeta(), connectionEnabled, {
|
||||||
|
beforeDisconnect: async () => {
|
||||||
|
const report = await cleanupAfterDisconnect();
|
||||||
|
if (report.failures.length > 0) {
|
||||||
|
console.warn("[browser-skill] session cleanup before disconnect was incomplete", report);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
onDisconnected: async () => {
|
||||||
|
const report = await cleanupAfterDisconnect();
|
||||||
|
if (report.failures.length > 0) {
|
||||||
|
console.warn("[browser-skill] session cleanup after disconnect was incomplete", report);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
});
|
||||||
})().catch((err) => {
|
})().catch((err) => {
|
||||||
console.error("[browser-skill] controller failed to attach", err);
|
console.error("[browser-skill] controller failed to attach", err);
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ import { BorrowConfirmationOverlay } from "@/content/BorrowConfirmationOverlay";
|
|||||||
import { ControlOverlay } from "@/content/ControlOverlay";
|
import { ControlOverlay } from "@/content/ControlOverlay";
|
||||||
import { createCaptureSuppressController } from "@/content/capture-suppress";
|
import { createCaptureSuppressController } from "@/content/capture-suppress";
|
||||||
import { HelpRequestOverlay } from "@/content/HelpRequestOverlay";
|
import { HelpRequestOverlay } from "@/content/HelpRequestOverlay";
|
||||||
|
import { createHelpRequestData } from "@/content/help-request";
|
||||||
import overlayCss from "@/content/overlay.css?inline";
|
import overlayCss from "@/content/overlay.css?inline";
|
||||||
import { OverlayController, shouldShowAgentControlOverlay } from "@/content/overlay-controller";
|
import { OverlayController, shouldShowAgentControlOverlay } from "@/content/overlay-controller";
|
||||||
import { RecordOverlay } from "@/content/RecordOverlay";
|
import { RecordOverlay } from "@/content/RecordOverlay";
|
||||||
@@ -32,6 +33,7 @@ import {
|
|||||||
isHelpCancelMessage,
|
isHelpCancelMessage,
|
||||||
isHelpRequestMessage,
|
isHelpRequestMessage,
|
||||||
} from "@/lib/help-bridge";
|
} from "@/lib/help-bridge";
|
||||||
|
import { getControlHintsHidden, STORAGE_KEYS } from "@/lib/instance-id";
|
||||||
import {
|
import {
|
||||||
isOverlayAgentOverlayResetMessage,
|
isOverlayAgentOverlayResetMessage,
|
||||||
isOverlayAgentStateMessage,
|
isOverlayAgentStateMessage,
|
||||||
@@ -76,6 +78,14 @@ export default defineContentScript({
|
|||||||
let hostLossReported = false;
|
let hostLossReported = false;
|
||||||
let remountInProgress = false;
|
let remountInProgress = false;
|
||||||
|
|
||||||
|
// Load the user's control-hints preference up front so an already-active
|
||||||
|
// Agent session does not flash the overlay before the stored value lands.
|
||||||
|
try {
|
||||||
|
overlays.setControlHintsHidden(await getControlHintsHidden());
|
||||||
|
} catch (err) {
|
||||||
|
console.debug("[bsk overlay] control-hints preference read failed", err);
|
||||||
|
}
|
||||||
|
|
||||||
const captureSuppress = createCaptureSuppressController(() => overlayHost);
|
const captureSuppress = createCaptureSuppressController(() => overlayHost);
|
||||||
|
|
||||||
const ui = await createShadowRootUi(ctx, {
|
const ui = await createShadowRootUi(ctx, {
|
||||||
@@ -335,16 +345,7 @@ export default defineContentScript({
|
|||||||
|
|
||||||
if (isHelpRequestMessage(message)) {
|
if (isHelpRequestMessage(message)) {
|
||||||
const helpMsg = message as HelpRequestMessage;
|
const helpMsg = message as HelpRequestMessage;
|
||||||
const previousHelp = overlays.setAgentHelpRequest({
|
const previousHelp = mountHelpRequest(helpMsg);
|
||||||
id: helpMsg.requestId,
|
|
||||||
prompt: helpMsg.prompt,
|
|
||||||
...(helpMsg.title ? { title: helpMsg.title } : {}),
|
|
||||||
...(helpMsg.displayMode ? { displayMode: helpMsg.displayMode } : {}),
|
|
||||||
selectors: helpMsg.selectors,
|
|
||||||
onContinue: (note: string) =>
|
|
||||||
void sendHelpFinish(helpMsg.requestId, "continued", note.trim() ? note : undefined),
|
|
||||||
onCancel: () => void sendHelpFinish(helpMsg.requestId, "cancelled"),
|
|
||||||
});
|
|
||||||
if (previousHelp && previousHelp.id !== helpMsg.requestId) {
|
if (previousHelp && previousHelp.id !== helpMsg.requestId) {
|
||||||
void sendHelpFinish(previousHelp.id, "cancelled");
|
void sendHelpFinish(previousHelp.id, "cancelled");
|
||||||
}
|
}
|
||||||
@@ -397,17 +398,18 @@ export default defineContentScript({
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
function mountHelpRequest(helpMsg: Omit<HelpRequestMessage, "type">): void {
|
function mountHelpRequest(helpMsg: Omit<HelpRequestMessage, "type">) {
|
||||||
overlays.setAgentHelpRequest({
|
return overlays.setAgentHelpRequest(
|
||||||
id: helpMsg.requestId,
|
createHelpRequestData(helpMsg, {
|
||||||
prompt: helpMsg.prompt,
|
finish: (requestId, outcome, note) => {
|
||||||
...(helpMsg.title ? { title: helpMsg.title } : {}),
|
void sendHelpFinish(requestId, outcome, note);
|
||||||
...(helpMsg.displayMode ? { displayMode: helpMsg.displayMode } : {}),
|
},
|
||||||
selectors: helpMsg.selectors,
|
query: () =>
|
||||||
onContinue: (note: string) =>
|
chrome.runtime.sendMessage({ type: HELP_QUERY }) as Promise<
|
||||||
void sendHelpFinish(helpMsg.requestId, "continued", note.trim() ? note : undefined),
|
HelpQueryResponse | undefined
|
||||||
onCancel: () => void sendHelpFinish(helpMsg.requestId, "cancelled"),
|
>,
|
||||||
});
|
}),
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
async function queryActiveHelpWithRetry(): Promise<void> {
|
async function queryActiveHelpWithRetry(): Promise<void> {
|
||||||
@@ -479,6 +481,19 @@ export default defineContentScript({
|
|||||||
if (event.persisted) void requestOverlayState();
|
if (event.persisted) void requestOverlayState();
|
||||||
};
|
};
|
||||||
|
|
||||||
|
// Live-apply popup toggles of the control-hints preference.
|
||||||
|
const onStorageChange = (
|
||||||
|
changes: Record<string, chrome.storage.StorageChange>,
|
||||||
|
areaName: string,
|
||||||
|
) => {
|
||||||
|
if (areaName !== "local") return;
|
||||||
|
const change = changes[STORAGE_KEYS.CONTROL_HINTS_HIDDEN];
|
||||||
|
if (!change) return;
|
||||||
|
overlays.setControlHintsHidden(change.newValue === true);
|
||||||
|
renderAll();
|
||||||
|
};
|
||||||
|
chrome.storage.onChanged.addListener(onStorageChange);
|
||||||
|
|
||||||
ui.mount();
|
ui.mount();
|
||||||
chrome.runtime.onMessage.addListener(onMessage);
|
chrome.runtime.onMessage.addListener(onMessage);
|
||||||
void requestOverlayState();
|
void requestOverlayState();
|
||||||
@@ -509,6 +524,7 @@ export default defineContentScript({
|
|||||||
ctx.onInvalidated(() => {
|
ctx.onInvalidated(() => {
|
||||||
hostObserver.disconnect();
|
hostObserver.disconnect();
|
||||||
chrome.runtime.onMessage.removeListener(onMessage);
|
chrome.runtime.onMessage.removeListener(onMessage);
|
||||||
|
chrome.storage.onChanged.removeListener(onStorageChange);
|
||||||
window.removeEventListener("pageshow", onPageShow);
|
window.removeEventListener("pageshow", onPageShow);
|
||||||
// Restore history hooks / remove capture listeners before the CS unloads.
|
// Restore history hooks / remove capture listeners before the CS unloads.
|
||||||
recordCapture?.dispose();
|
recordCapture?.dispose();
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
import { cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react";
|
import { cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react";
|
||||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||||
import type { SnapshotInfo } from "@/lib/connection-controller";
|
import type { SnapshotInfo } from "@/lib/connection-controller";
|
||||||
|
import { STORAGE_KEYS } from "@/lib/instance-id";
|
||||||
import { EXTENSION_VERSION } from "@/transport/handshake";
|
import { EXTENSION_VERSION } from "@/transport/handshake";
|
||||||
import { App } from "./App";
|
import { App } from "./App";
|
||||||
import { useConnectionState } from "./use-connection-state";
|
import { useConnectionState } from "./use-connection-state";
|
||||||
@@ -244,3 +245,94 @@ describe("App", () => {
|
|||||||
expect(screen.getByText("连接后可用")).toBeTruthy();
|
expect(screen.getByText("连接后可用")).toBeTruthy();
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe("control hints toggle", () => {
|
||||||
|
function stubChromeStorage(initial: Record<string, unknown> = {}) {
|
||||||
|
const store = { ...initial };
|
||||||
|
vi.stubGlobal("chrome", {
|
||||||
|
runtime: { lastError: undefined },
|
||||||
|
storage: {
|
||||||
|
local: {
|
||||||
|
get: (keys: string | string[], cb: (items: Record<string, unknown>) => void) => {
|
||||||
|
const items: Record<string, unknown> = {};
|
||||||
|
for (const k of Array.isArray(keys) ? keys : [keys]) {
|
||||||
|
if (k in store) items[k] = store[k];
|
||||||
|
}
|
||||||
|
cb(items);
|
||||||
|
},
|
||||||
|
set: (items: Record<string, unknown>, cb?: () => void) => {
|
||||||
|
Object.assign(store, items);
|
||||||
|
cb?.();
|
||||||
|
},
|
||||||
|
},
|
||||||
|
onChanged: {
|
||||||
|
addListener: vi.fn(),
|
||||||
|
removeListener: vi.fn(),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
});
|
||||||
|
return store;
|
||||||
|
}
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
cleanup();
|
||||||
|
vi.unstubAllGlobals();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("renders the control hints toggle on when no preference is stored", async () => {
|
||||||
|
stubChromeStorage();
|
||||||
|
|
||||||
|
render(<App />);
|
||||||
|
|
||||||
|
const toggle = await screen.findByRole("switch", { name: "控制提示" });
|
||||||
|
expect(toggle.getAttribute("aria-checked")).toBe("true");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("reflects the stored hidden preference", async () => {
|
||||||
|
stubChromeStorage({ [STORAGE_KEYS.CONTROL_HINTS_HIDDEN]: true });
|
||||||
|
|
||||||
|
render(<App />);
|
||||||
|
|
||||||
|
const toggle = await screen.findByRole("switch", { name: "控制提示" });
|
||||||
|
await waitFor(() => expect(toggle.getAttribute("aria-checked")).toBe("false"));
|
||||||
|
});
|
||||||
|
|
||||||
|
it("persists the hidden preference when the toggle is turned off", async () => {
|
||||||
|
const store = stubChromeStorage();
|
||||||
|
|
||||||
|
render(<App />);
|
||||||
|
|
||||||
|
const toggle = await screen.findByRole("switch", { name: "控制提示" });
|
||||||
|
fireEvent.click(toggle);
|
||||||
|
|
||||||
|
expect(store[STORAGE_KEYS.CONTROL_HINTS_HIDDEN]).toBe(true);
|
||||||
|
expect(toggle.getAttribute("aria-checked")).toBe("false");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps the hint copy in an accessible info tooltip", async () => {
|
||||||
|
stubChromeStorage();
|
||||||
|
|
||||||
|
render(<App />);
|
||||||
|
|
||||||
|
const info = await screen.findByRole("button", { name: "控制提示说明" });
|
||||||
|
expect(info).toBeTruthy();
|
||||||
|
const tooltip = screen.getByRole("tooltip");
|
||||||
|
expect(tooltip.textContent).toBe("Agent 控制页面时显示提示条和橙色闪光。");
|
||||||
|
// Hidden until the info button is hovered or focused.
|
||||||
|
expect(tooltip.className).toContain("opacity-0");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("uses the same switch component and size for both settings rows", async () => {
|
||||||
|
stubChromeStorage();
|
||||||
|
|
||||||
|
render(<App />);
|
||||||
|
|
||||||
|
const hintsToggle = await screen.findByRole("switch", { name: "控制提示" });
|
||||||
|
const connectionToggle = screen.getByRole("switch", { name: "BrowserSkill 连接" });
|
||||||
|
// One shared Switch component, one size — hierarchy comes from copy and
|
||||||
|
// the info icon, not control size. Both rows default to checked, so the
|
||||||
|
// class strings must be identical.
|
||||||
|
expect(hintsToggle.className).toContain("h-5 w-9");
|
||||||
|
expect(hintsToggle.className).toBe(connectionToggle.className);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -1,12 +1,20 @@
|
|||||||
import { useTranslation } from "@browser-skill/i18n/react";
|
import { useTranslation } from "@browser-skill/i18n/react";
|
||||||
import { Badge, Button, cn, Input, Label } from "@browser-skill/ui";
|
import { Badge, Button, Input, Label } from "@browser-skill/ui";
|
||||||
import { RiArrowLeftLine, RiArrowRightSLine, RiCheckLine, RiFileCopyLine } from "@remixicon/react";
|
import {
|
||||||
|
RiArrowLeftLine,
|
||||||
|
RiArrowRightSLine,
|
||||||
|
RiCheckLine,
|
||||||
|
RiFileCopyLine,
|
||||||
|
RiInformationLine,
|
||||||
|
} from "@remixicon/react";
|
||||||
import { type ChangeEvent, useEffect, useState } from "react";
|
import { type ChangeEvent, useEffect, useState } from "react";
|
||||||
import { PROTOCOL_VERSION } from "@/transport/handshake";
|
import { PROTOCOL_VERSION } from "@/transport/handshake";
|
||||||
import functionIconUrl from "../../../assets/function.svg";
|
import functionIconUrl from "../../../assets/function.svg";
|
||||||
import { ConnectionStatusIndicator } from "./connection-status-indicator";
|
import { ConnectionStatusIndicator } from "./connection-status-indicator";
|
||||||
import { POPUP_FEATURES, type PopupView } from "./features";
|
import { POPUP_FEATURES, type PopupView } from "./features";
|
||||||
|
import { Switch } from "./switch";
|
||||||
import { type PopupStatusState, useConnectionState } from "./use-connection-state";
|
import { type PopupStatusState, useConnectionState } from "./use-connection-state";
|
||||||
|
import { useControlHintsHidden } from "./use-control-hints-hidden";
|
||||||
|
|
||||||
const STATE_LABEL_KEYS = {
|
const STATE_LABEL_KEYS = {
|
||||||
disconnected: "popup.stateLabel.disconnected",
|
disconnected: "popup.stateLabel.disconnected",
|
||||||
@@ -32,6 +40,7 @@ function getLogoSrc() {
|
|||||||
export function App() {
|
export function App() {
|
||||||
const { t } = useTranslation("extension");
|
const { t } = useTranslation("extension");
|
||||||
const { snapshot, statusState, setConnectionEnabled } = useConnectionState();
|
const { snapshot, statusState, setConnectionEnabled } = useConnectionState();
|
||||||
|
const [controlHintsHidden, setControlHintsHidden] = useControlHintsHidden();
|
||||||
const [view, setView] = useState<PopupView>("main");
|
const [view, setView] = useState<PopupView>("main");
|
||||||
const [copiedInstanceId, setCopiedInstanceId] = useState(false);
|
const [copiedInstanceId, setCopiedInstanceId] = useState(false);
|
||||||
const [purposeDraft, setPurposeDraft] = useState("");
|
const [purposeDraft, setPurposeDraft] = useState("");
|
||||||
@@ -187,26 +196,12 @@ export function App() {
|
|||||||
>
|
>
|
||||||
{t(STATE_BADGE_KEYS[statusState])}
|
{t(STATE_BADGE_KEYS[statusState])}
|
||||||
</Badge>
|
</Badge>
|
||||||
<button
|
<Switch
|
||||||
type="button"
|
checked={snapshot.connectionEnabled}
|
||||||
role="switch"
|
onCheckedChange={setConnectionEnabled}
|
||||||
aria-checked={snapshot.connectionEnabled}
|
|
||||||
aria-label={t("popup.connectionToggleTitle")}
|
aria-label={t("popup.connectionToggleTitle")}
|
||||||
data-slot="popup-connection-toggle"
|
data-slot="popup-connection-toggle"
|
||||||
className={cn(
|
/>
|
||||||
"relative inline-flex h-5 w-9 shrink-0 cursor-pointer items-center rounded-full border border-transparent transition-colors focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring/50",
|
|
||||||
snapshot.connectionEnabled ? "bg-primary" : "bg-muted",
|
|
||||||
)}
|
|
||||||
onClick={() => setConnectionEnabled(!snapshot.connectionEnabled)}
|
|
||||||
>
|
|
||||||
<span
|
|
||||||
className={cn(
|
|
||||||
"pointer-events-none block size-4 rounded-full bg-background shadow-sm transition-transform",
|
|
||||||
snapshot.connectionEnabled ? "translate-x-4" : "translate-x-0.5",
|
|
||||||
)}
|
|
||||||
aria-hidden
|
|
||||||
/>
|
|
||||||
</button>
|
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
{isSkewed && (
|
{isSkewed && (
|
||||||
@@ -222,6 +217,41 @@ export function App() {
|
|||||||
)}
|
)}
|
||||||
</section>
|
</section>
|
||||||
|
|
||||||
|
<section
|
||||||
|
className="rounded-xl border border-border/80 bg-card/60 px-3 py-2.5"
|
||||||
|
data-slot="popup-control-hints-card"
|
||||||
|
>
|
||||||
|
<div className="flex items-center justify-between gap-2">
|
||||||
|
<span className="flex min-w-0 items-center gap-1">
|
||||||
|
<span className="truncate text-sm font-medium">
|
||||||
|
{t("popup.controlHintsToggleTitle")}
|
||||||
|
</span>
|
||||||
|
<span className="group relative inline-flex shrink-0">
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
aria-label={t("popup.controlHintsInfoLabel")}
|
||||||
|
data-slot="popup-control-hints-info"
|
||||||
|
className="flex size-4 items-center justify-center rounded-full text-muted-foreground/70 transition-colors hover:text-muted-foreground focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring/50"
|
||||||
|
>
|
||||||
|
<RiInformationLine className="size-3.5" aria-hidden />
|
||||||
|
</button>
|
||||||
|
<span
|
||||||
|
role="tooltip"
|
||||||
|
className="pointer-events-none absolute bottom-full left-0 z-10 mb-1.5 w-56 whitespace-normal rounded-md bg-foreground/65 px-2 py-1 text-[10px] font-medium leading-snug text-background opacity-0 shadow-md backdrop-blur-sm transition-opacity group-hover:opacity-100 group-focus-within:opacity-100"
|
||||||
|
>
|
||||||
|
{t("popup.controlHintsToggleHint")}
|
||||||
|
</span>
|
||||||
|
</span>
|
||||||
|
</span>
|
||||||
|
<Switch
|
||||||
|
checked={!controlHintsHidden}
|
||||||
|
onCheckedChange={(shown) => setControlHintsHidden(!shown)}
|
||||||
|
aria-label={t("popup.controlHintsToggleTitle")}
|
||||||
|
data-slot="popup-control-hints-toggle"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
|
||||||
{snapshot.lastError && (
|
{snapshot.lastError && (
|
||||||
<div
|
<div
|
||||||
className="rounded-lg border border-destructive/25 bg-destructive/10 px-3 py-2 text-xs leading-snug text-destructive"
|
className="rounded-lg border border-destructive/25 bg-destructive/10 px-3 py-2 text-xs leading-snug text-destructive"
|
||||||
|
|||||||
@@ -0,0 +1,29 @@
|
|||||||
|
import { cleanup, fireEvent, render, screen } from "@testing-library/react";
|
||||||
|
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||||
|
import { Switch } from "./switch";
|
||||||
|
|
||||||
|
describe("Switch", () => {
|
||||||
|
afterEach(() => {
|
||||||
|
cleanup();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("renders switch semantics with the checked state", () => {
|
||||||
|
render(<Switch checked={true} onCheckedChange={vi.fn()} aria-label="开关" />);
|
||||||
|
|
||||||
|
const toggle = screen.getByRole("switch", { name: "开关" });
|
||||||
|
expect(toggle.getAttribute("aria-checked")).toBe("true");
|
||||||
|
expect(toggle.className).toContain("h-5 w-9");
|
||||||
|
expect(toggle.className).toContain("bg-primary");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("calls onCheckedChange with the negated state when clicked", () => {
|
||||||
|
const onCheckedChange = vi.fn();
|
||||||
|
render(<Switch checked={false} onCheckedChange={onCheckedChange} aria-label="开关" />);
|
||||||
|
|
||||||
|
const toggle = screen.getByRole("switch", { name: "开关" });
|
||||||
|
expect(toggle.className).toContain("bg-muted");
|
||||||
|
|
||||||
|
fireEvent.click(toggle);
|
||||||
|
expect(onCheckedChange).toHaveBeenCalledWith(true);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,43 @@
|
|||||||
|
import { cn } from "@browser-skill/ui";
|
||||||
|
|
||||||
|
export interface SwitchProps {
|
||||||
|
checked: boolean;
|
||||||
|
onCheckedChange: (checked: boolean) => void;
|
||||||
|
"aria-label": string;
|
||||||
|
"data-slot"?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The popup's single toggle switch — every settings row uses this same
|
||||||
|
* component and size; hierarchy between primary and secondary settings is
|
||||||
|
* conveyed by copy/iconography, not by control size.
|
||||||
|
*/
|
||||||
|
export function Switch({
|
||||||
|
checked,
|
||||||
|
onCheckedChange,
|
||||||
|
"aria-label": ariaLabel,
|
||||||
|
"data-slot": dataSlot,
|
||||||
|
}: SwitchProps) {
|
||||||
|
return (
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
role="switch"
|
||||||
|
aria-checked={checked}
|
||||||
|
aria-label={ariaLabel}
|
||||||
|
data-slot={dataSlot}
|
||||||
|
className={cn(
|
||||||
|
"relative inline-flex h-5 w-9 shrink-0 cursor-pointer items-center rounded-full border border-transparent transition-colors focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring/50",
|
||||||
|
checked ? "bg-primary" : "bg-muted",
|
||||||
|
)}
|
||||||
|
onClick={() => onCheckedChange(!checked)}
|
||||||
|
>
|
||||||
|
<span
|
||||||
|
className={cn(
|
||||||
|
"pointer-events-none block size-4 rounded-full bg-background shadow-sm transition-transform",
|
||||||
|
checked ? "translate-x-4" : "translate-x-0.5",
|
||||||
|
)}
|
||||||
|
aria-hidden
|
||||||
|
/>
|
||||||
|
</button>
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -0,0 +1,44 @@
|
|||||||
|
import { useEffect, useState } from "react";
|
||||||
|
import { getControlHintsHidden, STORAGE_KEYS, setControlHintsHidden } from "@/lib/instance-id";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Popup-side view of the "hide control hints" preference in
|
||||||
|
* `chrome.storage.local`. Defaults to shown when storage is unavailable
|
||||||
|
* (e.g. unit tests) so the popup still renders; writes go straight to
|
||||||
|
* storage and content scripts pick them up via `storage.onChanged`.
|
||||||
|
*/
|
||||||
|
export function useControlHintsHidden(): [boolean, (hidden: boolean) => void] {
|
||||||
|
const [hidden, setHidden] = useState(false);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (typeof chrome === "undefined" || !chrome.storage?.local) return undefined;
|
||||||
|
let cancelled = false;
|
||||||
|
getControlHintsHidden()
|
||||||
|
.then((value) => {
|
||||||
|
if (!cancelled) setHidden(value);
|
||||||
|
})
|
||||||
|
.catch((err) => {
|
||||||
|
console.debug("[browser-skill] control-hints preference read failed", err);
|
||||||
|
});
|
||||||
|
const onChanged = (changes: Record<string, chrome.storage.StorageChange>, areaName: string) => {
|
||||||
|
if (areaName !== "local") return;
|
||||||
|
const change = changes[STORAGE_KEYS.CONTROL_HINTS_HIDDEN];
|
||||||
|
if (change) setHidden(change.newValue === true);
|
||||||
|
};
|
||||||
|
chrome.storage.onChanged.addListener(onChanged);
|
||||||
|
return () => {
|
||||||
|
cancelled = true;
|
||||||
|
chrome.storage.onChanged.removeListener(onChanged);
|
||||||
|
};
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
const update = (value: boolean) => {
|
||||||
|
setHidden(value);
|
||||||
|
if (typeof chrome === "undefined" || !chrome.storage?.local) return;
|
||||||
|
setControlHintsHidden(value).catch((err) => {
|
||||||
|
console.debug("[browser-skill] control-hints preference write failed", err);
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
return [hidden, update];
|
||||||
|
}
|
||||||
@@ -1,7 +1,7 @@
|
|||||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||||
import { MIN_COMPATIBLE_PROTOCOL } from "../../transport/handshake";
|
import { MIN_COMPATIBLE_PROTOCOL } from "../../transport/handshake";
|
||||||
import type { ConnectionStateHandler, Transport } from "../../transport/transport";
|
import type { ConnectionStateHandler, FrameHandler, Transport } from "../../transport/transport";
|
||||||
import type { ConnectionState, HandshakeResult } from "../../transport/types";
|
import type { ConnectionState, HandshakeResult, ProtocolFrame } from "../../transport/types";
|
||||||
import { __testing__, ConnectionController } from "../connection-controller";
|
import { __testing__, ConnectionController } from "../connection-controller";
|
||||||
|
|
||||||
vi.mock("../instance-id", () => ({
|
vi.mock("../instance-id", () => ({
|
||||||
@@ -97,6 +97,7 @@ describe("computeConnectedState (protocol-based compat)", () => {
|
|||||||
function makeMockTransport(initialState: ConnectionState = "disconnected") {
|
function makeMockTransport(initialState: ConnectionState = "disconnected") {
|
||||||
let state = initialState;
|
let state = initialState;
|
||||||
const stateHandlers = new Set<ConnectionStateHandler>();
|
const stateHandlers = new Set<ConnectionStateHandler>();
|
||||||
|
const messageHandlers = new Set<FrameHandler>();
|
||||||
const transport = {
|
const transport = {
|
||||||
get state() {
|
get state() {
|
||||||
return state;
|
return state;
|
||||||
@@ -110,7 +111,10 @@ function makeMockTransport(initialState: ConnectionState = "disconnected") {
|
|||||||
for (const h of stateHandlers) h("disconnected");
|
for (const h of stateHandlers) h("disconnected");
|
||||||
}),
|
}),
|
||||||
send: vi.fn(),
|
send: vi.fn(),
|
||||||
onMessage: vi.fn(() => ({ dispose: () => {} })),
|
onMessage: vi.fn((handler: FrameHandler) => {
|
||||||
|
messageHandlers.add(handler);
|
||||||
|
return { dispose: () => messageHandlers.delete(handler) };
|
||||||
|
}),
|
||||||
onConnectionStateChange: vi.fn((handler: ConnectionStateHandler) => {
|
onConnectionStateChange: vi.fn((handler: ConnectionStateHandler) => {
|
||||||
stateHandlers.add(handler);
|
stateHandlers.add(handler);
|
||||||
return { dispose: () => stateHandlers.delete(handler) };
|
return { dispose: () => stateHandlers.delete(handler) };
|
||||||
@@ -119,6 +123,9 @@ function makeMockTransport(initialState: ConnectionState = "disconnected") {
|
|||||||
state = next;
|
state = next;
|
||||||
for (const h of stateHandlers) h(next);
|
for (const h of stateHandlers) h(next);
|
||||||
},
|
},
|
||||||
|
emitMessage(frame: ProtocolFrame) {
|
||||||
|
for (const handler of messageHandlers) handler(frame);
|
||||||
|
},
|
||||||
};
|
};
|
||||||
return transport as typeof transport & Transport;
|
return transport as typeof transport & Transport;
|
||||||
}
|
}
|
||||||
@@ -128,6 +135,10 @@ describe("ConnectionController connectionEnabled", () => {
|
|||||||
vi.clearAllMocks();
|
vi.clearAllMocks();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
vi.useRealTimers();
|
||||||
|
});
|
||||||
|
|
||||||
it("does not connect on attach when connection is disabled", async () => {
|
it("does not connect on attach when connection is disabled", async () => {
|
||||||
const controller = new ConnectionController();
|
const controller = new ConnectionController();
|
||||||
const transport = makeMockTransport();
|
const transport = makeMockTransport();
|
||||||
@@ -152,6 +163,36 @@ describe("ConnectionController connectionEnabled", () => {
|
|||||||
expect(controller.snapshot().lastError).toBeNull();
|
expect(controller.snapshot().lastError).toBeNull();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("runs safe session cleanup before an intentional disconnect", async () => {
|
||||||
|
const controller = new ConnectionController();
|
||||||
|
const transport = makeMockTransport();
|
||||||
|
const order: string[] = [];
|
||||||
|
transport.disconnect.mockImplementation(async () => {
|
||||||
|
order.push("disconnect");
|
||||||
|
});
|
||||||
|
await controller.attach(transport, { name: "Chrome", version: "120" }, true, {
|
||||||
|
beforeDisconnect: async () => {
|
||||||
|
order.push("cleanup");
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
await controller.setConnectionEnabled(false);
|
||||||
|
|
||||||
|
expect(order).toEqual(["cleanup", "disconnect"]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("runs session cleanup when the transport disconnects unexpectedly", async () => {
|
||||||
|
const controller = new ConnectionController();
|
||||||
|
const transport = makeMockTransport();
|
||||||
|
const onDisconnected = vi.fn(async () => {});
|
||||||
|
await controller.attach(transport, { name: "Chrome", version: "120" }, true, {
|
||||||
|
onDisconnected,
|
||||||
|
});
|
||||||
|
|
||||||
|
transport.emitState("disconnected");
|
||||||
|
await vi.waitFor(() => expect(onDisconnected).toHaveBeenCalledTimes(1));
|
||||||
|
});
|
||||||
|
|
||||||
it("reconnects when connection is re-enabled", async () => {
|
it("reconnects when connection is re-enabled", async () => {
|
||||||
const controller = new ConnectionController();
|
const controller = new ConnectionController();
|
||||||
const transport = makeMockTransport();
|
const transport = makeMockTransport();
|
||||||
@@ -171,4 +212,41 @@ describe("ConnectionController connectionEnabled", () => {
|
|||||||
|
|
||||||
expect(controller.snapshot().state).toBe("disconnected");
|
expect(controller.snapshot().state).toBe("disconnected");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("disconnects and retries when handshake fails while the socket is still open", async () => {
|
||||||
|
vi.useFakeTimers();
|
||||||
|
const controller = new ConnectionController();
|
||||||
|
const transport = makeMockTransport();
|
||||||
|
await controller.attach(transport, { name: "Chrome", version: "120" }, true);
|
||||||
|
const request = transport.send.mock.calls[0]?.[0] as { id: string };
|
||||||
|
|
||||||
|
transport.emitMessage({
|
||||||
|
id: request.id,
|
||||||
|
error: { code: "protocol_error", message: "bad handshake" },
|
||||||
|
});
|
||||||
|
await vi.waitFor(() => expect(transport.disconnect).toHaveBeenCalledTimes(1));
|
||||||
|
expect(controller.snapshot().state).toBe("disconnected");
|
||||||
|
|
||||||
|
await vi.advanceTimersByTimeAsync(1_000);
|
||||||
|
expect(transport.connect).toHaveBeenCalledTimes(2);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("binds each handshake to the connection that initiated it", async () => {
|
||||||
|
const controller = new ConnectionController();
|
||||||
|
const transport = makeMockTransport();
|
||||||
|
await controller.attach(transport, { name: "Chrome", version: "120" }, true);
|
||||||
|
const first = transport.send.mock.calls[0]?.[0] as { id: string };
|
||||||
|
|
||||||
|
transport.emitState("disconnected");
|
||||||
|
transport.emitState("connected");
|
||||||
|
const second = transport.send.mock.calls[1]?.[0] as { id: string };
|
||||||
|
expect(second.id).not.toBe(first.id);
|
||||||
|
|
||||||
|
transport.emitMessage({ id: first.id, result: handshake("1.0", "1.0") });
|
||||||
|
await Promise.resolve();
|
||||||
|
expect(controller.snapshot().state).not.toBe("connected");
|
||||||
|
|
||||||
|
transport.emitMessage({ id: second.id, result: handshake("1.0", "1.0") });
|
||||||
|
await vi.waitFor(() => expect(controller.snapshot().state).toBe("connected"));
|
||||||
|
});
|
||||||
});
|
});
|
||||||
@@ -0,0 +1,101 @@
|
|||||||
|
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||||
|
import type { CdpRunner } from "@/tools/shared";
|
||||||
|
import { waitForDocumentSettled } from "../recording/document-settle";
|
||||||
|
|
||||||
|
function quietCdp(): { cdp: CdpRunner; send: ReturnType<typeof vi.fn> } {
|
||||||
|
const send = vi.fn(async () => ({
|
||||||
|
result: { value: { idleMs: 1_000, readyState: "complete" } },
|
||||||
|
}));
|
||||||
|
return {
|
||||||
|
cdp: { send: send as unknown as CdpRunner["send"] },
|
||||||
|
send,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("waitForDocumentSettled", () => {
|
||||||
|
afterEach(() => {
|
||||||
|
vi.useRealTimers();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("waits through the minimum observation floor before accepting a quiet page", async () => {
|
||||||
|
vi.useFakeTimers();
|
||||||
|
const { cdp, send } = quietCdp();
|
||||||
|
|
||||||
|
const settled = waitForDocumentSettled(cdp, { target: { tabId: 7 } });
|
||||||
|
await vi.advanceTimersByTimeAsync(180);
|
||||||
|
|
||||||
|
await expect(settled).resolves.toBe("quiet");
|
||||||
|
expect(send).toHaveBeenCalledTimes(3);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("cancels before probing a superseded observation", async () => {
|
||||||
|
vi.useFakeTimers();
|
||||||
|
const { cdp, send } = quietCdp();
|
||||||
|
|
||||||
|
const controller = new AbortController();
|
||||||
|
const settled = waitForDocumentSettled(
|
||||||
|
cdp,
|
||||||
|
{ target: { tabId: 7 } },
|
||||||
|
{
|
||||||
|
signal: controller.signal,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
controller.abort();
|
||||||
|
await vi.advanceTimersByTimeAsync(60);
|
||||||
|
|
||||||
|
await expect(settled).resolves.toBe("cancelled");
|
||||||
|
expect(send).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("probes a same-process iframe through its isolated execution context", async () => {
|
||||||
|
vi.useFakeTimers();
|
||||||
|
const send = vi.fn(async (_tabId: number, method: string) => {
|
||||||
|
if (method === "Page.createIsolatedWorld") return { executionContextId: 91 };
|
||||||
|
return { result: { value: { idleMs: 1_000, readyState: "complete" } } };
|
||||||
|
});
|
||||||
|
const cdp: CdpRunner = { send: send as unknown as CdpRunner["send"] };
|
||||||
|
|
||||||
|
const settled = waitForDocumentSettled(cdp, {
|
||||||
|
target: { tabId: 7 },
|
||||||
|
frameId: "child-frame",
|
||||||
|
});
|
||||||
|
await vi.advanceTimersByTimeAsync(180);
|
||||||
|
|
||||||
|
await expect(settled).resolves.toBe("quiet");
|
||||||
|
expect(send).toHaveBeenCalledWith(
|
||||||
|
7,
|
||||||
|
"Page.createIsolatedWorld",
|
||||||
|
expect.objectContaining({ frameId: "child-frame" }),
|
||||||
|
);
|
||||||
|
expect(send).toHaveBeenCalledWith(
|
||||||
|
7,
|
||||||
|
"Runtime.evaluate",
|
||||||
|
expect.objectContaining({ contextId: 91 }),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("routes an OOPIF probe through its CDP target session", async () => {
|
||||||
|
vi.useFakeTimers();
|
||||||
|
const send = vi.fn();
|
||||||
|
const sendToTarget = vi.fn(async (_target, method: string) => {
|
||||||
|
if (method === "Page.createIsolatedWorld") return { executionContextId: 27 };
|
||||||
|
return { result: { value: { idleMs: 1_000, readyState: "complete" } } };
|
||||||
|
});
|
||||||
|
const cdp: CdpRunner = {
|
||||||
|
send: send as unknown as CdpRunner["send"],
|
||||||
|
sendToTarget: sendToTarget as unknown as NonNullable<CdpRunner["sendToTarget"]>,
|
||||||
|
};
|
||||||
|
|
||||||
|
const target = { tabId: 7, sessionId: "oopif-session" };
|
||||||
|
const settled = waitForDocumentSettled(cdp, { target, frameId: "oopif-frame" });
|
||||||
|
await vi.advanceTimersByTimeAsync(180);
|
||||||
|
|
||||||
|
await expect(settled).resolves.toBe("quiet");
|
||||||
|
expect(send).not.toHaveBeenCalled();
|
||||||
|
expect(sendToTarget).toHaveBeenCalledWith(
|
||||||
|
target,
|
||||||
|
"Runtime.evaluate",
|
||||||
|
expect.objectContaining({ contextId: 27 }),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -32,6 +32,7 @@ describe("help-bridge", () => {
|
|||||||
prompt: "log in",
|
prompt: "log in",
|
||||||
displayMode: "compact",
|
displayMode: "compact",
|
||||||
selectors: ["#login"],
|
selectors: ["#login"],
|
||||||
|
rects: [{ top: 10, left: 20, width: 100, height: 40 }],
|
||||||
timeoutMs: 1000,
|
timeoutMs: 1000,
|
||||||
}),
|
}),
|
||||||
).toBe(true);
|
).toBe(true);
|
||||||
|
|||||||
@@ -0,0 +1,125 @@
|
|||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import { evaluateHoverTrigger } from "../hover-trigger-policy";
|
||||||
|
|
||||||
|
describe("evaluateHoverTrigger", () => {
|
||||||
|
it("accepts explicit popup triggers", () => {
|
||||||
|
const decision = evaluateHoverTrigger({
|
||||||
|
tag: "button",
|
||||||
|
role: "button",
|
||||||
|
label: "Open user navigation menu",
|
||||||
|
attrs: { "aria-haspopup": "menu", "aria-expanded": "false" },
|
||||||
|
rect: { x: 900, y: 8, w: 32, h: 32 },
|
||||||
|
cursor: "pointer",
|
||||||
|
pointerEvents: "auto",
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(decision.eligible).toBe(true);
|
||||||
|
expect(decision.reasons).toEqual(
|
||||||
|
expect.arrayContaining(["aria-haspopup", "collapsed", "popup-signal"]),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("accepts compact topbar image buttons without popup attributes", () => {
|
||||||
|
const decision = evaluateHoverTrigger({
|
||||||
|
tag: "button",
|
||||||
|
role: "button",
|
||||||
|
label: "image",
|
||||||
|
attrs: {},
|
||||||
|
rect: { x: 900, y: 8, w: 32, h: 32 },
|
||||||
|
pointerEvents: "auto",
|
||||||
|
hasGraphicDescendant: true,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(decision.eligible).toBe(true);
|
||||||
|
expect(decision.reasons).toEqual(expect.arrayContaining(["icon-only", "topbar", "compact"]));
|
||||||
|
});
|
||||||
|
|
||||||
|
it("accepts compact topbar image links without popup attributes", () => {
|
||||||
|
const decision = evaluateHoverTrigger({
|
||||||
|
tag: "a",
|
||||||
|
role: "link",
|
||||||
|
label: "image",
|
||||||
|
attrs: {},
|
||||||
|
rect: { x: 900, y: 8, w: 32, h: 32 },
|
||||||
|
pointerEvents: "auto",
|
||||||
|
hasGraphicDescendant: true,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(decision.eligible).toBe(true);
|
||||||
|
expect(decision.reasons).toEqual(expect.arrayContaining(["icon-only", "topbar", "compact"]));
|
||||||
|
});
|
||||||
|
|
||||||
|
it("rejects topbar menu links without surface trigger evidence", () => {
|
||||||
|
const decision = evaluateHoverTrigger({
|
||||||
|
tag: "a",
|
||||||
|
role: "link",
|
||||||
|
label: "My profile",
|
||||||
|
attrs: {},
|
||||||
|
rect: { x: 820, y: 72, w: 96, h: 32 },
|
||||||
|
cursor: "pointer",
|
||||||
|
pointerEvents: "auto",
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(decision.eligible).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("rejects text navigation items even when their attributes contain popup words", () => {
|
||||||
|
const decision = evaluateHoverTrigger({
|
||||||
|
tag: "a",
|
||||||
|
role: "link",
|
||||||
|
label: "My profile",
|
||||||
|
attrs: { class: "dropdown-item profile-link" },
|
||||||
|
rect: { x: 820, y: 72, w: 96, h: 32 },
|
||||||
|
cursor: "pointer",
|
||||||
|
pointerEvents: "auto",
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(decision.eligible).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("rejects labelled dropdown list items as hover triggers", () => {
|
||||||
|
const decision = evaluateHoverTrigger({
|
||||||
|
tag: "li",
|
||||||
|
label: "文档C+D",
|
||||||
|
attrs: { class: "dropdown-item", tabindex: "0" },
|
||||||
|
rect: { x: 820, y: 48, w: 160, h: 32 },
|
||||||
|
cursor: "pointer",
|
||||||
|
pointerEvents: "auto",
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(decision.eligible).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("rejects ordinary controls without hover popup signals", () => {
|
||||||
|
const decision = evaluateHoverTrigger({
|
||||||
|
tag: "button",
|
||||||
|
role: "button",
|
||||||
|
label: "Search",
|
||||||
|
attrs: {},
|
||||||
|
rect: { x: 100, y: 240, w: 120, h: 36 },
|
||||||
|
pointerEvents: "auto",
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(decision.eligible).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("rejects unsafe or non-visible triggers", () => {
|
||||||
|
expect(
|
||||||
|
evaluateHoverTrigger({
|
||||||
|
tag: "input",
|
||||||
|
attrs: {},
|
||||||
|
rect: { x: 10, y: 10, w: 80, h: 24 },
|
||||||
|
pointerEvents: "auto",
|
||||||
|
}).eligible,
|
||||||
|
).toBe(false);
|
||||||
|
expect(
|
||||||
|
evaluateHoverTrigger({
|
||||||
|
tag: "button",
|
||||||
|
role: "button",
|
||||||
|
attrs: { hidden: "" },
|
||||||
|
rect: { x: 10, y: 10, w: 80, h: 24 },
|
||||||
|
pointerEvents: "auto",
|
||||||
|
}).eligible,
|
||||||
|
).toBe(false);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -1,10 +1,12 @@
|
|||||||
import { describe, expect, it, vi } from "vitest";
|
import { describe, expect, it, vi } from "vitest";
|
||||||
import {
|
import {
|
||||||
getConnectionEnabled,
|
getConnectionEnabled,
|
||||||
|
getControlHintsHidden,
|
||||||
getLabel,
|
getLabel,
|
||||||
getOrCreateInstanceId,
|
getOrCreateInstanceId,
|
||||||
STORAGE_KEYS,
|
STORAGE_KEYS,
|
||||||
setConnectionEnabled,
|
setConnectionEnabled,
|
||||||
|
setControlHintsHidden,
|
||||||
setLabel,
|
setLabel,
|
||||||
} from "../instance-id";
|
} from "../instance-id";
|
||||||
|
|
||||||
@@ -93,4 +95,26 @@ describe("instance-id", () => {
|
|||||||
expect(store[STORAGE_KEYS.CONNECTION_ENABLED]).toBe(false);
|
expect(store[STORAGE_KEYS.CONNECTION_ENABLED]).toBe(false);
|
||||||
expect(await getConnectionEnabled(backend)).toBe(false);
|
expect(await getConnectionEnabled(backend)).toBe(false);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("getControlHintsHidden returns false when storage is empty", async () => {
|
||||||
|
const { backend } = fakeStorage();
|
||||||
|
expect(await getControlHintsHidden(backend)).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("getControlHintsHidden returns persisted boolean values", async () => {
|
||||||
|
const { backend } = fakeStorage({ [STORAGE_KEYS.CONTROL_HINTS_HIDDEN]: true });
|
||||||
|
expect(await getControlHintsHidden(backend)).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("getControlHintsHidden treats non-boolean stored values as shown", async () => {
|
||||||
|
const { backend } = fakeStorage({ [STORAGE_KEYS.CONTROL_HINTS_HIDDEN]: "true" });
|
||||||
|
expect(await getControlHintsHidden(backend)).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("setControlHintsHidden persists the value retrievable by getControlHintsHidden", async () => {
|
||||||
|
const { backend, store } = fakeStorage();
|
||||||
|
await setControlHintsHidden(true, backend);
|
||||||
|
expect(store[STORAGE_KEYS.CONTROL_HINTS_HIDDEN]).toBe(true);
|
||||||
|
expect(await getControlHintsHidden(backend)).toBe(true);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
@@ -0,0 +1,141 @@
|
|||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import { ObservationNodeIndex } from "../recording/observation-capture";
|
||||||
|
import { RecordingObservationSession } from "../recording/observation-session";
|
||||||
|
import { RecordingStateRegistry } from "../recording/state-registry";
|
||||||
|
import { buildTraceV3 } from "../recording/trace-builder-v3";
|
||||||
|
import type { RecordingDraftStep } from "../recording/types";
|
||||||
|
|
||||||
|
const URL = "https://example.com/login";
|
||||||
|
|
||||||
|
function sessionWithInput(redactValues = false): RecordingObservationSession {
|
||||||
|
const session = new RecordingObservationSession({ redactValues });
|
||||||
|
const state = session.registry.register({
|
||||||
|
url: URL,
|
||||||
|
rawVomText: '@vom 1\ntextbox "Password" value="••••••" [ref=e1]',
|
||||||
|
});
|
||||||
|
session.cursor.lastSettled = {
|
||||||
|
stateId: state.id,
|
||||||
|
rootFrameId: "root",
|
||||||
|
index: new ObservationNodeIndex({
|
||||||
|
rootFrameId: "root",
|
||||||
|
matchNodes: [
|
||||||
|
{
|
||||||
|
frameId: "root",
|
||||||
|
backendNodeId: 42,
|
||||||
|
tag: "input",
|
||||||
|
rect: { x: 20, y: 40, w: 200, h: 30 },
|
||||||
|
localRect: { x: 20, y: 40, w: 200, h: 30 },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
refs: [{ ref: "e1", backendNodeId: 42, role: "textbox", name: "Password", line: 1 }],
|
||||||
|
}),
|
||||||
|
url: URL,
|
||||||
|
};
|
||||||
|
return session;
|
||||||
|
}
|
||||||
|
|
||||||
|
function finalizedFillBody(value: string, redactValues: boolean): string {
|
||||||
|
const session = sessionWithInput(redactValues);
|
||||||
|
const draft: RecordingDraftStep = {
|
||||||
|
op: "fill",
|
||||||
|
captureTarget: { tag: "input", role: "textbox", name: "Password" },
|
||||||
|
value,
|
||||||
|
targetHint: {
|
||||||
|
geometry: { rect: { x: 20, y: 40, w: 200, h: 30 }, tag: "input" },
|
||||||
|
},
|
||||||
|
};
|
||||||
|
session.bindDraft(draft, 1);
|
||||||
|
draft.postStateId = draft.preStateId;
|
||||||
|
return buildTraceV3({
|
||||||
|
registry: session.registry,
|
||||||
|
drafts: [draft],
|
||||||
|
annotations: session.annotations,
|
||||||
|
startedAt: "2026-08-12T00:00:00.000Z",
|
||||||
|
stoppedBy: "user_finish",
|
||||||
|
bskVersion: "test",
|
||||||
|
}).states[0]!.body;
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("record observation annotations", () => {
|
||||||
|
it("omits a fill literal when values are redacted", () => {
|
||||||
|
const secret = "hunter2-private";
|
||||||
|
const body = finalizedFillBody(secret, true);
|
||||||
|
expect(body).toContain("step 1: fill");
|
||||||
|
expect(body).not.toContain(secret);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps ordinary fill details", () => {
|
||||||
|
expect(finalizedFillBody("ordinary text", false)).toContain('step 1: fill: "ordinary text"');
|
||||||
|
});
|
||||||
|
|
||||||
|
it("encodes title and URL so line breaks cannot corrupt state metadata", () => {
|
||||||
|
const registry = new RecordingStateRegistry();
|
||||||
|
const state = registry.register({
|
||||||
|
url: "https://example.com/a\nb",
|
||||||
|
title: "hello\nworld",
|
||||||
|
rawVomText: "@vom 1",
|
||||||
|
});
|
||||||
|
const trace = buildTraceV3({
|
||||||
|
registry,
|
||||||
|
drafts: [{ op: "scroll", preStateId: state.id, postStateId: state.id }],
|
||||||
|
startedAt: "2026-08-12T00:00:00.000Z",
|
||||||
|
stoppedBy: "user_finish",
|
||||||
|
bskVersion: "test",
|
||||||
|
});
|
||||||
|
expect(trace.states[0]?.body).toContain('url: "https://example.com/a\\nb"');
|
||||||
|
expect(trace.states[0]?.body).toContain('title: "hello\\nworld"');
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("recording state ownership", () => {
|
||||||
|
it("deduplicates within one recording and isolates ids between recordings", () => {
|
||||||
|
const first = new RecordingStateRegistry();
|
||||||
|
const second = new RecordingStateRegistry();
|
||||||
|
expect(first.register({ url: URL, rawVomText: "same" }).id).toBe("s1");
|
||||||
|
expect(first.register({ url: URL, rawVomText: "same" }).id).toBe("s1");
|
||||||
|
expect(second.register({ url: URL, rawVomText: "other" }).id).toBe("s1");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("enriches metadata when a deduplicated observation becomes more complete", () => {
|
||||||
|
const registry = new RecordingStateRegistry();
|
||||||
|
registry.register({ url: URL, rawVomText: "same" });
|
||||||
|
const state = registry.register({
|
||||||
|
url: URL,
|
||||||
|
title: "Login",
|
||||||
|
rawVomText: "same",
|
||||||
|
truncated: true,
|
||||||
|
});
|
||||||
|
expect(state).toMatchObject({ id: "s1", title: "Login", truncated: true });
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("draft binding", () => {
|
||||||
|
it("keeps capture semantics when no observation exists", () => {
|
||||||
|
const session = new RecordingObservationSession();
|
||||||
|
const draft: RecordingDraftStep = {
|
||||||
|
op: "hover",
|
||||||
|
captureTarget: { tag: "button", role: "button", name: "新建" },
|
||||||
|
};
|
||||||
|
session.bindDraft(draft, 1);
|
||||||
|
expect(draft.matchedTarget).toEqual({ role: "button", name: "新建", unmatched: true });
|
||||||
|
expect(draft.preStateId).toBeUndefined();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not bind an unmatched new action to a stale observation", () => {
|
||||||
|
const session = sessionWithInput();
|
||||||
|
const draft: RecordingDraftStep = {
|
||||||
|
op: "click",
|
||||||
|
captureTarget: { tag: "button", role: "button", name: "Confirm" },
|
||||||
|
targetHint: {
|
||||||
|
geometry: { rect: { x: 400, y: 300, w: 80, h: 30 }, tag: "button" },
|
||||||
|
},
|
||||||
|
};
|
||||||
|
session.bindDraft(draft, 2, true);
|
||||||
|
expect(draft.preStateId).toBeUndefined();
|
||||||
|
expect("matchedTarget" in draft ? draft.matchedTarget : undefined).toEqual({
|
||||||
|
role: "button",
|
||||||
|
name: "Confirm",
|
||||||
|
unmatched: true,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,136 @@
|
|||||||
|
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||||
|
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
|
||||||
|
import { ObservationNodeIndex, type RegisteredObservation } from "../recording/observation-capture";
|
||||||
|
import { RecordingObservationSession } from "../recording/observation-session";
|
||||||
|
import { SettleController } from "../recording/settle-controller";
|
||||||
|
import type { RecordingDraftStep } from "../recording/types";
|
||||||
|
|
||||||
|
const OBSERVATION: RegisteredObservation = {
|
||||||
|
stateId: "s-next",
|
||||||
|
rootFrameId: "root",
|
||||||
|
index: new ObservationNodeIndex({ rootFrameId: "root", matchNodes: [], refs: [] }),
|
||||||
|
url: "https://example.com/next",
|
||||||
|
};
|
||||||
|
|
||||||
|
describe("SettleController", () => {
|
||||||
|
afterEach(() => vi.useRealTimers());
|
||||||
|
|
||||||
|
it("aborts superseded capture work and lands the prior action on the next origin", async () => {
|
||||||
|
vi.useFakeTimers();
|
||||||
|
const cdp: CdpRunner = {
|
||||||
|
send: vi.fn(async () => ({
|
||||||
|
result: { value: { idleMs: 1_000, readyState: "complete" } },
|
||||||
|
})) as unknown as CdpRunner["send"],
|
||||||
|
};
|
||||||
|
const tabsApi: ChromeTabsApi = {
|
||||||
|
get: vi.fn(async () => ({ id: 7, url: OBSERVATION.url }) as chrome.tabs.Tab),
|
||||||
|
query: vi.fn(async () => []),
|
||||||
|
};
|
||||||
|
const session = new RecordingObservationSession();
|
||||||
|
let firstSignal: AbortSignal | undefined;
|
||||||
|
vi.spyOn(session, "capture")
|
||||||
|
.mockImplementationOnce(async (_cdp, _tabs, _tabId, signal) => {
|
||||||
|
firstSignal = signal;
|
||||||
|
return new Promise<RegisteredObservation>((_resolve, reject) => {
|
||||||
|
signal?.addEventListener(
|
||||||
|
"abort",
|
||||||
|
() => reject(new DOMException("observation aborted", "AbortError")),
|
||||||
|
{ once: true },
|
||||||
|
);
|
||||||
|
});
|
||||||
|
})
|
||||||
|
.mockResolvedValueOnce(OBSERVATION);
|
||||||
|
|
||||||
|
const drafts: RecordingDraftStep[] = [
|
||||||
|
{ op: "click", captureTarget: { tag: "button", name: "First" }, preStateId: "s1" },
|
||||||
|
];
|
||||||
|
const controller = new SettleController({ session, cdp, tabsApi, tabId: 7 });
|
||||||
|
controller.schedule(drafts, 0);
|
||||||
|
await vi.advanceTimersByTimeAsync(180);
|
||||||
|
|
||||||
|
drafts.push({
|
||||||
|
op: "click",
|
||||||
|
captureTarget: { tag: "button", name: "Second" },
|
||||||
|
preStateId: "s-next",
|
||||||
|
});
|
||||||
|
controller.schedule(drafts, 1);
|
||||||
|
expect(firstSignal?.aborted).toBe(true);
|
||||||
|
expect(drafts[0]?.postStateId).toBe("s-next");
|
||||||
|
|
||||||
|
await vi.advanceTimersByTimeAsync(500);
|
||||||
|
await controller.flush();
|
||||||
|
expect(drafts[1]?.postStateId).toBe("s-next");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not consume a newer redirect while reading the prior landing URL", async () => {
|
||||||
|
vi.useFakeTimers();
|
||||||
|
const cdp: CdpRunner = {
|
||||||
|
send: vi.fn(async () => ({
|
||||||
|
result: { value: { idleMs: 1_000, readyState: "complete" } },
|
||||||
|
})) as unknown as CdpRunner["send"],
|
||||||
|
};
|
||||||
|
let resolveFirstTab!: (tab: chrome.tabs.Tab) => void;
|
||||||
|
const firstTab = new Promise<chrome.tabs.Tab>((resolve) => {
|
||||||
|
resolveFirstTab = resolve;
|
||||||
|
});
|
||||||
|
const tabsApi: ChromeTabsApi = {
|
||||||
|
get: vi
|
||||||
|
.fn()
|
||||||
|
.mockImplementationOnce(() => firstTab)
|
||||||
|
.mockResolvedValue({ id: 7, url: OBSERVATION.url }),
|
||||||
|
query: vi.fn(async () => []),
|
||||||
|
};
|
||||||
|
const session = new RecordingObservationSession();
|
||||||
|
session.cursor.lastSettled = OBSERVATION;
|
||||||
|
const drafts: RecordingDraftStep[] = [];
|
||||||
|
const controller = new SettleController({ session, cdp, tabsApi, tabId: 7 });
|
||||||
|
|
||||||
|
controller.scheduleRedirect(drafts, "https://example.com/intermediate");
|
||||||
|
await vi.advanceTimersByTimeAsync(180);
|
||||||
|
expect(tabsApi.get).toHaveBeenCalledTimes(1);
|
||||||
|
|
||||||
|
controller.scheduleRedirect(drafts, OBSERVATION.url);
|
||||||
|
resolveFirstTab({ id: 7, url: "https://example.com/intermediate" } as chrome.tabs.Tab);
|
||||||
|
await vi.advanceTimersByTimeAsync(500);
|
||||||
|
await controller.flushRedirects();
|
||||||
|
|
||||||
|
expect(drafts).toEqual([]);
|
||||||
|
expect(controller.hasPending).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("settles an action in its own OOPIF document scope", async () => {
|
||||||
|
vi.useFakeTimers();
|
||||||
|
const send = vi.fn();
|
||||||
|
const sendToTarget = vi.fn(async (_target, method: string) => {
|
||||||
|
if (method === "Page.createIsolatedWorld") return { executionContextId: 39 };
|
||||||
|
return { result: { value: { idleMs: 1_000, readyState: "complete" } } };
|
||||||
|
});
|
||||||
|
const cdp: CdpRunner = {
|
||||||
|
send: send as unknown as CdpRunner["send"],
|
||||||
|
sendToTarget: sendToTarget as unknown as NonNullable<CdpRunner["sendToTarget"]>,
|
||||||
|
};
|
||||||
|
const tabsApi: ChromeTabsApi = {
|
||||||
|
get: vi.fn(async () => ({ id: 7, url: OBSERVATION.url }) as chrome.tabs.Tab),
|
||||||
|
query: vi.fn(async () => []),
|
||||||
|
};
|
||||||
|
const session = new RecordingObservationSession();
|
||||||
|
vi.spyOn(session, "capture").mockResolvedValue(OBSERVATION);
|
||||||
|
const drafts: RecordingDraftStep[] = [
|
||||||
|
{ op: "click", captureTarget: { tag: "button", name: "Inside frame" } },
|
||||||
|
];
|
||||||
|
const controller = new SettleController({ session, cdp, tabsApi, tabId: 7 });
|
||||||
|
const target = { tabId: 7, sessionId: "oopif-session" };
|
||||||
|
|
||||||
|
controller.schedule(drafts, 0, { target, frameId: "oopif-frame" });
|
||||||
|
await vi.advanceTimersByTimeAsync(180);
|
||||||
|
await controller.flush();
|
||||||
|
|
||||||
|
expect(send).not.toHaveBeenCalled();
|
||||||
|
expect(sendToTarget).toHaveBeenCalledWith(
|
||||||
|
target,
|
||||||
|
"Runtime.evaluate",
|
||||||
|
expect.objectContaining({ contextId: 39 }),
|
||||||
|
);
|
||||||
|
expect(drafts[0]?.postStateId).toBe("s-next");
|
||||||
|
});
|
||||||
|
});
|
||||||
+8
-8
@@ -1,5 +1,5 @@
|
|||||||
import { describe, expect, it } from "vitest";
|
import { describe, expect, it } from "vitest";
|
||||||
import { appendRecordedPayload, observeRecordedNavigation } from "../recording-step-buffer";
|
import { appendRecordedPayload, observeRecordedNavigation } from "../recording/step-buffer";
|
||||||
|
|
||||||
describe("recording-step-buffer", () => {
|
describe("recording-step-buffer", () => {
|
||||||
it("stores semantic click without summary", () => {
|
it("stores semantic click without summary", () => {
|
||||||
@@ -12,7 +12,7 @@ describe("recording-step-buffer", () => {
|
|||||||
expect(buffer.steps).toEqual([
|
expect(buffer.steps).toEqual([
|
||||||
{
|
{
|
||||||
op: "click",
|
op: "click",
|
||||||
target: { tag: "button", role: "button", name: "发布" },
|
captureTarget: { tag: "button", role: "button", name: "发布" },
|
||||||
},
|
},
|
||||||
]);
|
]);
|
||||||
expect(buffer.pendingNavigation).toBe(true);
|
expect(buffer.pendingNavigation).toBe(true);
|
||||||
@@ -23,7 +23,7 @@ describe("recording-step-buffer", () => {
|
|||||||
steps: [
|
steps: [
|
||||||
{
|
{
|
||||||
op: "click" as const,
|
op: "click" as const,
|
||||||
target: { tag: "button", role: "button", name: "发布" },
|
captureTarget: { tag: "button", role: "button", name: "发布" },
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
currentUrl: "https://example.com/a",
|
currentUrl: "https://example.com/a",
|
||||||
@@ -34,8 +34,8 @@ describe("recording-step-buffer", () => {
|
|||||||
expect(buffer.steps).toEqual([
|
expect(buffer.steps).toEqual([
|
||||||
{
|
{
|
||||||
op: "click",
|
op: "click",
|
||||||
target: { tag: "button", role: "button", name: "发布" },
|
captureTarget: { tag: "button", role: "button", name: "发布" },
|
||||||
navigated_to: "https://example.com/b",
|
navigatedTo: "https://example.com/b",
|
||||||
},
|
},
|
||||||
]);
|
]);
|
||||||
expect(JSON.stringify(buffer.steps)).not.toContain("wait_for_navigation");
|
expect(JSON.stringify(buffer.steps)).not.toContain("wait_for_navigation");
|
||||||
@@ -46,7 +46,7 @@ describe("recording-step-buffer", () => {
|
|||||||
steps: [
|
steps: [
|
||||||
{
|
{
|
||||||
op: "select" as const,
|
op: "select" as const,
|
||||||
target: { tag: "select", role: "combobox", name: "分类" },
|
captureTarget: { tag: "select", role: "combobox", name: "分类" },
|
||||||
values: ["tech"],
|
values: ["tech"],
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
@@ -58,9 +58,9 @@ describe("recording-step-buffer", () => {
|
|||||||
expect(buffer.steps).toEqual([
|
expect(buffer.steps).toEqual([
|
||||||
{
|
{
|
||||||
op: "select",
|
op: "select",
|
||||||
target: { tag: "select", role: "combobox", name: "分类" },
|
captureTarget: { tag: "select", role: "combobox", name: "分类" },
|
||||||
values: ["tech"],
|
values: ["tech"],
|
||||||
navigated_to: "https://example.com/list?cat=tech",
|
navigatedTo: "https://example.com/list?cat=tech",
|
||||||
},
|
},
|
||||||
]);
|
]);
|
||||||
});
|
});
|
||||||
@@ -0,0 +1,127 @@
|
|||||||
|
import type { RenderedRef } from "@browser-skill/vom";
|
||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import type { CaptureVomMatchNode } from "@/tools/capture-vom-observation";
|
||||||
|
import { ObservationNodeIndex, type RegisteredObservation } from "../recording/observation-capture";
|
||||||
|
import { matchObservationTarget } from "../recording/target-matcher";
|
||||||
|
|
||||||
|
function node(backendNodeId: number, frameId: string, x = 10): CaptureVomMatchNode {
|
||||||
|
return {
|
||||||
|
backendNodeId,
|
||||||
|
frameId,
|
||||||
|
tag: "button",
|
||||||
|
rect: { x, y: 20, w: 100, h: 30 },
|
||||||
|
localRect: { x, y: 20, w: 100, h: 30 },
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function observation(
|
||||||
|
matchNodes: CaptureVomMatchNode[],
|
||||||
|
refs: RenderedRef[],
|
||||||
|
): RegisteredObservation {
|
||||||
|
return {
|
||||||
|
stateId: "s1",
|
||||||
|
rootFrameId: "root",
|
||||||
|
index: new ObservationNodeIndex({ rootFrameId: "root", matchNodes, refs }),
|
||||||
|
url: "https://example.com",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("matchObservationTarget", () => {
|
||||||
|
it("indexes the safe geometry DTO without discarding frame-local geometry", () => {
|
||||||
|
const geometry = node(42, "child");
|
||||||
|
geometry.localRect = { x: 5, y: 6, w: 100, h: 30 };
|
||||||
|
const ref: RenderedRef = {
|
||||||
|
ref: "e1",
|
||||||
|
backendNodeId: 42,
|
||||||
|
frameId: "child",
|
||||||
|
line: 1,
|
||||||
|
};
|
||||||
|
const candidate = observation([geometry], [ref]).index.candidates("child", "button")[0];
|
||||||
|
|
||||||
|
expect(candidate).toEqual({ frameId: "child", geometry, ref });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("matches a unique node using canonical top-level viewport geometry", () => {
|
||||||
|
const target = matchObservationTarget({
|
||||||
|
observation: observation(
|
||||||
|
[node(42, "root")],
|
||||||
|
[{ ref: "e1", backendNodeId: 42, role: "button", name: "发布", line: 1 }],
|
||||||
|
),
|
||||||
|
hint: { geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" } },
|
||||||
|
});
|
||||||
|
expect(target).toEqual({ ref: "e1", role: "button", name: "发布" });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("uses frame id with backend node id so sibling frames cannot collide", () => {
|
||||||
|
const target = matchObservationTarget({
|
||||||
|
observation: observation(
|
||||||
|
[node(42, "left"), node(42, "right")],
|
||||||
|
[
|
||||||
|
{ ref: "e1", backendNodeId: 42, frameId: "left", line: 1 },
|
||||||
|
{ ref: "e2", backendNodeId: 42, frameId: "right", line: 2 },
|
||||||
|
],
|
||||||
|
),
|
||||||
|
hint: {
|
||||||
|
frameId: "right",
|
||||||
|
geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" },
|
||||||
|
},
|
||||||
|
});
|
||||||
|
expect(target.ref).toBe("e2");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("restricts a missing frame hint to the root frame", () => {
|
||||||
|
const target = matchObservationTarget({
|
||||||
|
observation: observation(
|
||||||
|
[node(42, "child")],
|
||||||
|
[{ ref: "e1", backendNodeId: 42, frameId: "child", line: 1 }],
|
||||||
|
),
|
||||||
|
hint: { geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" } },
|
||||||
|
fallback: { tag: "button", name: "发布" },
|
||||||
|
});
|
||||||
|
expect(target).toEqual({ name: "发布", unmatched: true });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("returns unmatched for ambiguous geometry", () => {
|
||||||
|
const target = matchObservationTarget({
|
||||||
|
observation: observation(
|
||||||
|
[node(42, "root"), node(43, "root")],
|
||||||
|
[
|
||||||
|
{ ref: "e1", backendNodeId: 42, line: 1 },
|
||||||
|
{ ref: "e2", backendNodeId: 43, line: 2 },
|
||||||
|
],
|
||||||
|
),
|
||||||
|
hint: { geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" } },
|
||||||
|
});
|
||||||
|
expect(target.unmatched).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("uses semantics when geometry is unavailable without crossing frame boundaries", () => {
|
||||||
|
const target = matchObservationTarget({
|
||||||
|
observation: observation(
|
||||||
|
[node(41, "root"), node(42, "child")],
|
||||||
|
[
|
||||||
|
{ ref: "e1", backendNodeId: 41, role: "button", name: "保存", line: 1 },
|
||||||
|
{ ref: "e2", backendNodeId: 42, frameId: "child", role: "button", name: "保存", line: 2 },
|
||||||
|
],
|
||||||
|
),
|
||||||
|
hint: { frameId: "child" },
|
||||||
|
fallback: { tag: "button", role: "button", name: "保存" },
|
||||||
|
});
|
||||||
|
expect(target.ref).toBe("e2");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("uses semantics to disambiguate equal geometry in one frame", () => {
|
||||||
|
const target = matchObservationTarget({
|
||||||
|
observation: observation(
|
||||||
|
[node(42, "root"), node(43, "root")],
|
||||||
|
[
|
||||||
|
{ ref: "e1", backendNodeId: 42, role: "button", name: "保存", line: 1 },
|
||||||
|
{ ref: "e2", backendNodeId: 43, role: "button", name: "取消", line: 2 },
|
||||||
|
],
|
||||||
|
),
|
||||||
|
hint: { geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" } },
|
||||||
|
fallback: { tag: "button", role: "button", name: "取消" },
|
||||||
|
});
|
||||||
|
expect(target.ref).toBe("e2");
|
||||||
|
});
|
||||||
|
});
|
||||||
+76
-21
@@ -1,6 +1,15 @@
|
|||||||
import { describe, expect, it } from "vitest";
|
import { describe, expect, it } from "vitest";
|
||||||
import type { DraftTraceStep } from "@/transport/types";
|
import type { RecordingDraftStep } from "@/lib/recording/types";
|
||||||
import { reduceTraceSteps, resolveTraceStartUrl, shouldRecordPress } from "../trace-reducer";
|
import { shouldRecordPress } from "../recording/draft-policy";
|
||||||
|
import { buildTraceV2 } from "../recording/trace-reducer-v2";
|
||||||
|
|
||||||
|
function reduceTraceSteps(steps: RecordingDraftStep[], startUrl?: string) {
|
||||||
|
return buildTraceV2({
|
||||||
|
steps,
|
||||||
|
startedAt: "2026-01-01T00:00:00.000Z",
|
||||||
|
...(startUrl ? { startUrl } : {}),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
describe("shouldRecordPress", () => {
|
describe("shouldRecordPress", () => {
|
||||||
it("keeps Enter and Escape", () => {
|
it("keeps Enter and Escape", () => {
|
||||||
@@ -19,35 +28,35 @@ describe("shouldRecordPress", () => {
|
|||||||
|
|
||||||
describe("reduceTraceSteps", () => {
|
describe("reduceTraceSteps", () => {
|
||||||
it("builds steps with pages dictionary and page id references", () => {
|
it("builds steps with pages dictionary and page id references", () => {
|
||||||
const drafts: DraftTraceStep[] = [
|
const drafts: RecordingDraftStep[] = [
|
||||||
{
|
{
|
||||||
op: "navigate",
|
op: "navigate",
|
||||||
url: "https://example.com/search?q=hello&utm_source=x",
|
url: "https://example.com/search?q=hello&utm_source=x",
|
||||||
page_url: "https://example.com/search?q=hello&utm_source=x",
|
pageUrl: "https://example.com/search?q=hello&utm_source=x",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
op: "fill",
|
op: "fill",
|
||||||
target: { tag: "input", role: "textbox", name: "搜索", name_attr: "q" },
|
captureTarget: { tag: "input", role: "textbox", name: "搜索", name_attr: "q" },
|
||||||
value: "browser skill",
|
value: "browser skill",
|
||||||
page_url: "https://example.com/search?q=hello&utm_source=x",
|
pageUrl: "https://example.com/search?q=hello&utm_source=x",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
op: "press",
|
op: "press",
|
||||||
key: "Enter",
|
key: "Enter",
|
||||||
target: { tag: "input", role: "textbox", name: "搜索", name_attr: "q" },
|
captureTarget: { tag: "input", role: "textbox", name: "搜索", name_attr: "q" },
|
||||||
navigated_to: "https://example.com/results/42",
|
navigatedTo: "https://example.com/results/42",
|
||||||
page_url: "https://example.com/search?q=hello&utm_source=x",
|
pageUrl: "https://example.com/search?q=hello&utm_source=x",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
op: "click",
|
op: "click",
|
||||||
target: { tag: "button", role: "button", name: "发布" },
|
captureTarget: { tag: "button", role: "button", name: "发布" },
|
||||||
navigated_to: "https://example.com/p/99",
|
navigatedTo: "https://example.com/p/99",
|
||||||
page_url: "https://example.com/results/42",
|
pageUrl: "https://example.com/results/42",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
op: "press",
|
op: "press",
|
||||||
key: "a",
|
key: "a",
|
||||||
page_url: "https://example.com/p/99",
|
pageUrl: "https://example.com/p/99",
|
||||||
},
|
},
|
||||||
];
|
];
|
||||||
|
|
||||||
@@ -104,16 +113,37 @@ describe("reduceTraceSteps", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
it("maps select navigated_to onto effect.navigated_to (page id)", () => {
|
it("keeps committed empty fill steps", () => {
|
||||||
|
const { steps } = reduceTraceSteps(
|
||||||
|
[
|
||||||
|
{
|
||||||
|
op: "fill",
|
||||||
|
captureTarget: { tag: "input", role: "textbox", name: "Search query" },
|
||||||
|
value: "",
|
||||||
|
pageUrl: "https://example.com/search",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
"https://example.com/search",
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(steps).toEqual([
|
||||||
|
expect.objectContaining({
|
||||||
|
op: "fill",
|
||||||
|
value: "",
|
||||||
|
}),
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("maps select navigatedTo onto effect.navigatedTo (page id)", () => {
|
||||||
const { pages, steps } = reduceTraceSteps(
|
const { pages, steps } = reduceTraceSteps(
|
||||||
[
|
[
|
||||||
{
|
{
|
||||||
op: "select",
|
op: "select",
|
||||||
target: { tag: "select", role: "combobox", name: "分类" },
|
captureTarget: { tag: "select", role: "combobox", name: "分类" },
|
||||||
values: ["tech"],
|
values: ["tech"],
|
||||||
labels: ["技术"],
|
labels: ["技术"],
|
||||||
navigated_to: "https://example.com/list?cat=tech",
|
navigatedTo: "https://example.com/list?cat=tech",
|
||||||
page_url: "https://example.com/list",
|
pageUrl: "https://example.com/list",
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
"https://example.com/list",
|
"https://example.com/list",
|
||||||
@@ -130,12 +160,37 @@ describe("reduceTraceSteps", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("keeps hover steps before menu clicks", () => {
|
||||||
|
const { steps } = reduceTraceSteps(
|
||||||
|
[
|
||||||
|
{
|
||||||
|
op: "hover",
|
||||||
|
captureTarget: { tag: "span", role: "button", name: "Account" },
|
||||||
|
pageUrl: "https://example.com/app",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
op: "click",
|
||||||
|
captureTarget: { tag: "a", role: "link", name: "Profile" },
|
||||||
|
pageUrl: "https://example.com/app",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
"https://example.com/app",
|
||||||
|
);
|
||||||
|
expect(steps.map((s) => s.op)).toEqual(["hover", "click"]);
|
||||||
|
expect(steps[0]).toMatchObject({
|
||||||
|
op: "hover",
|
||||||
|
page: "p1",
|
||||||
|
target: { name: "Account" },
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
it("resolveTraceStartUrl prefers explicit start URL", () => {
|
it("resolveTraceStartUrl prefers explicit start URL", () => {
|
||||||
expect(
|
expect(
|
||||||
resolveTraceStartUrl(
|
buildTraceV2({
|
||||||
[{ op: "navigate", url: "https://example.com/other" }],
|
steps: [{ op: "navigate", url: "https://example.com/other" }],
|
||||||
"https://example.com/start",
|
startedAt: "2026-01-01T00:00:00.000Z",
|
||||||
),
|
startUrl: "https://example.com/start",
|
||||||
|
}).entry.start_url,
|
||||||
).toBe("https://example.com/start");
|
).toBe("https://example.com/start");
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
@@ -0,0 +1,62 @@
|
|||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import { TRACE_VERSION_V3, VOM_FORMAT_VERSION } from "@/transport/types";
|
||||||
|
import { RecordingStateRegistry } from "../recording/state-registry";
|
||||||
|
import { buildTraceV3 } from "../recording/trace-builder-v3";
|
||||||
|
import { reduceTraceStepsV3 } from "../recording/trace-reducer-v3";
|
||||||
|
import type { RecordingDraftStep } from "../recording/types";
|
||||||
|
|
||||||
|
describe("trace reducer v3", () => {
|
||||||
|
it("collapses redirect hops while retaining draft-to-step identity", () => {
|
||||||
|
const drafts: RecordingDraftStep[] = [
|
||||||
|
{ op: "navigate", url: "https://example.com/start", preStateId: "s1", postStateId: "s2" },
|
||||||
|
{
|
||||||
|
op: "navigate",
|
||||||
|
url: "https://example.com/final",
|
||||||
|
transitionQualifiers: ["server_redirect"],
|
||||||
|
preStateId: "s2",
|
||||||
|
postStateId: "s3",
|
||||||
|
},
|
||||||
|
];
|
||||||
|
const before = structuredClone(drafts);
|
||||||
|
const reduced = reduceTraceStepsV3(drafts);
|
||||||
|
|
||||||
|
expect(drafts).toEqual(before);
|
||||||
|
expect(reduced.steps).toEqual([
|
||||||
|
expect.objectContaining({
|
||||||
|
id: 1,
|
||||||
|
op: "navigate",
|
||||||
|
state: "s1",
|
||||||
|
to: "https://example.com/final",
|
||||||
|
result: { state: "s3" },
|
||||||
|
}),
|
||||||
|
]);
|
||||||
|
expect(reduced.stepIdByDraftId.get(1)).toBe(1);
|
||||||
|
expect(reduced.stepIdByDraftId.get(2)).toBe(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("builds the wire model from protocol constants", () => {
|
||||||
|
const registry = new RecordingStateRegistry();
|
||||||
|
const state = registry.register({ url: "https://example.com", rawVomText: "@vom 1" });
|
||||||
|
const trace = buildTraceV3({
|
||||||
|
registry,
|
||||||
|
drafts: [
|
||||||
|
{
|
||||||
|
op: "click",
|
||||||
|
captureTarget: { tag: "button", role: "button", name: "Save" },
|
||||||
|
preStateId: state.id,
|
||||||
|
postStateId: state.id,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
startedAt: "2026-08-12T00:00:00.000Z",
|
||||||
|
stoppedBy: "user_finish",
|
||||||
|
bskVersion: "test",
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(trace.version).toBe(TRACE_VERSION_V3);
|
||||||
|
expect(trace.recorder.vom).toBe(VOM_FORMAT_VERSION);
|
||||||
|
expect(trace.steps[0]).toMatchObject({
|
||||||
|
op: "click",
|
||||||
|
target: { role: "button", name: "Save", unmatched: true },
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -9,6 +9,8 @@ import type { ConnectionState, HandshakeResult } from "../transport/types";
|
|||||||
import { getLabel, getOrCreateInstanceId } from "./instance-id";
|
import { getLabel, getOrCreateInstanceId } from "./instance-id";
|
||||||
import { compareProtocol, parseProtocolMajor } from "./semver";
|
import { compareProtocol, parseProtocolMajor } from "./semver";
|
||||||
|
|
||||||
|
const HANDSHAKE_RETRY_DELAY_MS = 1_000;
|
||||||
|
|
||||||
export interface SnapshotInfo {
|
export interface SnapshotInfo {
|
||||||
state: ConnectionState;
|
state: ConnectionState;
|
||||||
instanceId: string;
|
instanceId: string;
|
||||||
@@ -21,6 +23,13 @@ export interface SnapshotInfo {
|
|||||||
|
|
||||||
type Listener = (s: SnapshotInfo) => void;
|
type Listener = (s: SnapshotInfo) => void;
|
||||||
|
|
||||||
|
export interface ConnectionLifecycleHooks {
|
||||||
|
/** Safe local teardown that must finish before an intentional disconnect. */
|
||||||
|
beforeDisconnect?: () => void | Promise<void>;
|
||||||
|
/** Best-effort teardown after an unexpected transport loss. */
|
||||||
|
onDisconnected?: () => void | Promise<void>;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Orchestrates Transport + handshake lifecycle for the background SW.
|
* Orchestrates Transport + handshake lifecycle for the background SW.
|
||||||
*
|
*
|
||||||
@@ -36,7 +45,12 @@ export class ConnectionController {
|
|||||||
private lastError: string | null = null;
|
private lastError: string | null = null;
|
||||||
private connectionEnabled = true;
|
private connectionEnabled = true;
|
||||||
private listeners = new Set<Listener>();
|
private listeners = new Set<Listener>();
|
||||||
private handshakeInFlight = false;
|
private lifecycleHooks: ConnectionLifecycleHooks = {};
|
||||||
|
private disconnectRecovery: Promise<void> | null = null;
|
||||||
|
private connectionGeneration = 0;
|
||||||
|
private handshakeAbort: AbortController | null = null;
|
||||||
|
private handshakeRetryTimer: ReturnType<typeof setTimeout> | null = null;
|
||||||
|
private suppressDisconnectRecovery = false;
|
||||||
|
|
||||||
get isConnectionEnabled(): boolean {
|
get isConnectionEnabled(): boolean {
|
||||||
return this.connectionEnabled;
|
return this.connectionEnabled;
|
||||||
@@ -68,20 +82,36 @@ export class ConnectionController {
|
|||||||
transport: Transport,
|
transport: Transport,
|
||||||
browser: { name: string; version: string },
|
browser: { name: string; version: string },
|
||||||
connectionEnabled = true,
|
connectionEnabled = true,
|
||||||
|
lifecycleHooks: ConnectionLifecycleHooks = {},
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
this.transport = transport;
|
this.transport = transport;
|
||||||
this.connectionEnabled = connectionEnabled;
|
this.connectionEnabled = connectionEnabled;
|
||||||
|
this.lifecycleHooks = lifecycleHooks;
|
||||||
this.instanceId = await getOrCreateInstanceId();
|
this.instanceId = await getOrCreateInstanceId();
|
||||||
this.label = await getLabel();
|
this.label = await getLabel();
|
||||||
|
|
||||||
transport.onConnectionStateChange((s) => {
|
transport.onConnectionStateChange((s) => {
|
||||||
if (!this.connectionEnabled) return;
|
if (s === "disconnected") {
|
||||||
if (s === "connected") {
|
// An in-flight handshake belongs to the dead connection — cancel it so
|
||||||
void this.runHandshake(browser);
|
// a late resolution can never clobber the next attempt.
|
||||||
|
this.cancelHandshake();
|
||||||
|
this.handshake = null;
|
||||||
|
if (this.connectionEnabled) {
|
||||||
|
this.setState("disconnected");
|
||||||
|
if (this.suppressDisconnectRecovery) {
|
||||||
|
// Deliberate disconnect from runHandshake (rejected peer or failed
|
||||||
|
// handshake bounce); the caller owns the reconnect policy there.
|
||||||
|
this.suppressDisconnectRecovery = false;
|
||||||
|
} else {
|
||||||
|
void this.recoverFromDisconnect();
|
||||||
|
}
|
||||||
|
}
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if (s === "disconnected") {
|
if (!this.connectionEnabled) return;
|
||||||
this.handshake = null;
|
if (s === "connected") {
|
||||||
|
this.startHandshake(browser);
|
||||||
|
return;
|
||||||
}
|
}
|
||||||
this.setState(s);
|
this.setState(s);
|
||||||
});
|
});
|
||||||
@@ -126,41 +156,69 @@ export class ConnectionController {
|
|||||||
this.fire();
|
this.fire();
|
||||||
}
|
}
|
||||||
|
|
||||||
private async runHandshake(browser: { name: string; version: string }): Promise<void> {
|
private startHandshake(browser: { name: string; version: string }): void {
|
||||||
|
this.cancelHandshake();
|
||||||
|
const generation = ++this.connectionGeneration;
|
||||||
|
const abort = new AbortController();
|
||||||
|
this.handshakeAbort = abort;
|
||||||
|
void this.runHandshake(browser, generation, abort.signal);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async runHandshake(
|
||||||
|
browser: { name: string; version: string },
|
||||||
|
generation: number,
|
||||||
|
signal: AbortSignal,
|
||||||
|
): Promise<void> {
|
||||||
if (!this.transport) return;
|
if (!this.transport) return;
|
||||||
if (!this.connectionEnabled) return;
|
if (!this.connectionEnabled) return;
|
||||||
if (this.handshakeInFlight) return;
|
|
||||||
this.handshakeInFlight = true;
|
|
||||||
this.setState("connecting");
|
this.setState("connecting");
|
||||||
try {
|
try {
|
||||||
const outcome = await performHandshake(this.transport, {
|
const outcome = await performHandshake(
|
||||||
instanceId: this.instanceId,
|
this.transport,
|
||||||
browser,
|
{
|
||||||
label: this.label,
|
instanceId: this.instanceId,
|
||||||
});
|
browser,
|
||||||
|
label: this.label,
|
||||||
|
},
|
||||||
|
{ signal },
|
||||||
|
);
|
||||||
|
if (generation !== this.connectionGeneration || signal.aborted) return;
|
||||||
this.handshake = outcome.result;
|
this.handshake = outcome.result;
|
||||||
const verdict = computeConnectedState(outcome.result);
|
const verdict = computeConnectedState(outcome.result);
|
||||||
if (verdict.kind === "rejected") {
|
if (verdict.kind === "rejected") {
|
||||||
this.lastError = `version_too_old: ${verdict.reason}`;
|
this.lastError = `version_too_old: ${verdict.reason}`;
|
||||||
this.handshake = null;
|
this.handshake = null;
|
||||||
|
// Rejected peers must not be auto-retried: mark this disconnect as
|
||||||
|
// deliberate so the listener skips unexpected-loss recovery.
|
||||||
|
this.suppressDisconnectRecovery = true;
|
||||||
await this.transport.disconnect().catch(() => {});
|
await this.transport.disconnect().catch(() => {});
|
||||||
this.setState("disconnected");
|
this.setState("disconnected");
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
this.clearHandshakeRetry();
|
||||||
this.lastError = null;
|
this.lastError = null;
|
||||||
this.setState(verdict.kind);
|
this.setState(verdict.kind);
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
|
if (generation !== this.connectionGeneration || signal.aborted || isAbortError(err)) return;
|
||||||
this.handshake = null;
|
this.handshake = null;
|
||||||
this.lastError = err instanceof Error ? err.message : String(err);
|
this.lastError = err instanceof Error ? err.message : String(err);
|
||||||
|
// Bounce the half-open socket deliberately; the retry timer below owns
|
||||||
|
// the reconnect, so skip unexpected-loss recovery for this disconnect.
|
||||||
|
this.suppressDisconnectRecovery = true;
|
||||||
|
await this.transport.disconnect().catch(() => {});
|
||||||
this.setState("disconnected");
|
this.setState("disconnected");
|
||||||
|
this.scheduleHandshakeRetry(browser);
|
||||||
} finally {
|
} finally {
|
||||||
this.handshakeInFlight = false;
|
if (generation === this.connectionGeneration) this.handshakeAbort = null;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private async applyDisabledState(): Promise<void> {
|
private async applyDisabledState(): Promise<void> {
|
||||||
|
this.cancelHandshake();
|
||||||
|
this.clearHandshakeRetry();
|
||||||
this.handshake = null;
|
this.handshake = null;
|
||||||
this.lastError = null;
|
this.lastError = null;
|
||||||
|
await this.lifecycleHooks.beforeDisconnect?.();
|
||||||
if (this.transport) {
|
if (this.transport) {
|
||||||
await this.transport.disconnect().catch(() => {});
|
await this.transport.disconnect().catch(() => {});
|
||||||
}
|
}
|
||||||
@@ -169,6 +227,60 @@ export class ConnectionController {
|
|||||||
if (was === "disconnected") this.fire();
|
if (was === "disconnected") this.fire();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private recoverFromDisconnect(): Promise<void> {
|
||||||
|
if (this.disconnectRecovery) return this.disconnectRecovery;
|
||||||
|
const recovery = (async () => {
|
||||||
|
// WSTransport schedules its reconnect timer immediately after notifying
|
||||||
|
// state listeners. Yield once, then disconnect explicitly so that timer
|
||||||
|
// is cancelled before local session teardown begins.
|
||||||
|
await Promise.resolve();
|
||||||
|
if (!this.connectionEnabled || !this.transport) return;
|
||||||
|
// Another path already reconnected while we yielded — nothing to do.
|
||||||
|
if (this.currentState !== "disconnected") return;
|
||||||
|
await this.transport.disconnect().catch(() => {});
|
||||||
|
try {
|
||||||
|
await this.lifecycleHooks.onDisconnected?.();
|
||||||
|
} catch (err) {
|
||||||
|
console.warn("[browser-skill] session cleanup after disconnect failed", err);
|
||||||
|
}
|
||||||
|
if (!this.connectionEnabled) return;
|
||||||
|
try {
|
||||||
|
await this.transport.connect();
|
||||||
|
} catch (err) {
|
||||||
|
this.lastError = err instanceof Error ? err.message : String(err);
|
||||||
|
this.setState("disconnected");
|
||||||
|
}
|
||||||
|
})();
|
||||||
|
this.disconnectRecovery = recovery.finally(() => {
|
||||||
|
this.disconnectRecovery = null;
|
||||||
|
});
|
||||||
|
return this.disconnectRecovery;
|
||||||
|
}
|
||||||
|
|
||||||
|
private cancelHandshake(): void {
|
||||||
|
this.connectionGeneration += 1;
|
||||||
|
this.handshakeAbort?.abort();
|
||||||
|
this.handshakeAbort = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
private scheduleHandshakeRetry(browser: { name: string; version: string }): void {
|
||||||
|
if (this.handshakeRetryTimer || !this.connectionEnabled) return;
|
||||||
|
this.handshakeRetryTimer = setTimeout(() => {
|
||||||
|
this.handshakeRetryTimer = null;
|
||||||
|
if (!this.connectionEnabled || !this.transport) return;
|
||||||
|
void this.transport.connect().catch((err) => {
|
||||||
|
this.lastError = err instanceof Error ? err.message : String(err);
|
||||||
|
this.setState("disconnected");
|
||||||
|
});
|
||||||
|
}, HANDSHAKE_RETRY_DELAY_MS);
|
||||||
|
}
|
||||||
|
|
||||||
|
private clearHandshakeRetry(): void {
|
||||||
|
if (!this.handshakeRetryTimer) return;
|
||||||
|
clearTimeout(this.handshakeRetryTimer);
|
||||||
|
this.handshakeRetryTimer = null;
|
||||||
|
}
|
||||||
|
|
||||||
private setState(next: ConnectionState): void {
|
private setState(next: ConnectionState): void {
|
||||||
if (this.currentState === next) return;
|
if (this.currentState === next) return;
|
||||||
this.currentState = next;
|
this.currentState = next;
|
||||||
@@ -187,6 +299,12 @@ export class ConnectionController {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function isAbortError(err: unknown): boolean {
|
||||||
|
return (
|
||||||
|
typeof err === "object" && err !== null && (err as { name?: string }).name === "AbortError"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Verdict of the symmetric post-handshake compat check.
|
* Verdict of the symmetric post-handshake compat check.
|
||||||
*/
|
*/
|
||||||
|
|||||||
@@ -1,12 +1,13 @@
|
|||||||
/**
|
/**
|
||||||
* Build a semantic TargetDescriptor for an interacted element.
|
* Build a semantic capture descriptor for an interacted element.
|
||||||
*
|
*
|
||||||
* Trace steps are an LLM *textbook*: each click must say what to look for
|
* Trace steps are an LLM *textbook*: each click must say what to look for
|
||||||
* on screen (usually a short visible name). Tag-only noise like
|
* on screen (usually a short visible name). Tag-only noise like
|
||||||
* `{ "tag": "div" }` / “点击div” is useless and must not be recorded.
|
* `{ "tag": "div" }` / “点击div” is useless and must not be recorded.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
export interface TargetDescriptor {
|
/** Content-script capture descriptor before VOM geometric matching. */
|
||||||
|
export interface CaptureTargetDescriptor {
|
||||||
role?: string;
|
role?: string;
|
||||||
name?: string;
|
name?: string;
|
||||||
tag: string;
|
tag: string;
|
||||||
@@ -228,12 +229,26 @@ export function resolveClickableElement(target: Element): Element | null {
|
|||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function resolveHoverElement(target: Element): Element | null {
|
||||||
|
const clickable = resolveClickableElement(target);
|
||||||
|
if (clickable) return clickable;
|
||||||
|
|
||||||
|
let node: Element | null = target;
|
||||||
|
let depth = 0;
|
||||||
|
while (node && node !== document.body && node !== document.documentElement && depth < 8) {
|
||||||
|
if (looksClickable(node)) return node;
|
||||||
|
node = node.parentElement;
|
||||||
|
depth += 1;
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Teachable clicks: the LLM must get a label it can later find via snapshot
|
* Teachable clicks: the LLM must get a label it can later find via snapshot
|
||||||
* (visible name), or at least a form `name_attr` for checkbox/radio.
|
* (visible name), or at least a form `name_attr` for checkbox/radio.
|
||||||
* Recording “点击div” with no name fails this bar.
|
* Recording “点击div” with no name fails this bar.
|
||||||
*/
|
*/
|
||||||
export function isMeaningfulClickTarget(target: TargetDescriptor): boolean {
|
export function isMeaningfulClickTarget(target: CaptureTargetDescriptor): boolean {
|
||||||
const name = target.name?.trim();
|
const name = target.name?.trim();
|
||||||
if (name && isActionableLabel(name)) return true;
|
if (name && isActionableLabel(name)) return true;
|
||||||
if (
|
if (
|
||||||
@@ -309,7 +324,7 @@ function nearbyLabelText(el: Element): string | undefined {
|
|||||||
return undefined;
|
return undefined;
|
||||||
}
|
}
|
||||||
|
|
||||||
export function describeTarget(el: Element): TargetDescriptor {
|
export function describeTarget(el: Element): CaptureTargetDescriptor {
|
||||||
const tag = el.tagName.toLowerCase();
|
const tag = el.tagName.toLowerCase();
|
||||||
const role = inferRole(el);
|
const role = inferRole(el);
|
||||||
const name = accessibleName(el);
|
const name = accessibleName(el);
|
||||||
@@ -336,7 +351,7 @@ export function describeTarget(el: Element): TargetDescriptor {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
export function describeEventTarget(target: EventTarget | null): TargetDescriptor | null {
|
export function describeEventTarget(target: EventTarget | null): CaptureTargetDescriptor | null {
|
||||||
if (!(target instanceof Element)) return null;
|
if (!(target instanceof Element)) return null;
|
||||||
const clickable = resolveClickableElement(target);
|
const clickable = resolveClickableElement(target);
|
||||||
if (!clickable) return null;
|
if (!clickable) return null;
|
||||||
|
|||||||
@@ -16,6 +16,13 @@ export const HELP_ACK = "bsk-help-ack";
|
|||||||
export const HELP_QUERY = "bsk-help-query";
|
export const HELP_QUERY = "bsk-help-query";
|
||||||
export const HELP_FINISH = "bsk-help-finish";
|
export const HELP_FINISH = "bsk-help-finish";
|
||||||
|
|
||||||
|
export interface HelpHighlightRect {
|
||||||
|
top: number;
|
||||||
|
left: number;
|
||||||
|
width: number;
|
||||||
|
height: number;
|
||||||
|
}
|
||||||
|
|
||||||
export interface HelpRequestMessage {
|
export interface HelpRequestMessage {
|
||||||
type: typeof HELP_REQUEST;
|
type: typeof HELP_REQUEST;
|
||||||
requestId: string;
|
requestId: string;
|
||||||
@@ -26,6 +33,8 @@ export interface HelpRequestMessage {
|
|||||||
displayMode?: "full" | "compact";
|
displayMode?: "full" | "compact";
|
||||||
/** CSS selectors to scroll to + flash-highlight (may be empty). */
|
/** CSS selectors to scroll to + flash-highlight (may be empty). */
|
||||||
selectors: string[];
|
selectors: string[];
|
||||||
|
/** Top-viewport rectangles for targets that cannot be represented by a root selector. */
|
||||||
|
rects?: HelpHighlightRect[];
|
||||||
timeoutMs: number;
|
timeoutMs: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -74,6 +83,16 @@ export function isHelpRequestMessage(msg: unknown): msg is HelpRequestMessage {
|
|||||||
(m.displayMode === undefined || m.displayMode === "full" || m.displayMode === "compact") &&
|
(m.displayMode === undefined || m.displayMode === "full" || m.displayMode === "compact") &&
|
||||||
Array.isArray(m.selectors) &&
|
Array.isArray(m.selectors) &&
|
||||||
m.selectors.every((selector) => typeof selector === "string") &&
|
m.selectors.every((selector) => typeof selector === "string") &&
|
||||||
|
(m.rects === undefined ||
|
||||||
|
(Array.isArray(m.rects) &&
|
||||||
|
m.rects.every(
|
||||||
|
(rect) =>
|
||||||
|
typeof rect === "object" &&
|
||||||
|
rect !== null &&
|
||||||
|
["top", "left", "width", "height"].every((key) =>
|
||||||
|
Number.isFinite((rect as Record<string, unknown>)[key]),
|
||||||
|
),
|
||||||
|
))) &&
|
||||||
typeof m.timeoutMs === "number"
|
typeof m.timeoutMs === "number"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,178 @@
|
|||||||
|
export interface HoverTriggerRect {
|
||||||
|
x: number;
|
||||||
|
y: number;
|
||||||
|
w: number;
|
||||||
|
h: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface HoverTriggerSignals {
|
||||||
|
tag: string;
|
||||||
|
role?: string;
|
||||||
|
label?: string;
|
||||||
|
attrs?: Record<string, string | undefined>;
|
||||||
|
rect?: HoverTriggerRect | null;
|
||||||
|
cursor?: string;
|
||||||
|
pointerEvents?: string;
|
||||||
|
hasGraphicDescendant?: boolean;
|
||||||
|
cssHoverMatch?: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface HoverTriggerDecision {
|
||||||
|
eligible: boolean;
|
||||||
|
score: number;
|
||||||
|
reasons: string[];
|
||||||
|
}
|
||||||
|
|
||||||
|
const HOVER_TRIGGER_MIN_SCORE = 45;
|
||||||
|
const HOVER_POPUP_SIGNAL_RE =
|
||||||
|
/(menu|dropdown|popover|popup|avatar|profile|account|user|more|ellipsis|caret)/;
|
||||||
|
const HOVER_TEXT_ITEM_RE = /(^|[\s_-])(dropdown-item|menu-item|context-menu-item|option)($|[\s_-])/;
|
||||||
|
|
||||||
|
function attr(signals: HoverTriggerSignals, name: string): string | undefined {
|
||||||
|
return signals.attrs?.[name.toLowerCase()];
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isUnsafeHoverTrigger(signals: HoverTriggerSignals): boolean {
|
||||||
|
const tag = signals.tag.toLowerCase();
|
||||||
|
if (["input", "textarea", "select", "option"].includes(tag)) return true;
|
||||||
|
if ((attr(signals, "contenteditable") ?? "").toLowerCase() === "true") return true;
|
||||||
|
if (attr(signals, "disabled") !== undefined || attr(signals, "inert") !== undefined) return true;
|
||||||
|
return (attr(signals, "aria-disabled") ?? "").toLowerCase() === "true";
|
||||||
|
}
|
||||||
|
|
||||||
|
export function hasHoverPopupSignal(signals: HoverTriggerSignals): boolean {
|
||||||
|
const attrs = signals.attrs ?? {};
|
||||||
|
const haystack = [
|
||||||
|
attrs.id,
|
||||||
|
attrs.class,
|
||||||
|
attrs["data-testid"],
|
||||||
|
attrs["data-test"],
|
||||||
|
attrs["data-cy"],
|
||||||
|
attrs["aria-label"],
|
||||||
|
attrs["aria-haspopup"],
|
||||||
|
attrs["aria-controls"],
|
||||||
|
attrs.title,
|
||||||
|
signals.role === "button" ? signals.label : undefined,
|
||||||
|
]
|
||||||
|
.filter(Boolean)
|
||||||
|
.join(" ")
|
||||||
|
.toLowerCase();
|
||||||
|
return HOVER_POPUP_SIGNAL_RE.test(haystack);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function hasStrongHoverExpansionSignal(signals: HoverTriggerSignals): boolean {
|
||||||
|
return (
|
||||||
|
signals.cssHoverMatch === true ||
|
||||||
|
attr(signals, "aria-haspopup") !== undefined ||
|
||||||
|
(attr(signals, "aria-expanded") ?? "").toLowerCase() === "false"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function hasDirectHoverInteractiveSignal(signals: HoverTriggerSignals): boolean {
|
||||||
|
const tag = signals.tag.toLowerCase();
|
||||||
|
const role = (signals.role ?? "").toLowerCase();
|
||||||
|
return (
|
||||||
|
["button", "a", "summary"].includes(tag) ||
|
||||||
|
["button", "link", "menuitem", "tab", "combobox"].includes(role) ||
|
||||||
|
attr(signals, "tabindex") !== undefined ||
|
||||||
|
signals.cursor === "pointer" ||
|
||||||
|
attr(signals, "onclick") !== undefined ||
|
||||||
|
attr(signals, "onmouseenter") !== undefined ||
|
||||||
|
attr(signals, "onmouseover") !== undefined ||
|
||||||
|
attr(signals, "aria-haspopup") !== undefined ||
|
||||||
|
attr(signals, "aria-expanded") === "false"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function evaluateHoverTrigger(signals: HoverTriggerSignals): HoverTriggerDecision {
|
||||||
|
const rect = signals.rect;
|
||||||
|
if (
|
||||||
|
!rect ||
|
||||||
|
rect.w <= 0 ||
|
||||||
|
rect.h <= 0 ||
|
||||||
|
signals.pointerEvents === "none" ||
|
||||||
|
attr(signals, "hidden") !== undefined ||
|
||||||
|
attr(signals, "inert") !== undefined ||
|
||||||
|
(attr(signals, "aria-hidden") ?? "").toLowerCase() === "true" ||
|
||||||
|
isUnsafeHoverTrigger(signals)
|
||||||
|
) {
|
||||||
|
return { eligible: false, score: 0, reasons: [] };
|
||||||
|
}
|
||||||
|
|
||||||
|
const area = rect.w * rect.h;
|
||||||
|
if (area <= 0 || area > 160_000) {
|
||||||
|
return { eligible: false, score: 0, reasons: [] };
|
||||||
|
}
|
||||||
|
|
||||||
|
const reasons: string[] = [];
|
||||||
|
let score = 0;
|
||||||
|
const tag = signals.tag.toLowerCase();
|
||||||
|
const role = (signals.role ?? "").toLowerCase();
|
||||||
|
const label = signals.label?.trim();
|
||||||
|
const graphic = signals.hasGraphicDescendant === true;
|
||||||
|
const directInteractive = hasDirectHoverInteractiveSignal(signals);
|
||||||
|
const popupSignal = hasHoverPopupSignal(signals);
|
||||||
|
const isTextNavigationItem =
|
||||||
|
(tag === "a" ||
|
||||||
|
tag === "li" ||
|
||||||
|
role === "link" ||
|
||||||
|
role === "option" ||
|
||||||
|
role.startsWith("menuitem") ||
|
||||||
|
HOVER_TEXT_ITEM_RE.test(attr(signals, "class") ?? "")) &&
|
||||||
|
!graphic &&
|
||||||
|
!!label &&
|
||||||
|
label.toLowerCase() !== "image";
|
||||||
|
const compactTopbarIcon =
|
||||||
|
(role === "button" || role === "link" || tag === "button" || tag === "a") &&
|
||||||
|
graphic &&
|
||||||
|
rect.y < 120 &&
|
||||||
|
rect.w <= 96 &&
|
||||||
|
rect.h <= 96 &&
|
||||||
|
(!label || label.toLowerCase() === "image");
|
||||||
|
const surfaceTriggerEvidence =
|
||||||
|
(popupSignal && !isTextNavigationItem) ||
|
||||||
|
compactTopbarIcon ||
|
||||||
|
hasStrongHoverExpansionSignal(signals);
|
||||||
|
if (!directInteractive || !surfaceTriggerEvidence) {
|
||||||
|
return { eligible: false, score: 0, reasons: [] };
|
||||||
|
}
|
||||||
|
|
||||||
|
if (attr(signals, "aria-haspopup") !== undefined) {
|
||||||
|
score += 80;
|
||||||
|
reasons.push("aria-haspopup");
|
||||||
|
}
|
||||||
|
if ((attr(signals, "aria-expanded") ?? "").toLowerCase() === "false") {
|
||||||
|
score += 55;
|
||||||
|
reasons.push("collapsed");
|
||||||
|
}
|
||||||
|
if (popupSignal) {
|
||||||
|
score += 45;
|
||||||
|
reasons.push("popup-signal");
|
||||||
|
}
|
||||||
|
if (tag === "button" || role === "button") {
|
||||||
|
score += 30;
|
||||||
|
reasons.push("button");
|
||||||
|
}
|
||||||
|
if (signals.cursor === "pointer") {
|
||||||
|
score += 25;
|
||||||
|
reasons.push("pointer");
|
||||||
|
}
|
||||||
|
if (graphic && (!label || label.toLowerCase() === "image")) {
|
||||||
|
score += 40;
|
||||||
|
reasons.push("icon-only");
|
||||||
|
}
|
||||||
|
if (rect.y < 120) {
|
||||||
|
score += 25;
|
||||||
|
reasons.push("topbar");
|
||||||
|
}
|
||||||
|
if (rect.w <= 80 && rect.h <= 80) {
|
||||||
|
score += 20;
|
||||||
|
reasons.push("compact");
|
||||||
|
}
|
||||||
|
if (signals.cssHoverMatch) {
|
||||||
|
score += 40;
|
||||||
|
reasons.push("css-hover");
|
||||||
|
}
|
||||||
|
|
||||||
|
return { eligible: score >= HOVER_TRIGGER_MIN_SCORE, score, reasons };
|
||||||
|
}
|
||||||
@@ -1,6 +1,7 @@
|
|||||||
const STORAGE_KEY = "bsk_instance_id";
|
const STORAGE_KEY = "bsk_instance_id";
|
||||||
const LABEL_STORAGE_KEY = "bh_label";
|
const LABEL_STORAGE_KEY = "bh_label";
|
||||||
const CONNECTION_ENABLED_KEY = "bh_connection_enabled";
|
const CONNECTION_ENABLED_KEY = "bh_connection_enabled";
|
||||||
|
const CONTROL_HINTS_HIDDEN_KEY = "bsk_control_hints_hidden";
|
||||||
|
|
||||||
export interface StorageBackend {
|
export interface StorageBackend {
|
||||||
get(keys: string | string[]): Promise<Record<string, unknown>>;
|
get(keys: string | string[]): Promise<Record<string, unknown>>;
|
||||||
@@ -105,8 +106,30 @@ export async function setConnectionEnabled(
|
|||||||
await storage.set({ [CONNECTION_ENABLED_KEY]: enabled });
|
await storage.set({ [CONNECTION_ENABLED_KEY]: enabled });
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* User preference for the in-page control hints (status pill + orange glow
|
||||||
|
* shown while the Agent controls a tab). Defaults to shown when unset or
|
||||||
|
* non-boolean; when hidden the whole control overlay — including its input
|
||||||
|
* blocker — is skipped so the page looks and behaves normally.
|
||||||
|
*/
|
||||||
|
export async function getControlHintsHidden(
|
||||||
|
storage: StorageBackend = defaultStorage(),
|
||||||
|
): Promise<boolean> {
|
||||||
|
const items = await storage.get(CONTROL_HINTS_HIDDEN_KEY);
|
||||||
|
const raw = items[CONTROL_HINTS_HIDDEN_KEY];
|
||||||
|
return typeof raw === "boolean" ? raw : false;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function setControlHintsHidden(
|
||||||
|
hidden: boolean,
|
||||||
|
storage: StorageBackend = defaultStorage(),
|
||||||
|
): Promise<void> {
|
||||||
|
await storage.set({ [CONTROL_HINTS_HIDDEN_KEY]: hidden });
|
||||||
|
}
|
||||||
|
|
||||||
export const STORAGE_KEYS = {
|
export const STORAGE_KEYS = {
|
||||||
INSTANCE_ID: STORAGE_KEY,
|
INSTANCE_ID: STORAGE_KEY,
|
||||||
LABEL: LABEL_STORAGE_KEY,
|
LABEL: LABEL_STORAGE_KEY,
|
||||||
CONNECTION_ENABLED: CONNECTION_ENABLED_KEY,
|
CONNECTION_ENABLED: CONNECTION_ENABLED_KEY,
|
||||||
|
CONTROL_HINTS_HIDDEN: CONTROL_HINTS_HIDDEN_KEY,
|
||||||
} as const;
|
} as const;
|
||||||
@@ -3,7 +3,7 @@
|
|||||||
* service worker and a tab's content script.
|
* service worker and a tab's content script.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import type { TargetDescriptor } from "./describe-target";
|
import type { CaptureTargetDescriptor } from "./describe-target";
|
||||||
|
|
||||||
export const RECORD_START = "bsk-record-start";
|
export const RECORD_START = "bsk-record-start";
|
||||||
export const RECORD_STEP = "bsk-record-step";
|
export const RECORD_STEP = "bsk-record-step";
|
||||||
@@ -41,8 +41,8 @@ export interface RecordStartMessage {
|
|||||||
}
|
}
|
||||||
|
|
||||||
export interface RecordStepPayload {
|
export interface RecordStepPayload {
|
||||||
op: "click" | "fill" | "press" | "select" | "navigate";
|
op: "click" | "hover" | "fill" | "press" | "select" | "navigate";
|
||||||
target?: TargetDescriptor;
|
target?: CaptureTargetDescriptor;
|
||||||
value?: string;
|
value?: string;
|
||||||
key?: string;
|
key?: string;
|
||||||
modifiers?: Array<"alt" | "ctrl" | "meta" | "shift">;
|
modifiers?: Array<"alt" | "ctrl" | "meta" | "shift">;
|
||||||
|
|||||||
@@ -0,0 +1,134 @@
|
|||||||
|
import type { CdpTarget } from "@/browser-driver/frame-graph";
|
||||||
|
import { type CdpRunner, sendToCdpTarget } from "@/tools/shared";
|
||||||
|
|
||||||
|
const SETTLE_MIN_MS = 150;
|
||||||
|
const SETTLE_QUIET_MS = 250;
|
||||||
|
const SETTLE_MAX_MS = 2_000;
|
||||||
|
const SETTLE_POLL_MS = 60;
|
||||||
|
const SETTLE_WORLD_NAME = "__bsk_record_settle__";
|
||||||
|
|
||||||
|
const QUIET_PROBE = `(() => {
|
||||||
|
const scope = window;
|
||||||
|
let probe = scope.__bskRecordQuiet;
|
||||||
|
if (!probe) {
|
||||||
|
probe = { changedAt: Date.now() };
|
||||||
|
const observer = new MutationObserver(() => {
|
||||||
|
probe.changedAt = Date.now();
|
||||||
|
});
|
||||||
|
observer.observe(document.documentElement, {
|
||||||
|
subtree: true,
|
||||||
|
childList: true,
|
||||||
|
attributes: true,
|
||||||
|
characterData: true,
|
||||||
|
});
|
||||||
|
scope.__bskRecordQuiet = probe;
|
||||||
|
}
|
||||||
|
return { idleMs: Date.now() - probe.changedAt, readyState: document.readyState };
|
||||||
|
})()`;
|
||||||
|
|
||||||
|
export interface DocumentSettleScope {
|
||||||
|
target: CdpTarget;
|
||||||
|
/** Omit only for the target's root document. */
|
||||||
|
frameId?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
type SettleOutcome = "quiet" | "timeout" | "cancelled";
|
||||||
|
|
||||||
|
interface QuietProbe {
|
||||||
|
idleMs: number;
|
||||||
|
readyState: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface ProbeContext {
|
||||||
|
executionContextId?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
function sleep(ms: number, signal?: AbortSignal): Promise<void> {
|
||||||
|
return new Promise((resolve) => {
|
||||||
|
if (signal?.aborted) return resolve();
|
||||||
|
const timer = setTimeout(resolve, ms);
|
||||||
|
signal?.addEventListener(
|
||||||
|
"abort",
|
||||||
|
() => {
|
||||||
|
clearTimeout(timer);
|
||||||
|
resolve();
|
||||||
|
},
|
||||||
|
{ once: true },
|
||||||
|
);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async function executionContextForScope(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
scope: DocumentSettleScope,
|
||||||
|
): Promise<number | undefined> {
|
||||||
|
if (!scope.frameId) return undefined;
|
||||||
|
const result = await sendToCdpTarget<{ executionContextId?: number }>(
|
||||||
|
cdp,
|
||||||
|
scope.target,
|
||||||
|
"Page.createIsolatedWorld",
|
||||||
|
{
|
||||||
|
frameId: scope.frameId,
|
||||||
|
worldName: SETTLE_WORLD_NAME,
|
||||||
|
grantUniveralAccess: false,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
return result.executionContextId;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function readQuietProbe(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
scope: DocumentSettleScope,
|
||||||
|
context: ProbeContext,
|
||||||
|
): Promise<QuietProbe | null> {
|
||||||
|
try {
|
||||||
|
if (scope.frameId && context.executionContextId === undefined) {
|
||||||
|
context.executionContextId = await executionContextForScope(cdp, scope);
|
||||||
|
if (context.executionContextId === undefined) return null;
|
||||||
|
}
|
||||||
|
const reply = await sendToCdpTarget<{ result?: { value?: unknown } }>(
|
||||||
|
cdp,
|
||||||
|
scope.target,
|
||||||
|
"Runtime.evaluate",
|
||||||
|
{
|
||||||
|
expression: QUIET_PROBE,
|
||||||
|
returnByValue: true,
|
||||||
|
...(context.executionContextId !== undefined
|
||||||
|
? { contextId: context.executionContextId }
|
||||||
|
: {}),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
const value = reply.result?.value;
|
||||||
|
if (!value || typeof value !== "object") return null;
|
||||||
|
const { idleMs, readyState } = value as { idleMs?: unknown; readyState?: unknown };
|
||||||
|
if (typeof idleMs !== "number" || typeof readyState !== "string") return null;
|
||||||
|
return { idleMs, readyState };
|
||||||
|
} catch {
|
||||||
|
context.executionContextId = undefined;
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function waitForDocumentSettled(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
scope: DocumentSettleScope,
|
||||||
|
options: { signal?: AbortSignal } = {},
|
||||||
|
): Promise<SettleOutcome> {
|
||||||
|
const startedAt = Date.now();
|
||||||
|
const floor = startedAt + SETTLE_MIN_MS;
|
||||||
|
const deadline = startedAt + SETTLE_MAX_MS;
|
||||||
|
const context: ProbeContext = {};
|
||||||
|
|
||||||
|
for (;;) {
|
||||||
|
if (options.signal?.aborted) return "cancelled";
|
||||||
|
await sleep(SETTLE_POLL_MS, options.signal);
|
||||||
|
if (options.signal?.aborted) return "cancelled";
|
||||||
|
|
||||||
|
const probe = await readQuietProbe(cdp, scope, context);
|
||||||
|
const now = Date.now();
|
||||||
|
if (now < floor) continue;
|
||||||
|
if (now >= deadline) return "timeout";
|
||||||
|
if (!probe || probe.readyState === "loading") continue;
|
||||||
|
if (probe.idleMs >= SETTLE_QUIET_MS) return "quiet";
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,34 @@
|
|||||||
|
import type { RecordingDraftStep } from "./types";
|
||||||
|
|
||||||
|
const CLIPBOARD_KEYS = new Set(["a", "c", "v", "x", "A", "C", "V", "X"]);
|
||||||
|
const MODIFIER_ONLY_KEYS = new Set(["Meta", "Control", "Alt", "Shift", "OS", "Hyper", "Super"]);
|
||||||
|
|
||||||
|
export function shouldRecordPress(
|
||||||
|
key: string,
|
||||||
|
modifiers?: Array<"alt" | "ctrl" | "meta" | "shift">,
|
||||||
|
): boolean {
|
||||||
|
if (MODIFIER_ONLY_KEYS.has(key)) return false;
|
||||||
|
const mods = modifiers ?? [];
|
||||||
|
const hasCtrlOrMeta = mods.includes("ctrl") || mods.includes("meta");
|
||||||
|
if (hasCtrlOrMeta && CLIPBOARD_KEYS.has(key)) return false;
|
||||||
|
if (key === "Enter" || key === "Escape") return true;
|
||||||
|
if (key.length === 1 && !hasCtrlOrMeta && !mods.includes("alt")) return false;
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function shouldIncludeDraft(step: RecordingDraftStep): boolean {
|
||||||
|
return step.op !== "press" || shouldRecordPress(step.key, step.modifiers);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function resolveDraftStartUrl(
|
||||||
|
drafts: RecordingDraftStep[],
|
||||||
|
explicitStartUrl?: string,
|
||||||
|
fallbackUrl?: string,
|
||||||
|
): string {
|
||||||
|
if (explicitStartUrl) return explicitStartUrl;
|
||||||
|
const navigation = drafts.find(
|
||||||
|
(step): step is Extract<RecordingDraftStep, { op: "navigate" }> => step.op === "navigate",
|
||||||
|
);
|
||||||
|
if (navigation) return navigation.url;
|
||||||
|
return drafts.find((step) => step.pageUrl)?.pageUrl ?? fallbackUrl ?? "about:blank";
|
||||||
|
}
|
||||||
@@ -0,0 +1,116 @@
|
|||||||
|
import type { RenderedRef } from "@browser-skill/vom";
|
||||||
|
import {
|
||||||
|
type CaptureVomMatchNode,
|
||||||
|
type CaptureVomObservationResult,
|
||||||
|
captureVomObservation,
|
||||||
|
} from "@/tools/capture-vom-observation";
|
||||||
|
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
|
||||||
|
|
||||||
|
export interface IndexedObservationNode {
|
||||||
|
frameId: string;
|
||||||
|
geometry: CaptureVomMatchNode;
|
||||||
|
ref?: RenderedRef;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface CapturedRecordingObservation {
|
||||||
|
rootFrameId: string;
|
||||||
|
index: ObservationNodeIndex;
|
||||||
|
url: string;
|
||||||
|
title?: string;
|
||||||
|
vomText: string;
|
||||||
|
truncated: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface RegisteredObservation {
|
||||||
|
stateId: string;
|
||||||
|
rootFrameId: string;
|
||||||
|
index: ObservationNodeIndex;
|
||||||
|
url: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
function nodeKey(frameId: string, backendNodeId: number): string {
|
||||||
|
return `${frameId}:${backendNodeId}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function frameTagKey(frameId: string, tag: string): string {
|
||||||
|
return `${frameId}:${tag.toLowerCase()}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
export class ObservationNodeIndex {
|
||||||
|
readonly #nodesByFrameTag = new Map<string, IndexedObservationNode[]>();
|
||||||
|
readonly #refById = new Map<string, RenderedRef>();
|
||||||
|
readonly #refsByFrame = new Map<string, RenderedRef[]>();
|
||||||
|
|
||||||
|
constructor(input: Pick<CaptureVomObservationResult, "rootFrameId" | "matchNodes" | "refs">) {
|
||||||
|
const refByNode = new Map<string, RenderedRef>();
|
||||||
|
for (const ref of input.refs) {
|
||||||
|
const frameId = ref.frameId ?? input.rootFrameId;
|
||||||
|
refByNode.set(nodeKey(frameId, ref.backendNodeId), ref);
|
||||||
|
this.#refById.set(ref.ref, ref);
|
||||||
|
const frameRefs = this.#refsByFrame.get(frameId) ?? [];
|
||||||
|
frameRefs.push(ref);
|
||||||
|
this.#refsByFrame.set(frameId, frameRefs);
|
||||||
|
}
|
||||||
|
for (const geometry of input.matchNodes) {
|
||||||
|
const { frameId } = geometry;
|
||||||
|
const entry = {
|
||||||
|
frameId,
|
||||||
|
geometry,
|
||||||
|
ref: refByNode.get(nodeKey(frameId, geometry.backendNodeId)),
|
||||||
|
};
|
||||||
|
const key = frameTagKey(frameId, geometry.tag);
|
||||||
|
const bucket = this.#nodesByFrameTag.get(key) ?? [];
|
||||||
|
bucket.push(entry);
|
||||||
|
this.#nodesByFrameTag.set(key, bucket);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
candidates(frameId: string, tag: string): readonly IndexedObservationNode[] {
|
||||||
|
return this.#nodesByFrameTag.get(frameTagKey(frameId, tag)) ?? [];
|
||||||
|
}
|
||||||
|
|
||||||
|
ref(refId: string): RenderedRef | undefined {
|
||||||
|
return this.#refById.get(refId);
|
||||||
|
}
|
||||||
|
|
||||||
|
refs(frameId: string): readonly RenderedRef[] {
|
||||||
|
return this.#refsByFrame.get(frameId) ?? [];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function readTabMeta(
|
||||||
|
tabsApi: ChromeTabsApi,
|
||||||
|
tabId: number,
|
||||||
|
): Promise<{ url: string; title?: string }> {
|
||||||
|
try {
|
||||||
|
const tab = await tabsApi.get(tabId);
|
||||||
|
return { url: tab.url ?? "about:blank", title: tab.title };
|
||||||
|
} catch {
|
||||||
|
return { url: "about:blank" };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function captureRecordingObservation(input: {
|
||||||
|
cdp: CdpRunner;
|
||||||
|
tabsApi: ChromeTabsApi;
|
||||||
|
tabId: number;
|
||||||
|
maxTokens: number;
|
||||||
|
redactValues: boolean;
|
||||||
|
signal?: AbortSignal;
|
||||||
|
}): Promise<CapturedRecordingObservation> {
|
||||||
|
const { url, title } = await readTabMeta(input.tabsApi, input.tabId);
|
||||||
|
const captured = await captureVomObservation(input.cdp, input.tabId, url, {
|
||||||
|
maxTokens: input.maxTokens,
|
||||||
|
redactValues: input.redactValues,
|
||||||
|
conditionalSurfaceProbe: false,
|
||||||
|
signal: input.signal,
|
||||||
|
});
|
||||||
|
return {
|
||||||
|
rootFrameId: captured.rootFrameId,
|
||||||
|
index: new ObservationNodeIndex(captured),
|
||||||
|
url,
|
||||||
|
title,
|
||||||
|
vomText: captured.text,
|
||||||
|
truncated: captured.truncated,
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -0,0 +1,122 @@
|
|||||||
|
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
|
||||||
|
import { captureRecordingObservation, type RegisteredObservation } from "./observation-capture";
|
||||||
|
import { RecordingStateRegistry } from "./state-registry";
|
||||||
|
import { matchObservationTarget, unmatchedTarget } from "./target-matcher";
|
||||||
|
import type { RecordingDraftStep, StepAnnotation, TargetedRecordingDraft } from "./types";
|
||||||
|
|
||||||
|
const DEFAULT_MAX_PAGE_TOKENS = 3_000;
|
||||||
|
const MIN_CAPTURE_INTERVAL_MS = 200;
|
||||||
|
|
||||||
|
export interface TabObservationCursor {
|
||||||
|
lastSettled: RegisteredObservation | null;
|
||||||
|
lastCaptureAt: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
function abortableDelay(ms: number, signal?: AbortSignal): Promise<void> {
|
||||||
|
return new Promise((resolve) => {
|
||||||
|
if (signal?.aborted) return resolve();
|
||||||
|
const timer = setTimeout(resolve, ms);
|
||||||
|
signal?.addEventListener(
|
||||||
|
"abort",
|
||||||
|
() => {
|
||||||
|
clearTimeout(timer);
|
||||||
|
resolve();
|
||||||
|
},
|
||||||
|
{ once: true },
|
||||||
|
);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function isTargeted(draft: RecordingDraftStep): draft is TargetedRecordingDraft {
|
||||||
|
return draft.op !== "navigate" && draft.op !== "scroll";
|
||||||
|
}
|
||||||
|
|
||||||
|
export class RecordingObservationSession {
|
||||||
|
readonly registry: RecordingStateRegistry;
|
||||||
|
readonly cursor: TabObservationCursor;
|
||||||
|
readonly annotations: StepAnnotation[];
|
||||||
|
readonly #maxTokens: number;
|
||||||
|
readonly #redactValues: boolean;
|
||||||
|
|
||||||
|
constructor(
|
||||||
|
options: {
|
||||||
|
registry?: RecordingStateRegistry;
|
||||||
|
cursor?: TabObservationCursor;
|
||||||
|
annotations?: StepAnnotation[];
|
||||||
|
maxTokens?: number;
|
||||||
|
redactValues?: boolean;
|
||||||
|
} = {},
|
||||||
|
) {
|
||||||
|
this.registry = options.registry ?? new RecordingStateRegistry();
|
||||||
|
this.cursor = options.cursor ?? { lastSettled: null, lastCaptureAt: 0 };
|
||||||
|
this.annotations = options.annotations ?? [];
|
||||||
|
this.#maxTokens = options.maxTokens ?? DEFAULT_MAX_PAGE_TOKENS;
|
||||||
|
this.#redactValues = options.redactValues ?? false;
|
||||||
|
}
|
||||||
|
|
||||||
|
async capture(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
tabsApi: ChromeTabsApi,
|
||||||
|
tabId: number,
|
||||||
|
signal?: AbortSignal,
|
||||||
|
): Promise<RegisteredObservation> {
|
||||||
|
const waitMs = Math.max(0, MIN_CAPTURE_INTERVAL_MS - (Date.now() - this.cursor.lastCaptureAt));
|
||||||
|
if (waitMs > 0) await abortableDelay(waitMs, signal);
|
||||||
|
if (signal?.aborted) throw new DOMException("observation aborted", "AbortError");
|
||||||
|
const captured = await captureRecordingObservation({
|
||||||
|
cdp,
|
||||||
|
tabsApi,
|
||||||
|
tabId,
|
||||||
|
maxTokens: this.#maxTokens,
|
||||||
|
redactValues: this.#redactValues,
|
||||||
|
signal,
|
||||||
|
});
|
||||||
|
const state = this.registry.register({
|
||||||
|
url: captured.url,
|
||||||
|
title: captured.title,
|
||||||
|
rawVomText: captured.vomText,
|
||||||
|
truncated: captured.truncated,
|
||||||
|
});
|
||||||
|
const observation: RegisteredObservation = {
|
||||||
|
stateId: state.id,
|
||||||
|
rootFrameId: captured.rootFrameId,
|
||||||
|
index: captured.index,
|
||||||
|
url: captured.url,
|
||||||
|
};
|
||||||
|
this.cursor.lastSettled = observation;
|
||||||
|
this.cursor.lastCaptureAt = Date.now();
|
||||||
|
return observation;
|
||||||
|
}
|
||||||
|
|
||||||
|
bindDraft(draft: RecordingDraftStep, draftId: number, previousActionPending = false): void {
|
||||||
|
const observation = this.cursor.lastSettled;
|
||||||
|
if (isTargeted(draft)) {
|
||||||
|
draft.matchedTarget = observation
|
||||||
|
? matchObservationTarget({
|
||||||
|
observation,
|
||||||
|
hint: draft.targetHint,
|
||||||
|
fallback: draft.captureTarget,
|
||||||
|
})
|
||||||
|
: unmatchedTarget(draft.captureTarget);
|
||||||
|
}
|
||||||
|
if (!observation) return;
|
||||||
|
|
||||||
|
const unmatched = isTargeted(draft) && draft.matchedTarget?.unmatched === true;
|
||||||
|
if (previousActionPending && unmatched) return;
|
||||||
|
draft.preStateId = observation.stateId;
|
||||||
|
this.registry.markStep(observation.stateId, draftId);
|
||||||
|
|
||||||
|
if (!isTargeted(draft) || !draft.matchedTarget?.ref) return;
|
||||||
|
const ref = observation.index.ref(draft.matchedTarget.ref);
|
||||||
|
if (!ref) return;
|
||||||
|
this.annotations.push({
|
||||||
|
draftId,
|
||||||
|
op: draft.op,
|
||||||
|
line: ref.line,
|
||||||
|
stateId: observation.stateId,
|
||||||
|
...(draft.op === "fill" && !this.#redactValues
|
||||||
|
? { detail: JSON.stringify(draft.value) }
|
||||||
|
: {}),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,232 @@
|
|||||||
|
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
|
||||||
|
import { type DocumentSettleScope, waitForDocumentSettled } from "./document-settle";
|
||||||
|
import { RecordingObservationSession } from "./observation-session";
|
||||||
|
import type { RecordingDraftStep } from "./types";
|
||||||
|
|
||||||
|
interface PendingSettle {
|
||||||
|
abort: AbortController;
|
||||||
|
scope: DocumentSettleScope;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface PendingRedirect {
|
||||||
|
url: string;
|
||||||
|
abort: AbortController;
|
||||||
|
}
|
||||||
|
|
||||||
|
const CAPTURE_RETRY_DELAY_MS = 250;
|
||||||
|
|
||||||
|
function delay(ms: number, signal?: AbortSignal): Promise<void> {
|
||||||
|
return new Promise((resolve) => {
|
||||||
|
if (signal?.aborted) return resolve();
|
||||||
|
const timer = setTimeout(resolve, ms);
|
||||||
|
signal?.addEventListener(
|
||||||
|
"abort",
|
||||||
|
() => {
|
||||||
|
clearTimeout(timer);
|
||||||
|
resolve();
|
||||||
|
},
|
||||||
|
{ once: true },
|
||||||
|
);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function isAbortError(error: unknown): boolean {
|
||||||
|
return (
|
||||||
|
typeof error === "object" &&
|
||||||
|
error !== null &&
|
||||||
|
(error as { name?: string }).name === "AbortError"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function inferMissingPostStates(drafts: RecordingDraftStep[]): void {
|
||||||
|
for (let index = 0; index < drafts.length - 1; index += 1) {
|
||||||
|
const draft = drafts[index];
|
||||||
|
const next = drafts[index + 1];
|
||||||
|
if (draft && next && !draft.postStateId && next.preStateId) {
|
||||||
|
draft.postStateId = next.preStateId;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export class SettleController {
|
||||||
|
readonly #session: RecordingObservationSession;
|
||||||
|
readonly #cdp: CdpRunner;
|
||||||
|
readonly #tabsApi: ChromeTabsApi;
|
||||||
|
readonly #tabId: number;
|
||||||
|
readonly #rootScope: DocumentSettleScope;
|
||||||
|
readonly #pending = new Map<number, PendingSettle>();
|
||||||
|
#queue = Promise.resolve();
|
||||||
|
#pendingRedirect: PendingRedirect | null = null;
|
||||||
|
#redirectQueue = Promise.resolve();
|
||||||
|
|
||||||
|
constructor(input: {
|
||||||
|
session: RecordingObservationSession;
|
||||||
|
cdp: CdpRunner;
|
||||||
|
tabsApi: ChromeTabsApi;
|
||||||
|
tabId: number;
|
||||||
|
}) {
|
||||||
|
this.#session = input.session;
|
||||||
|
this.#cdp = input.cdp;
|
||||||
|
this.#tabsApi = input.tabsApi;
|
||||||
|
this.#tabId = input.tabId;
|
||||||
|
this.#rootScope = { target: { tabId: input.tabId } };
|
||||||
|
}
|
||||||
|
|
||||||
|
get hasPending(): boolean {
|
||||||
|
return this.#pending.size > 0 || this.#pendingRedirect !== null;
|
||||||
|
}
|
||||||
|
|
||||||
|
async #captureWithRetry(signal?: AbortSignal) {
|
||||||
|
try {
|
||||||
|
return await this.#session.capture(this.#cdp, this.#tabsApi, this.#tabId, signal);
|
||||||
|
} catch (error) {
|
||||||
|
if (signal?.aborted) throw error;
|
||||||
|
await delay(CAPTURE_RETRY_DELAY_MS, signal);
|
||||||
|
return this.#session.capture(this.#cdp, this.#tabsApi, this.#tabId, signal);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
schedule(
|
||||||
|
drafts: RecordingDraftStep[],
|
||||||
|
draftIndex: number,
|
||||||
|
scope: DocumentSettleScope = this.#rootScope,
|
||||||
|
): void {
|
||||||
|
if (scope.target.tabId !== this.#tabId) {
|
||||||
|
throw new Error(
|
||||||
|
`settle scope tab ${scope.target.tabId} does not belong to tab ${this.#tabId}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
const landing = drafts[draftIndex]?.preStateId;
|
||||||
|
for (const [index, pending] of this.#pending) {
|
||||||
|
if (index >= draftIndex) continue;
|
||||||
|
pending.abort.abort();
|
||||||
|
this.#pending.delete(index);
|
||||||
|
if (landing && drafts[index] && !drafts[index].postStateId) {
|
||||||
|
drafts[index].postStateId = landing;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
this.#pending.get(draftIndex)?.abort.abort();
|
||||||
|
const pending = { abort: new AbortController(), scope };
|
||||||
|
this.#pending.set(draftIndex, pending);
|
||||||
|
const task = async () => {
|
||||||
|
if (pending.abort.signal.aborted) return;
|
||||||
|
try {
|
||||||
|
const outcome = await waitForDocumentSettled(this.#cdp, pending.scope, {
|
||||||
|
signal: pending.abort.signal,
|
||||||
|
});
|
||||||
|
if (outcome === "cancelled") return;
|
||||||
|
const observation = await this.#captureWithRetry(pending.abort.signal);
|
||||||
|
if (!pending.abort.signal.aborted && drafts[draftIndex]) {
|
||||||
|
drafts[draftIndex].postStateId = observation.stateId;
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
if (!isAbortError(error)) {
|
||||||
|
console.warn(
|
||||||
|
`[bsk record] post-action observation failed for step ${draftIndex + 1}`,
|
||||||
|
error,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
if (this.#pending.get(draftIndex) === pending) this.#pending.delete(draftIndex);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
this.#queue = this.#queue.then(task, task).catch(() => {});
|
||||||
|
}
|
||||||
|
|
||||||
|
cancel(): void {
|
||||||
|
for (const pending of this.#pending.values()) pending.abort.abort();
|
||||||
|
this.#pending.clear();
|
||||||
|
this.clearRedirect();
|
||||||
|
}
|
||||||
|
|
||||||
|
async flush(): Promise<void> {
|
||||||
|
for (let round = 0; round < 10; round += 1) {
|
||||||
|
const current = this.#queue;
|
||||||
|
await current;
|
||||||
|
if (current === this.#queue) return;
|
||||||
|
}
|
||||||
|
console.warn("[bsk record] settle queue kept growing; using completed observations");
|
||||||
|
}
|
||||||
|
|
||||||
|
async settleTrailing(drafts: RecordingDraftStep[]): Promise<void> {
|
||||||
|
const trailing: RecordingDraftStep[] = [];
|
||||||
|
for (let index = drafts.length - 1; index >= 0; index -= 1) {
|
||||||
|
const draft = drafts[index];
|
||||||
|
if (!draft || draft.postStateId) break;
|
||||||
|
trailing.push(draft);
|
||||||
|
}
|
||||||
|
if (trailing.length === 0) return;
|
||||||
|
try {
|
||||||
|
const observation = await this.#captureWithRetry();
|
||||||
|
for (const draft of trailing) draft.postStateId = observation.stateId;
|
||||||
|
} catch (error) {
|
||||||
|
console.warn("[bsk record] final observation at stop failed", error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
clearRedirect(): void {
|
||||||
|
this.#pendingRedirect?.abort.abort();
|
||||||
|
this.#pendingRedirect = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
scheduleRedirect(drafts: RecordingDraftStep[], url: string): void {
|
||||||
|
this.#pendingRedirect?.abort.abort();
|
||||||
|
const pending: PendingRedirect = { url, abort: new AbortController() };
|
||||||
|
this.#pendingRedirect = pending;
|
||||||
|
const task = () => this.#settleRedirect(drafts, pending);
|
||||||
|
this.#redirectQueue = this.#redirectQueue.then(task, task).catch(() => {});
|
||||||
|
}
|
||||||
|
|
||||||
|
async #settleRedirect(drafts: RecordingDraftStep[], pending: PendingRedirect): Promise<void> {
|
||||||
|
const outcome = await waitForDocumentSettled(this.#cdp, this.#rootScope, {
|
||||||
|
signal: pending.abort.signal,
|
||||||
|
});
|
||||||
|
if (outcome === "cancelled" || this.#pendingRedirect !== pending) return;
|
||||||
|
|
||||||
|
let finalUrl = pending.url;
|
||||||
|
try {
|
||||||
|
finalUrl = (await this.#tabsApi.get(this.#tabId)).url || finalUrl;
|
||||||
|
} catch {
|
||||||
|
// The navigation event URL remains the best available destination.
|
||||||
|
}
|
||||||
|
if (this.#pendingRedirect !== pending) return;
|
||||||
|
this.#pendingRedirect = null;
|
||||||
|
if (
|
||||||
|
!finalUrl ||
|
||||||
|
finalUrl === "about:blank" ||
|
||||||
|
this.#session.cursor.lastSettled?.url === finalUrl
|
||||||
|
) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const last = drafts[drafts.length - 1];
|
||||||
|
if (last?.op === "navigate" && last.url === finalUrl) {
|
||||||
|
if (!last.postStateId) this.schedule(drafts, drafts.length - 1);
|
||||||
|
await this.flush();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const draft: RecordingDraftStep = {
|
||||||
|
op: "navigate",
|
||||||
|
url: finalUrl,
|
||||||
|
pageUrl: finalUrl,
|
||||||
|
cause: "browser",
|
||||||
|
preStateId: this.#session.cursor.lastSettled?.stateId,
|
||||||
|
};
|
||||||
|
drafts.push(draft);
|
||||||
|
const draftIndex = drafts.length - 1;
|
||||||
|
if (draft.preStateId) this.#session.registry.markStep(draft.preStateId, draftIndex + 1);
|
||||||
|
this.schedule(drafts, draftIndex);
|
||||||
|
await this.flush();
|
||||||
|
}
|
||||||
|
|
||||||
|
async flushRedirects(): Promise<void> {
|
||||||
|
for (let round = 0; round < 10; round += 1) {
|
||||||
|
const current = this.#redirectQueue;
|
||||||
|
await current;
|
||||||
|
if (current === this.#redirectQueue) return;
|
||||||
|
}
|
||||||
|
console.warn("[bsk record] redirect queue kept growing; using the final completed landing");
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,56 @@
|
|||||||
|
export interface RecordedStateEntry {
|
||||||
|
id: string;
|
||||||
|
url: string;
|
||||||
|
title?: string;
|
||||||
|
rawVomText: string;
|
||||||
|
truncated: boolean;
|
||||||
|
stepsHere: number[];
|
||||||
|
}
|
||||||
|
|
||||||
|
function stateIdentity(url: string, body: string): string {
|
||||||
|
return `${url}\0${body}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
export class RecordingStateRegistry {
|
||||||
|
readonly #entriesById = new Map<string, RecordedStateEntry>();
|
||||||
|
readonly #idByIdentity = new Map<string, string>();
|
||||||
|
#nextId = 1;
|
||||||
|
|
||||||
|
register(input: {
|
||||||
|
url: string;
|
||||||
|
title?: string;
|
||||||
|
rawVomText: string;
|
||||||
|
truncated?: boolean;
|
||||||
|
}): RecordedStateEntry {
|
||||||
|
const identity = stateIdentity(input.url, input.rawVomText);
|
||||||
|
const existingId = this.#idByIdentity.get(identity);
|
||||||
|
if (existingId) {
|
||||||
|
const existing = this.#entriesById.get(existingId)!;
|
||||||
|
if (!existing.title && input.title) existing.title = input.title;
|
||||||
|
if (input.truncated) existing.truncated = true;
|
||||||
|
return existing;
|
||||||
|
}
|
||||||
|
|
||||||
|
const entry: RecordedStateEntry = {
|
||||||
|
id: `s${this.#nextId}`,
|
||||||
|
url: input.url,
|
||||||
|
...(input.title ? { title: input.title } : {}),
|
||||||
|
rawVomText: input.rawVomText,
|
||||||
|
truncated: input.truncated ?? false,
|
||||||
|
stepsHere: [],
|
||||||
|
};
|
||||||
|
this.#nextId += 1;
|
||||||
|
this.#entriesById.set(entry.id, entry);
|
||||||
|
this.#idByIdentity.set(identity, entry.id);
|
||||||
|
return entry;
|
||||||
|
}
|
||||||
|
|
||||||
|
values(): RecordedStateEntry[] {
|
||||||
|
return [...this.#entriesById.values()];
|
||||||
|
}
|
||||||
|
|
||||||
|
markStep(stateId: string, draftId: number): void {
|
||||||
|
const entry = this.#entriesById.get(stateId);
|
||||||
|
if (entry && !entry.stepsHere.includes(draftId)) entry.stepsHere.push(draftId);
|
||||||
|
}
|
||||||
|
}
|
||||||
+23
-15
@@ -1,8 +1,8 @@
|
|||||||
import type { DraftTraceStep } from "@/transport/types";
|
import type { RecordStepPayload } from "../record-bridge";
|
||||||
import type { RecordStepPayload } from "./record-bridge";
|
import type { RecordingDraftStep } from "./types";
|
||||||
|
|
||||||
export interface RecordingStepBuffer {
|
export interface RecordingStepBuffer {
|
||||||
steps: DraftTraceStep[];
|
steps: RecordingDraftStep[];
|
||||||
currentUrl?: string;
|
currentUrl?: string;
|
||||||
pendingNavigation: boolean;
|
pendingNavigation: boolean;
|
||||||
pendingNavigationDeadline?: number;
|
pendingNavigationDeadline?: number;
|
||||||
@@ -10,25 +10,33 @@ export interface RecordingStepBuffer {
|
|||||||
|
|
||||||
const NAVIGATION_TRIGGER_WINDOW_MS = 3_000;
|
const NAVIGATION_TRIGGER_WINDOW_MS = 3_000;
|
||||||
|
|
||||||
function toDraftStep(payload: RecordStepPayload): DraftTraceStep | null {
|
function toDraftStep(payload: RecordStepPayload): RecordingDraftStep | null {
|
||||||
const pageUrl = payload.page_url;
|
const pageUrl = payload.page_url;
|
||||||
switch (payload.op) {
|
switch (payload.op) {
|
||||||
case "click":
|
case "click":
|
||||||
return payload.target
|
return payload.target
|
||||||
? {
|
? {
|
||||||
op: "click",
|
op: "click",
|
||||||
target: payload.target,
|
captureTarget: payload.target,
|
||||||
...(pageUrl ? { page_url: pageUrl } : {}),
|
...(pageUrl ? { pageUrl } : {}),
|
||||||
|
}
|
||||||
|
: null;
|
||||||
|
case "hover":
|
||||||
|
return payload.target
|
||||||
|
? {
|
||||||
|
op: "hover",
|
||||||
|
captureTarget: payload.target,
|
||||||
|
...(pageUrl ? { pageUrl } : {}),
|
||||||
}
|
}
|
||||||
: null;
|
: null;
|
||||||
case "fill":
|
case "fill":
|
||||||
return payload.target
|
return payload.target
|
||||||
? {
|
? {
|
||||||
op: "fill",
|
op: "fill",
|
||||||
target: payload.target,
|
captureTarget: payload.target,
|
||||||
value: payload.value ?? "",
|
value: payload.value ?? "",
|
||||||
...(payload.redacted ? { redacted: true } : {}),
|
...(payload.redacted ? { redacted: true } : {}),
|
||||||
...(pageUrl ? { page_url: pageUrl } : {}),
|
...(pageUrl ? { pageUrl } : {}),
|
||||||
}
|
}
|
||||||
: null;
|
: null;
|
||||||
case "press":
|
case "press":
|
||||||
@@ -36,19 +44,19 @@ function toDraftStep(payload: RecordStepPayload): DraftTraceStep | null {
|
|||||||
? {
|
? {
|
||||||
op: "press",
|
op: "press",
|
||||||
key: payload.key,
|
key: payload.key,
|
||||||
...(payload.target ? { target: payload.target } : {}),
|
...(payload.target ? { captureTarget: payload.target } : {}),
|
||||||
...(payload.modifiers?.length ? { modifiers: payload.modifiers } : {}),
|
...(payload.modifiers?.length ? { modifiers: payload.modifiers } : {}),
|
||||||
...(pageUrl ? { page_url: pageUrl } : {}),
|
...(pageUrl ? { pageUrl } : {}),
|
||||||
}
|
}
|
||||||
: null;
|
: null;
|
||||||
case "select":
|
case "select":
|
||||||
return payload.target && payload.values
|
return payload.target && payload.values
|
||||||
? {
|
? {
|
||||||
op: "select",
|
op: "select",
|
||||||
target: payload.target,
|
captureTarget: payload.target,
|
||||||
values: payload.values,
|
values: payload.values,
|
||||||
...(payload.labels?.length ? { labels: payload.labels } : {}),
|
...(payload.labels?.length ? { labels: payload.labels } : {}),
|
||||||
...(pageUrl ? { page_url: pageUrl } : {}),
|
...(pageUrl ? { pageUrl } : {}),
|
||||||
}
|
}
|
||||||
: null;
|
: null;
|
||||||
case "navigate":
|
case "navigate":
|
||||||
@@ -61,7 +69,7 @@ function annotateLastStepNavigation(buffer: RecordingStepBuffer, url: string): b
|
|||||||
const step = buffer.steps[i];
|
const step = buffer.steps[i];
|
||||||
if (!step) continue;
|
if (!step) continue;
|
||||||
if (step.op === "click" || step.op === "press" || step.op === "select") {
|
if (step.op === "click" || step.op === "press" || step.op === "select") {
|
||||||
buffer.steps[i] = { ...step, navigated_to: url };
|
buffer.steps[i] = { ...step, navigatedTo: url };
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
break;
|
break;
|
||||||
@@ -88,7 +96,7 @@ export function observeRecordedNavigation(
|
|||||||
buffer.steps.push({
|
buffer.steps.push({
|
||||||
op: "navigate",
|
op: "navigate",
|
||||||
url,
|
url,
|
||||||
page_url: url,
|
pageUrl: url,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
return;
|
return;
|
||||||
@@ -99,7 +107,7 @@ export function observeRecordedNavigation(
|
|||||||
buffer.steps.push({
|
buffer.steps.push({
|
||||||
op: "navigate",
|
op: "navigate",
|
||||||
url,
|
url,
|
||||||
page_url: url,
|
pageUrl: url,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -0,0 +1,76 @@
|
|||||||
|
import type { RenderedRef } from "@browser-skill/vom";
|
||||||
|
import type { TargetDescriptorV3 } from "@/transport/types";
|
||||||
|
import type { CaptureTargetDescriptor } from "../describe-target";
|
||||||
|
import type { IndexedObservationNode, RegisteredObservation } from "./observation-capture";
|
||||||
|
import type { TargetGeometry, TargetMatchHint } from "./types";
|
||||||
|
|
||||||
|
const MATCH_TOLERANCE_PX = 2;
|
||||||
|
|
||||||
|
function close(a: number, b: number): boolean {
|
||||||
|
return Math.abs(a - b) <= MATCH_TOLERANCE_PX;
|
||||||
|
}
|
||||||
|
|
||||||
|
function rectMatches(a: TargetGeometry["rect"], b: TargetGeometry["rect"]): boolean {
|
||||||
|
return close(a.x, b.x) && close(a.y, b.y) && close(a.w, b.w) && close(a.h, b.h);
|
||||||
|
}
|
||||||
|
|
||||||
|
function candidateMatches(candidate: IndexedObservationNode, geometry: TargetGeometry): boolean {
|
||||||
|
return candidate.geometry.rect !== null && rectMatches(geometry.rect, candidate.geometry.rect);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function unmatchedTarget(fallback?: CaptureTargetDescriptor): TargetDescriptorV3 {
|
||||||
|
return {
|
||||||
|
...(fallback?.role ? { role: fallback.role } : {}),
|
||||||
|
...(fallback?.name ? { name: fallback.name } : {}),
|
||||||
|
unmatched: true,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalized(value: string | undefined): string {
|
||||||
|
return value?.replace(/\s+/g, " ").trim().toLowerCase() ?? "";
|
||||||
|
}
|
||||||
|
|
||||||
|
function matchesSemantics(ref: RenderedRef, fallback?: CaptureTargetDescriptor): boolean {
|
||||||
|
const role = normalized(fallback?.role);
|
||||||
|
const name = normalized(fallback?.name);
|
||||||
|
if (!role && !name) return false;
|
||||||
|
if (role && normalized(ref.role) !== role) return false;
|
||||||
|
if (name && normalized(ref.name) !== name) return false;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
function descriptor(ref: RenderedRef): TargetDescriptorV3 {
|
||||||
|
return {
|
||||||
|
ref: ref.ref,
|
||||||
|
...(ref.role ? { role: ref.role } : {}),
|
||||||
|
...(ref.name ? { name: ref.name } : {}),
|
||||||
|
...(ref.ctx ? { ctx: ref.ctx } : {}),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function matchObservationTarget(input: {
|
||||||
|
observation: RegisteredObservation;
|
||||||
|
hint?: TargetMatchHint;
|
||||||
|
fallback?: CaptureTargetDescriptor;
|
||||||
|
}): TargetDescriptorV3 {
|
||||||
|
const frameId = input.hint?.frameId ?? input.observation.rootFrameId;
|
||||||
|
const geometry = input.hint?.geometry;
|
||||||
|
if (geometry) {
|
||||||
|
const matches = input.observation.index
|
||||||
|
.candidates(frameId, geometry.tag)
|
||||||
|
.filter((candidate) => candidate.ref && candidateMatches(candidate, geometry));
|
||||||
|
if (matches.length === 1) return descriptor(matches[0]!.ref!);
|
||||||
|
const semanticMatches = matches.filter(
|
||||||
|
(candidate) => candidate.ref && matchesSemantics(candidate.ref, input.fallback),
|
||||||
|
);
|
||||||
|
if (semanticMatches.length === 1) return descriptor(semanticMatches[0]!.ref!);
|
||||||
|
return unmatchedTarget(input.fallback);
|
||||||
|
}
|
||||||
|
|
||||||
|
const semanticMatches = input.observation.index
|
||||||
|
.refs(frameId)
|
||||||
|
.filter((ref) => matchesSemantics(ref, input.fallback));
|
||||||
|
return semanticMatches.length === 1
|
||||||
|
? descriptor(semanticMatches[0]!)
|
||||||
|
: unmatchedTarget(input.fallback);
|
||||||
|
}
|
||||||
@@ -0,0 +1,87 @@
|
|||||||
|
import {
|
||||||
|
type StepV3,
|
||||||
|
type StopReason,
|
||||||
|
TRACE_VERSION_V3,
|
||||||
|
type TraceStateV3,
|
||||||
|
type TraceV3,
|
||||||
|
VOM_FORMAT_VERSION,
|
||||||
|
} from "@/transport/types";
|
||||||
|
import { resolveDraftStartUrl } from "./draft-policy";
|
||||||
|
import type { RecordedStateEntry, RecordingStateRegistry } from "./state-registry";
|
||||||
|
import { reduceTraceStepsV3 } from "./trace-reducer-v3";
|
||||||
|
import { formatTraceStateBody } from "./trace-state-body";
|
||||||
|
import type { RecordingDraftStep, StepAnnotation } from "./types";
|
||||||
|
|
||||||
|
function publishedEntries(registry: RecordingStateRegistry, steps: StepV3[]): RecordedStateEntry[] {
|
||||||
|
const entries = registry.values();
|
||||||
|
if (steps.length === 0) return entries.slice(0, 1);
|
||||||
|
const referenced = new Set(steps.flatMap((step) => [step.state, step.result.state]));
|
||||||
|
return entries.filter((entry) => referenced.has(entry.id));
|
||||||
|
}
|
||||||
|
|
||||||
|
function remapDraftIds(draftIds: number[], stepIdByDraftId: Map<number, number>): number[] {
|
||||||
|
return [
|
||||||
|
...new Set(
|
||||||
|
draftIds.flatMap((id) => {
|
||||||
|
const stepId = stepIdByDraftId.get(id);
|
||||||
|
return stepId === undefined ? [] : [stepId];
|
||||||
|
}),
|
||||||
|
),
|
||||||
|
].sort((a, b) => a - b);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function buildTraceV3(input: {
|
||||||
|
registry: RecordingStateRegistry;
|
||||||
|
drafts: RecordingDraftStep[];
|
||||||
|
annotations?: StepAnnotation[];
|
||||||
|
startedAt: string;
|
||||||
|
purpose?: string;
|
||||||
|
startUrl?: string;
|
||||||
|
stoppedBy: StopReason;
|
||||||
|
bskVersion: string;
|
||||||
|
}): TraceV3 {
|
||||||
|
const reduced = reduceTraceStepsV3(input.drafts);
|
||||||
|
const entries = publishedEntries(input.registry, reduced.steps);
|
||||||
|
const publishedId = new Map(entries.map((entry, index) => [entry.id, `s${index + 1}`]));
|
||||||
|
const annotationsByState = new Map<string, StepAnnotation[]>();
|
||||||
|
for (const annotation of input.annotations ?? []) {
|
||||||
|
const bucket = annotationsByState.get(annotation.stateId) ?? [];
|
||||||
|
bucket.push(annotation);
|
||||||
|
annotationsByState.set(annotation.stateId, bucket);
|
||||||
|
}
|
||||||
|
const steps = reduced.steps.map((step) => ({
|
||||||
|
...step,
|
||||||
|
state: publishedId.get(step.state) ?? step.state,
|
||||||
|
result: { state: publishedId.get(step.result.state) ?? step.result.state },
|
||||||
|
}));
|
||||||
|
const states: TraceStateV3[] = entries.map((entry) => {
|
||||||
|
const id = publishedId.get(entry.id) ?? entry.id;
|
||||||
|
return {
|
||||||
|
id,
|
||||||
|
url: entry.url,
|
||||||
|
...(entry.title ? { title: entry.title } : {}),
|
||||||
|
body: formatTraceStateBody({
|
||||||
|
stateId: id,
|
||||||
|
url: entry.url,
|
||||||
|
title: entry.title,
|
||||||
|
stepIds: remapDraftIds(entry.stepsHere, reduced.stepIdByDraftId),
|
||||||
|
vomText: entry.rawVomText,
|
||||||
|
annotations: annotationsByState.get(entry.id) ?? [],
|
||||||
|
stepIdByDraftId: reduced.stepIdByDraftId,
|
||||||
|
}),
|
||||||
|
...(entry.truncated ? { truncated: true } : {}),
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
return {
|
||||||
|
version: TRACE_VERSION_V3,
|
||||||
|
...(input.purpose ? { purpose: input.purpose } : {}),
|
||||||
|
recorded_at: new Date().toISOString(),
|
||||||
|
started_at: input.startedAt,
|
||||||
|
stopped_by: input.stoppedBy,
|
||||||
|
entry: { start_url: resolveDraftStartUrl(input.drafts, input.startUrl, states[0]?.url) },
|
||||||
|
recorder: { bsk: input.bskVersion, vom: VOM_FORMAT_VERSION },
|
||||||
|
states,
|
||||||
|
steps,
|
||||||
|
};
|
||||||
|
}
|
||||||
+66
-65
@@ -1,31 +1,10 @@
|
|||||||
import type { DraftTraceStep, PageRef, SelectedOption, Step } from "@/transport/types";
|
import type { PageRefV2, SelectedOptionV2, StepV2, TraceV2 } from "@/transport/types";
|
||||||
|
import { resolveDraftStartUrl, shouldIncludeDraft } from "./draft-policy";
|
||||||
const CLIPBOARD_KEYS = new Set(["a", "c", "v", "x", "A", "C", "V", "X"]);
|
import type { RecordingDraftStep } from "./types";
|
||||||
const MODIFIER_ONLY_KEYS = new Set(["Meta", "Control", "Alt", "Shift", "OS", "Hyper", "Super"]);
|
|
||||||
|
|
||||||
export function shouldRecordPress(
|
|
||||||
key: string,
|
|
||||||
modifiers?: Array<"alt" | "ctrl" | "meta" | "shift">,
|
|
||||||
): boolean {
|
|
||||||
if (MODIFIER_ONLY_KEYS.has(key)) return false;
|
|
||||||
const mods = modifiers ?? [];
|
|
||||||
const hasCtrlOrMeta = mods.includes("ctrl") || mods.includes("meta");
|
|
||||||
if (hasCtrlOrMeta && CLIPBOARD_KEYS.has(key)) return false;
|
|
||||||
if (key === "Enter" || key === "Escape") return true;
|
|
||||||
// Drop bare character typing — FillSession already records the value.
|
|
||||||
if (key.length === 1 && !hasCtrlOrMeta && !mods.includes("alt")) return false;
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
function shouldIncludeDraft(step: DraftTraceStep): boolean {
|
|
||||||
if (step.op === "fill" && !(step.value ?? "").trim() && !step.redacted) return false;
|
|
||||||
if (step.op === "press" && !shouldRecordPress(step.key, step.modifiers)) return false;
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Collapse consecutive navigations to the last hop. */
|
/** Collapse consecutive navigations to the last hop. */
|
||||||
function collapseNavigations(steps: DraftTraceStep[]): DraftTraceStep[] {
|
function collapseNavigations(steps: RecordingDraftStep[]): RecordingDraftStep[] {
|
||||||
const out: DraftTraceStep[] = [];
|
const out: RecordingDraftStep[] = [];
|
||||||
for (const step of steps) {
|
for (const step of steps) {
|
||||||
const prev = out[out.length - 1];
|
const prev = out[out.length - 1];
|
||||||
if (step.op === "navigate" && prev?.op === "navigate") {
|
if (step.op === "navigate" && prev?.op === "navigate") {
|
||||||
@@ -37,7 +16,7 @@ function collapseNavigations(steps: DraftTraceStep[]): DraftTraceStep[] {
|
|||||||
return out;
|
return out;
|
||||||
}
|
}
|
||||||
|
|
||||||
function collectUrls(steps: DraftTraceStep[], startUrl?: string): string[] {
|
function collectUrls(steps: RecordingDraftStep[], startUrl?: string): string[] {
|
||||||
const urls: string[] = [];
|
const urls: string[] = [];
|
||||||
if (startUrl) urls.push(startUrl);
|
if (startUrl) urls.push(startUrl);
|
||||||
for (const step of steps) {
|
for (const step of steps) {
|
||||||
@@ -45,8 +24,8 @@ function collectUrls(steps: DraftTraceStep[], startUrl?: string): string[] {
|
|||||||
urls.push(step.url);
|
urls.push(step.url);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
if ("page_url" in step && step.page_url) urls.push(step.page_url);
|
if (step.pageUrl) urls.push(step.pageUrl);
|
||||||
if ("navigated_to" in step && step.navigated_to) urls.push(step.navigated_to);
|
if ("navigatedTo" in step && step.navigatedTo) urls.push(step.navigatedTo);
|
||||||
}
|
}
|
||||||
const seen = new Set<string>();
|
const seen = new Set<string>();
|
||||||
const unique: string[] = [];
|
const unique: string[] = [];
|
||||||
@@ -59,9 +38,9 @@ function collectUrls(steps: DraftTraceStep[], startUrl?: string): string[] {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function buildPageRegistry(
|
function buildPageRegistry(
|
||||||
steps: DraftTraceStep[],
|
steps: RecordingDraftStep[],
|
||||||
startUrl?: string,
|
startUrl?: string,
|
||||||
): { pages: PageRef[]; urlToId: Map<string, string> } {
|
): { pages: PageRefV2[]; urlToId: Map<string, string> } {
|
||||||
const urls = collectUrls(steps, startUrl);
|
const urls = collectUrls(steps, startUrl);
|
||||||
const urlToId = new Map<string, string>();
|
const urlToId = new Map<string, string>();
|
||||||
const pages = urls.map((url, index) => {
|
const pages = urls.map((url, index) => {
|
||||||
@@ -82,28 +61,28 @@ function pageIdFor(
|
|||||||
return urlToId.values().next().value ?? "p1";
|
return urlToId.values().next().value ?? "p1";
|
||||||
}
|
}
|
||||||
|
|
||||||
function pageUrlForDraft(step: DraftTraceStep, fallbackUrl?: string): string | undefined {
|
function pageUrlForDraft(step: RecordingDraftStep, fallbackUrl?: string): string | undefined {
|
||||||
if (step.op === "navigate") return step.page_url ?? step.url;
|
if (step.op === "navigate") return step.pageUrl ?? step.url;
|
||||||
if ("page_url" in step && step.page_url) return step.page_url;
|
if (step.pageUrl) return step.pageUrl;
|
||||||
return fallbackUrl;
|
return fallbackUrl;
|
||||||
}
|
}
|
||||||
|
|
||||||
function effectForNavigation(
|
function effectForNavigation(
|
||||||
navigatedTo: string | undefined,
|
navigatedTo: string | undefined,
|
||||||
urlToId: Map<string, string>,
|
urlToId: Map<string, string>,
|
||||||
): Step["effect"] {
|
): StepV2["effect"] {
|
||||||
if (!navigatedTo) return undefined;
|
if (!navigatedTo) return undefined;
|
||||||
const pageId = urlToId.get(navigatedTo);
|
const pageId = urlToId.get(navigatedTo);
|
||||||
if (!pageId) return undefined;
|
if (!pageId) return undefined;
|
||||||
return { navigated_to: pageId };
|
return { navigated_to: pageId };
|
||||||
}
|
}
|
||||||
|
|
||||||
function withEffect(step: Step, effect: Step["effect"]): Step {
|
function withEffect(step: StepV2, effect: StepV2["effect"]): StepV2 {
|
||||||
if (!effect) return step;
|
if (!effect) return step;
|
||||||
return { ...step, effect };
|
return { ...step, effect };
|
||||||
}
|
}
|
||||||
|
|
||||||
function toSelection(values: string[], labels?: string[]): SelectedOption[] {
|
function toSelection(values: string[], labels?: string[]): SelectedOptionV2[] {
|
||||||
return values.map((value, index) => ({
|
return values.map((value, index) => ({
|
||||||
value,
|
value,
|
||||||
...(labels?.[index] ? { label: labels[index] } : {}),
|
...(labels?.[index] ? { label: labels[index] } : {}),
|
||||||
@@ -111,11 +90,11 @@ function toSelection(values: string[], labels?: string[]): SelectedOption[] {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function toV2Step(
|
function toV2Step(
|
||||||
step: DraftTraceStep,
|
step: RecordingDraftStep,
|
||||||
id: number,
|
id: number,
|
||||||
urlToId: Map<string, string>,
|
urlToId: Map<string, string>,
|
||||||
fallbackUrl?: string,
|
fallbackUrl?: string,
|
||||||
): Step | null {
|
): StepV2 | null {
|
||||||
if (!shouldIncludeDraft(step)) return null;
|
if (!shouldIncludeDraft(step)) return null;
|
||||||
|
|
||||||
const pageUrl = pageUrlForDraft(step, fallbackUrl);
|
const pageUrl = pageUrlForDraft(step, fallbackUrl);
|
||||||
@@ -130,21 +109,31 @@ function toV2Step(
|
|||||||
to: step.url,
|
to: step.url,
|
||||||
};
|
};
|
||||||
case "click":
|
case "click":
|
||||||
|
if (!step.captureTarget) return null;
|
||||||
return withEffect(
|
return withEffect(
|
||||||
{
|
{
|
||||||
op: "click",
|
op: "click",
|
||||||
id,
|
id,
|
||||||
page,
|
page,
|
||||||
target: step.target,
|
target: step.captureTarget,
|
||||||
},
|
},
|
||||||
effectForNavigation(step.navigated_to, urlToId),
|
effectForNavigation(step.navigatedTo, urlToId),
|
||||||
);
|
);
|
||||||
|
case "hover":
|
||||||
|
if (!step.captureTarget) return null;
|
||||||
|
return {
|
||||||
|
op: "hover",
|
||||||
|
id,
|
||||||
|
page,
|
||||||
|
target: step.captureTarget,
|
||||||
|
};
|
||||||
case "fill":
|
case "fill":
|
||||||
|
if (!step.captureTarget) return null;
|
||||||
return {
|
return {
|
||||||
op: "fill",
|
op: "fill",
|
||||||
id,
|
id,
|
||||||
page,
|
page,
|
||||||
target: step.target,
|
target: step.captureTarget,
|
||||||
value: step.value,
|
value: step.value,
|
||||||
...(step.redacted ? { redacted: true } : {}),
|
...(step.redacted ? { redacted: true } : {}),
|
||||||
};
|
};
|
||||||
@@ -155,44 +144,47 @@ function toV2Step(
|
|||||||
id,
|
id,
|
||||||
page,
|
page,
|
||||||
key: step.key,
|
key: step.key,
|
||||||
...(step.target ? { target: step.target } : {}),
|
...(step.captureTarget ? { target: step.captureTarget } : {}),
|
||||||
...(step.modifiers?.length ? { modifiers: step.modifiers } : {}),
|
...(step.modifiers?.length ? { modifiers: step.modifiers } : {}),
|
||||||
},
|
},
|
||||||
effectForNavigation(step.navigated_to, urlToId),
|
effectForNavigation(step.navigatedTo, urlToId),
|
||||||
);
|
);
|
||||||
case "select":
|
case "select":
|
||||||
|
if (!step.captureTarget) return null;
|
||||||
return withEffect(
|
return withEffect(
|
||||||
{
|
{
|
||||||
op: "select",
|
op: "select",
|
||||||
id,
|
id,
|
||||||
page,
|
page,
|
||||||
target: step.target,
|
target: step.captureTarget,
|
||||||
selection: toSelection(step.values, step.labels),
|
selection: toSelection(step.values, step.labels),
|
||||||
},
|
},
|
||||||
effectForNavigation(step.navigated_to, urlToId),
|
effectForNavigation(step.navigatedTo, urlToId),
|
||||||
);
|
);
|
||||||
|
case "scroll":
|
||||||
|
return null;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface ReducedTrace {
|
interface ReducedTrace {
|
||||||
pages: PageRef[];
|
pages: PageRefV2[];
|
||||||
steps: Step[];
|
steps: StepV2[];
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Compile capture drafts into record-only trace v2 steps.
|
* Compile capture drafts into record-only trace v2 steps.
|
||||||
* Variable inputs are NOT classified here — executing agents infer that at run time.
|
* Variable inputs are NOT classified here — executing agents infer that at run time.
|
||||||
*/
|
*/
|
||||||
export function reduceTraceSteps(steps: DraftTraceStep[], startUrl?: string): ReducedTrace {
|
function reduceTraceSteps(steps: RecordingDraftStep[], startUrl?: string): ReducedTrace {
|
||||||
const collapsed = collapseNavigations(steps);
|
const collapsed = collapseNavigations(steps);
|
||||||
const { pages, urlToId } = buildPageRegistry(collapsed, startUrl);
|
const { pages, urlToId } = buildPageRegistry(collapsed, startUrl);
|
||||||
const out: Step[] = [];
|
const out: StepV2[] = [];
|
||||||
let id = 1;
|
let id = 1;
|
||||||
let lastUrl = startUrl;
|
let lastUrl = startUrl;
|
||||||
for (const draft of collapsed) {
|
for (const draft of collapsed) {
|
||||||
if (draft.op === "navigate") lastUrl = draft.url;
|
if (draft.op === "navigate") lastUrl = draft.url;
|
||||||
else if ("navigated_to" in draft && draft.navigated_to) lastUrl = draft.navigated_to;
|
else if ("navigatedTo" in draft && draft.navigatedTo) lastUrl = draft.navigatedTo;
|
||||||
else if ("page_url" in draft && draft.page_url) lastUrl = draft.page_url;
|
else if (draft.pageUrl) lastUrl = draft.pageUrl;
|
||||||
const step = toV2Step(draft, id, urlToId, lastUrl);
|
const step = toV2Step(draft, id, urlToId, lastUrl);
|
||||||
if (!step) continue;
|
if (!step) continue;
|
||||||
out.push(step);
|
out.push(step);
|
||||||
@@ -201,18 +193,27 @@ export function reduceTraceSteps(steps: DraftTraceStep[], startUrl?: string): Re
|
|||||||
return { pages, steps: out };
|
return { pages, steps: out };
|
||||||
}
|
}
|
||||||
|
|
||||||
export function resolveTraceStartUrl(
|
function resolveTraceStartUrl(
|
||||||
drafts: DraftTraceStep[],
|
drafts: RecordingDraftStep[],
|
||||||
startUrl?: string,
|
startUrl?: string,
|
||||||
pages?: PageRef[],
|
pages?: PageRefV2[],
|
||||||
): string {
|
): string {
|
||||||
if (startUrl) return startUrl;
|
return resolveDraftStartUrl(drafts, startUrl, pages?.[0]?.url);
|
||||||
const navigate = drafts.find((step): step is Extract<DraftTraceStep, { op: "navigate" }> => {
|
}
|
||||||
return step.op === "navigate";
|
|
||||||
});
|
export function buildTraceV2(input: {
|
||||||
if (navigate) return navigate.url;
|
steps: RecordingDraftStep[];
|
||||||
for (const draft of drafts) {
|
startedAt: string;
|
||||||
if ("page_url" in draft && draft.page_url) return draft.page_url;
|
startUrl?: string;
|
||||||
}
|
purpose?: string;
|
||||||
return pages?.[0]?.url ?? "about:blank";
|
}): TraceV2 {
|
||||||
|
const { pages, steps } = reduceTraceSteps(input.steps, input.startUrl);
|
||||||
|
return {
|
||||||
|
recorded_at: new Date().toISOString(),
|
||||||
|
started_at: input.startedAt,
|
||||||
|
...(input.purpose ? { purpose: input.purpose } : {}),
|
||||||
|
entry: { start_url: resolveTraceStartUrl(input.steps, input.startUrl, pages) },
|
||||||
|
pages,
|
||||||
|
steps,
|
||||||
|
};
|
||||||
}
|
}
|
||||||
@@ -0,0 +1,129 @@
|
|||||||
|
import type { NavigationCause, StepV3 } from "@/transport/types";
|
||||||
|
import { shouldIncludeDraft } from "./draft-policy";
|
||||||
|
import { unmatchedTarget } from "./target-matcher";
|
||||||
|
import type { RecordingDraftStep } from "./types";
|
||||||
|
|
||||||
|
interface CollapsedDraft {
|
||||||
|
draft: RecordingDraftStep;
|
||||||
|
draftIds: number[];
|
||||||
|
}
|
||||||
|
|
||||||
|
const REDIRECT_QUALIFIERS = new Set(["client_redirect", "server_redirect"]);
|
||||||
|
const TRANSITION_CAUSES: Record<string, NavigationCause> = {
|
||||||
|
typed: "user_typed",
|
||||||
|
generated: "user_typed",
|
||||||
|
keyword: "user_typed",
|
||||||
|
keyword_generated: "user_typed",
|
||||||
|
link: "link",
|
||||||
|
form_submit: "form_submit",
|
||||||
|
reload: "reload",
|
||||||
|
auto_bookmark: "browser",
|
||||||
|
start_page: "browser",
|
||||||
|
};
|
||||||
|
|
||||||
|
function isRedirect(step: Extract<RecordingDraftStep, { op: "navigate" }>): boolean {
|
||||||
|
return (step.transitionQualifiers ?? []).some((qualifier) => REDIRECT_QUALIFIERS.has(qualifier));
|
||||||
|
}
|
||||||
|
|
||||||
|
function collapseRedirects(steps: RecordingDraftStep[]): CollapsedDraft[] {
|
||||||
|
const output: CollapsedDraft[] = [];
|
||||||
|
steps.forEach((step, index) => {
|
||||||
|
const previous = output[output.length - 1];
|
||||||
|
if (step.op === "navigate" && previous?.draft.op === "navigate" && isRedirect(step)) {
|
||||||
|
previous.draft = {
|
||||||
|
...previous.draft,
|
||||||
|
url: step.url,
|
||||||
|
postStateId: step.postStateId ?? previous.draft.postStateId,
|
||||||
|
};
|
||||||
|
previous.draftIds.push(index + 1);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
output.push({ draft: { ...step }, draftIds: [index + 1] });
|
||||||
|
});
|
||||||
|
return output;
|
||||||
|
}
|
||||||
|
|
||||||
|
function navigationCause(step: Extract<RecordingDraftStep, { op: "navigate" }>): NavigationCause {
|
||||||
|
if (step.cause) return step.cause;
|
||||||
|
const qualifiers = step.transitionQualifiers ?? [];
|
||||||
|
if (qualifiers.includes("forward_back")) return "history";
|
||||||
|
if (qualifiers.includes("from_address_bar")) return "user_typed";
|
||||||
|
return TRANSITION_CAUSES[step.transitionType ?? ""] ?? "browser";
|
||||||
|
}
|
||||||
|
|
||||||
|
function selection(values: string[], labels?: string[]): Array<{ value: string; label?: string }> {
|
||||||
|
return values.map((value, index) => ({
|
||||||
|
value,
|
||||||
|
...(labels?.[index] ? { label: labels[index] } : {}),
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
function reduceDraft(draft: RecordingDraftStep, id: number): StepV3 | null {
|
||||||
|
if (!shouldIncludeDraft(draft)) return null;
|
||||||
|
const state = draft.preStateId ?? draft.postStateId;
|
||||||
|
const resultState = draft.postStateId ?? draft.preStateId;
|
||||||
|
if (!state || !resultState) return null;
|
||||||
|
const common = { id, state, result: { state: resultState } };
|
||||||
|
|
||||||
|
switch (draft.op) {
|
||||||
|
case "navigate":
|
||||||
|
return { op: "navigate", ...common, to: draft.url, cause: navigationCause(draft) };
|
||||||
|
case "click":
|
||||||
|
return {
|
||||||
|
op: "click",
|
||||||
|
...common,
|
||||||
|
target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget),
|
||||||
|
};
|
||||||
|
case "hover":
|
||||||
|
return {
|
||||||
|
op: "hover",
|
||||||
|
...common,
|
||||||
|
target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget),
|
||||||
|
};
|
||||||
|
case "fill":
|
||||||
|
return {
|
||||||
|
op: "fill",
|
||||||
|
...common,
|
||||||
|
target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget),
|
||||||
|
value: draft.value,
|
||||||
|
commit: draft.commit ?? "blur",
|
||||||
|
...(draft.redacted ? { redacted: true } : {}),
|
||||||
|
};
|
||||||
|
case "press":
|
||||||
|
return {
|
||||||
|
op: "press",
|
||||||
|
...common,
|
||||||
|
key: draft.key,
|
||||||
|
...(draft.captureTarget || draft.matchedTarget
|
||||||
|
? { target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget) }
|
||||||
|
: {}),
|
||||||
|
...(draft.modifiers?.length ? { modifiers: draft.modifiers } : {}),
|
||||||
|
};
|
||||||
|
case "select":
|
||||||
|
return {
|
||||||
|
op: "select",
|
||||||
|
...common,
|
||||||
|
target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget),
|
||||||
|
selection: selection(draft.values, draft.labels),
|
||||||
|
};
|
||||||
|
case "scroll":
|
||||||
|
return { op: "scroll", ...common };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ReducedTraceV3 {
|
||||||
|
steps: StepV3[];
|
||||||
|
stepIdByDraftId: Map<number, number>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function reduceTraceStepsV3(steps: RecordingDraftStep[]): ReducedTraceV3 {
|
||||||
|
const output: StepV3[] = [];
|
||||||
|
const stepIdByDraftId = new Map<number, number>();
|
||||||
|
for (const { draft, draftIds } of collapseRedirects(steps)) {
|
||||||
|
const step = reduceDraft(draft, output.length + 1);
|
||||||
|
if (!step) continue;
|
||||||
|
output.push(step);
|
||||||
|
for (const draftId of draftIds) stepIdByDraftId.set(draftId, step.id);
|
||||||
|
}
|
||||||
|
return { steps: output, stepIdByDraftId };
|
||||||
|
}
|
||||||
@@ -0,0 +1,39 @@
|
|||||||
|
import type { StepAnnotation } from "./types";
|
||||||
|
|
||||||
|
function annotationText(annotation: StepAnnotation, stepId: number): string {
|
||||||
|
return ` ⟵ step ${stepId}: ${annotation.op}${annotation.detail ? `: ${annotation.detail}` : ""}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function formatTraceStateBody(input: {
|
||||||
|
stateId: string;
|
||||||
|
url: string;
|
||||||
|
title?: string;
|
||||||
|
stepIds: number[];
|
||||||
|
vomText: string;
|
||||||
|
annotations: StepAnnotation[];
|
||||||
|
stepIdByDraftId: Map<number, number>;
|
||||||
|
}): string {
|
||||||
|
const lines = ["# bsk-observation 1", `state: ${JSON.stringify(input.stateId)}`];
|
||||||
|
lines.push(`url: ${JSON.stringify(input.url)}`);
|
||||||
|
if (input.title) lines.push(`title: ${JSON.stringify(input.title)}`);
|
||||||
|
if (input.stepIds.length > 0) lines.push(`steps_here: [${input.stepIds.join(", ")}]`);
|
||||||
|
lines.push("---");
|
||||||
|
|
||||||
|
const byLine = new Map<number, Array<{ annotation: StepAnnotation; stepId: number }>>();
|
||||||
|
for (const annotation of input.annotations) {
|
||||||
|
const stepId = input.stepIdByDraftId.get(annotation.draftId);
|
||||||
|
if (stepId === undefined) continue;
|
||||||
|
const bucket = byLine.get(annotation.line) ?? [];
|
||||||
|
bucket.push({ annotation, stepId });
|
||||||
|
byLine.set(annotation.line, bucket);
|
||||||
|
}
|
||||||
|
|
||||||
|
input.vomText.split("\n").forEach((bodyLine, lineIndex) => {
|
||||||
|
let line = bodyLine;
|
||||||
|
for (const item of byLine.get(lineIndex) ?? []) {
|
||||||
|
line += annotationText(item.annotation, item.stepId);
|
||||||
|
}
|
||||||
|
lines.push(line);
|
||||||
|
});
|
||||||
|
return `${lines.join("\n")}\n`;
|
||||||
|
}
|
||||||
@@ -0,0 +1,83 @@
|
|||||||
|
import type { CaptureTargetDescriptor } from "@/lib/describe-target";
|
||||||
|
import type {
|
||||||
|
FillCommit,
|
||||||
|
KeyModifier,
|
||||||
|
NavigationCause,
|
||||||
|
StepV3,
|
||||||
|
TargetDescriptorV3,
|
||||||
|
} from "@/transport/types";
|
||||||
|
|
||||||
|
export interface TargetGeometry {
|
||||||
|
/** Top-level viewport-relative CSS pixels, as defined by the geometry module. */
|
||||||
|
rect: { x: number; y: number; w: number; h: number };
|
||||||
|
tag: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface TargetMatchHint {
|
||||||
|
geometry?: TargetGeometry;
|
||||||
|
/** Missing means the current top frame, never an unrestricted frame search. */
|
||||||
|
frameId?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface StepAnnotation {
|
||||||
|
draftId: number;
|
||||||
|
op: StepV3["op"];
|
||||||
|
line: number;
|
||||||
|
stateId: string;
|
||||||
|
detail?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface DraftStateLink {
|
||||||
|
pageUrl?: string;
|
||||||
|
preStateId?: string;
|
||||||
|
postStateId?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface DraftTarget {
|
||||||
|
captureTarget?: CaptureTargetDescriptor;
|
||||||
|
targetHint?: TargetMatchHint;
|
||||||
|
matchedTarget?: TargetDescriptorV3;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface DraftNavigationEffect {
|
||||||
|
navigatedTo?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export type RecordingDraftStep =
|
||||||
|
| ({ op: "click" } & DraftStateLink & DraftTarget & DraftNavigationEffect)
|
||||||
|
| ({ op: "hover" } & DraftStateLink & DraftTarget)
|
||||||
|
| ({
|
||||||
|
op: "fill";
|
||||||
|
value: string;
|
||||||
|
commit?: FillCommit;
|
||||||
|
redacted?: boolean;
|
||||||
|
} & DraftStateLink &
|
||||||
|
DraftTarget &
|
||||||
|
DraftNavigationEffect)
|
||||||
|
| ({
|
||||||
|
op: "press";
|
||||||
|
key: string;
|
||||||
|
modifiers?: KeyModifier[];
|
||||||
|
} & DraftStateLink &
|
||||||
|
DraftTarget &
|
||||||
|
DraftNavigationEffect)
|
||||||
|
| ({
|
||||||
|
op: "select";
|
||||||
|
values: string[];
|
||||||
|
labels?: string[];
|
||||||
|
} & DraftStateLink &
|
||||||
|
DraftTarget &
|
||||||
|
DraftNavigationEffect)
|
||||||
|
| ({ op: "scroll" } & DraftStateLink)
|
||||||
|
| ({
|
||||||
|
op: "navigate";
|
||||||
|
url: string;
|
||||||
|
cause?: NavigationCause;
|
||||||
|
transitionType?: string;
|
||||||
|
transitionQualifiers?: string[];
|
||||||
|
} & DraftStateLink);
|
||||||
|
|
||||||
|
export type TargetedRecordingDraft = Extract<
|
||||||
|
RecordingDraftStep,
|
||||||
|
{ op: "click" | "hover" | "fill" | "press" | "select" }
|
||||||
|
>;
|
||||||
@@ -44,3 +44,53 @@ describe("chromeAgentWindowApi.ensureActiveTab", () => {
|
|||||||
expect(update).not.toHaveBeenCalled();
|
expect(update).not.toHaveBeenCalled();
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe("chromeAgentWindowApi.create", () => {
|
||||||
|
const create = vi.fn();
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
vi.stubGlobal("chrome", {
|
||||||
|
windows: { create },
|
||||||
|
});
|
||||||
|
create.mockReset();
|
||||||
|
create.mockResolvedValue({ id: 100 });
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
vi.unstubAllGlobals();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("focuses Agent Windows by default", async () => {
|
||||||
|
await chromeAgentWindowApi.create(AGENT_WINDOW_HOME);
|
||||||
|
|
||||||
|
expect(create).toHaveBeenCalledWith({
|
||||||
|
type: "normal",
|
||||||
|
focused: true,
|
||||||
|
url: AGENT_WINDOW_HOME,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("can create an Agent Window without stealing focus", async () => {
|
||||||
|
await chromeAgentWindowApi.create(AGENT_WINDOW_HOME, { focused: false });
|
||||||
|
|
||||||
|
expect(create).toHaveBeenCalledWith({
|
||||||
|
type: "normal",
|
||||||
|
focused: false,
|
||||||
|
url: AGENT_WINDOW_HOME,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("passes an optional window size through the options object", async () => {
|
||||||
|
await chromeAgentWindowApi.create(AGENT_WINDOW_HOME, {
|
||||||
|
size: { width: 1280, height: 800 },
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(create).toHaveBeenCalledWith({
|
||||||
|
type: "normal",
|
||||||
|
focused: true,
|
||||||
|
url: AGENT_WINDOW_HOME,
|
||||||
|
width: 1280,
|
||||||
|
height: 800,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,58 @@
|
|||||||
|
import { describe, expect, it, vi } from "vitest";
|
||||||
|
import { createDisconnectCleanup } from "../disconnect-cleanup";
|
||||||
|
import { SessionManager } from "../manager";
|
||||||
|
|
||||||
|
describe("disconnect session cleanup", () => {
|
||||||
|
it("stops every local session through the safe session-stop path", async () => {
|
||||||
|
const remove = vi.fn(async () => {});
|
||||||
|
let nextWindowId = 100;
|
||||||
|
const manager = new SessionManager({
|
||||||
|
agentWindow: {
|
||||||
|
create: vi.fn(async () => nextWindowId++),
|
||||||
|
ensureActiveTab: vi.fn(async () => {}),
|
||||||
|
remove,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
await manager.start("aa11");
|
||||||
|
await manager.start("bb22");
|
||||||
|
const detachSession = vi.fn(async () => {});
|
||||||
|
const onSessionsChanged = vi.fn();
|
||||||
|
const cleanup = createDisconnectCleanup({
|
||||||
|
manager,
|
||||||
|
sessionStopDeps: { cdp: { detachSession } },
|
||||||
|
onSessionsChanged,
|
||||||
|
});
|
||||||
|
|
||||||
|
const report = await cleanup();
|
||||||
|
|
||||||
|
expect(report).toEqual({ stoppedSessionIds: ["aa11", "bb22"], failures: [] });
|
||||||
|
expect(manager.list()).toEqual([]);
|
||||||
|
expect(remove).toHaveBeenCalledTimes(2);
|
||||||
|
expect(detachSession).toHaveBeenCalledWith("aa11");
|
||||||
|
expect(detachSession).toHaveBeenCalledWith("bb22");
|
||||||
|
expect(onSessionsChanged).toHaveBeenCalledTimes(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("coalesces overlapping disconnect notifications", async () => {
|
||||||
|
let releaseRemove: () => void = () => {};
|
||||||
|
const removeGate = new Promise<void>((resolve) => {
|
||||||
|
releaseRemove = resolve;
|
||||||
|
});
|
||||||
|
const manager = new SessionManager({
|
||||||
|
agentWindow: {
|
||||||
|
create: vi.fn(async () => 100),
|
||||||
|
ensureActiveTab: vi.fn(async () => {}),
|
||||||
|
remove: vi.fn(() => removeGate),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
await manager.start("aa11");
|
||||||
|
const cleanup = createDisconnectCleanup({ manager });
|
||||||
|
|
||||||
|
const first = cleanup();
|
||||||
|
const second = cleanup();
|
||||||
|
expect(second).toBe(first);
|
||||||
|
releaseRemove();
|
||||||
|
|
||||||
|
await expect(first).resolves.toEqual({ stoppedSessionIds: ["aa11"], failures: [] });
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -1,5 +1,5 @@
|
|||||||
import { describe, expect, it, vi } from "vitest";
|
import { describe, expect, it, vi } from "vitest";
|
||||||
import type { AgentWindowApi } from "../agent-window";
|
import type { AgentWindowApi, AgentWindowCreateOptions } from "../agent-window";
|
||||||
import { SessionManager } from "../manager";
|
import { SessionManager } from "../manager";
|
||||||
|
|
||||||
function fakeAgentWindow(): AgentWindowApi & {
|
function fakeAgentWindow(): AgentWindowApi & {
|
||||||
@@ -8,7 +8,7 @@ function fakeAgentWindow(): AgentWindowApi & {
|
|||||||
ensureActiveTabMock: ReturnType<typeof vi.fn>;
|
ensureActiveTabMock: ReturnType<typeof vi.fn>;
|
||||||
} {
|
} {
|
||||||
let nextId = 100;
|
let nextId = 100;
|
||||||
const createMock = vi.fn(async (_url: string) => {
|
const createMock = vi.fn(async (_url: string, _opts?: AgentWindowCreateOptions) => {
|
||||||
const id = nextId++;
|
const id = nextId++;
|
||||||
return id;
|
return id;
|
||||||
});
|
});
|
||||||
@@ -30,7 +30,7 @@ describe("SessionManager", () => {
|
|||||||
const sm = new SessionManager({ agentWindow: aw, now: () => 1700000000000 });
|
const sm = new SessionManager({ agentWindow: aw, now: () => 1700000000000 });
|
||||||
const ctx = await sm.start("aa11");
|
const ctx = await sm.start("aa11");
|
||||||
expect(aw.createMock).toHaveBeenCalledOnce();
|
expect(aw.createMock).toHaveBeenCalledOnce();
|
||||||
expect(aw.createMock).toHaveBeenCalledWith("about:blank");
|
expect(aw.createMock).toHaveBeenCalledWith("about:blank", {});
|
||||||
expect(aw.ensureActiveTabMock).toHaveBeenCalledOnce();
|
expect(aw.ensureActiveTabMock).toHaveBeenCalledOnce();
|
||||||
expect(aw.ensureActiveTabMock).toHaveBeenCalledWith(100, "about:blank");
|
expect(aw.ensureActiveTabMock).toHaveBeenCalledWith(100, "about:blank");
|
||||||
expect(ctx.sessionId).toBe("aa11");
|
expect(ctx.sessionId).toBe("aa11");
|
||||||
@@ -42,11 +42,22 @@ describe("SessionManager", () => {
|
|||||||
it("forwards an optional window size when starting a session", async () => {
|
it("forwards an optional window size when starting a session", async () => {
|
||||||
const aw = fakeAgentWindow();
|
const aw = fakeAgentWindow();
|
||||||
const sm = new SessionManager({ agentWindow: aw });
|
const sm = new SessionManager({ agentWindow: aw });
|
||||||
const ctx = await sm.start("aa11", { width: 1280, height: 800 });
|
const ctx = await sm.start("aa11", { size: { width: 1280, height: 800 } });
|
||||||
expect(aw.createMock).toHaveBeenCalledWith("about:blank", { width: 1280, height: 800 });
|
expect(aw.createMock).toHaveBeenCalledWith("about:blank", {
|
||||||
|
size: { width: 1280, height: 800 },
|
||||||
|
});
|
||||||
expect(ctx.agentWindowId).toBe(100);
|
expect(ctx.agentWindowId).toBe(100);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("forwards an explicit unfocused start to the Agent Window", async () => {
|
||||||
|
const aw = fakeAgentWindow();
|
||||||
|
const sm = new SessionManager({ agentWindow: aw });
|
||||||
|
|
||||||
|
await sm.start("aa11", { focused: false });
|
||||||
|
|
||||||
|
expect(aw.createMock).toHaveBeenCalledWith("about:blank", { focused: false });
|
||||||
|
});
|
||||||
|
|
||||||
it("indexes the session by sessionId and agent window id", async () => {
|
it("indexes the session by sessionId and agent window id", async () => {
|
||||||
const aw = fakeAgentWindow();
|
const aw = fakeAgentWindow();
|
||||||
const sm = new SessionManager({ agentWindow: aw });
|
const sm = new SessionManager({ agentWindow: aw });
|
||||||
@@ -64,6 +75,52 @@ describe("SessionManager", () => {
|
|||||||
await expect(sm.start("aa11")).rejects.toThrow(/already exists/);
|
await expect(sm.start("aa11")).rejects.toThrow(/already exists/);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("removes a newly created Agent Window when startup is aborted", async () => {
|
||||||
|
const aw = fakeAgentWindow();
|
||||||
|
let resolveCreate: (windowId: number) => void = () => {};
|
||||||
|
aw.createMock.mockImplementationOnce(
|
||||||
|
() =>
|
||||||
|
new Promise<number>((resolve) => {
|
||||||
|
resolveCreate = resolve;
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
const sm = new SessionManager({ agentWindow: aw });
|
||||||
|
const controller = new AbortController();
|
||||||
|
const pending = sm.start("aa11", { signal: controller.signal });
|
||||||
|
|
||||||
|
controller.abort();
|
||||||
|
resolveCreate(777);
|
||||||
|
|
||||||
|
await expect(pending).rejects.toMatchObject({ name: "AbortError" });
|
||||||
|
expect(aw.removeMock).toHaveBeenCalledWith(777);
|
||||||
|
expect(sm.has("aa11")).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("removes an incomplete Agent Window when active-tab setup fails", async () => {
|
||||||
|
const aw = fakeAgentWindow();
|
||||||
|
aw.ensureActiveTabMock.mockRejectedValueOnce(new Error("tab setup failed"));
|
||||||
|
const sm = new SessionManager({ agentWindow: aw });
|
||||||
|
|
||||||
|
await expect(sm.start("aa11")).rejects.toThrow("tab setup failed");
|
||||||
|
|
||||||
|
expect(aw.removeMock).toHaveBeenCalledWith(100);
|
||||||
|
expect(sm.has("aa11")).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("surfaces the orphan Agent Window id when startup cleanup fails", async () => {
|
||||||
|
const aw = fakeAgentWindow();
|
||||||
|
aw.ensureActiveTabMock.mockRejectedValueOnce(new Error("tab setup failed"));
|
||||||
|
aw.removeMock.mockRejectedValueOnce(new Error("window removal denied"));
|
||||||
|
const sm = new SessionManager({ agentWindow: aw });
|
||||||
|
|
||||||
|
await expect(sm.start("aa11")).rejects.toMatchObject({
|
||||||
|
name: "SessionStartCleanupError",
|
||||||
|
windowId: 100,
|
||||||
|
message: expect.stringMatching(/cleanup of Agent Window 100 failed.*window removal denied/),
|
||||||
|
});
|
||||||
|
expect(sm.has("aa11")).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
it("stop() closes the Agent Window and forgets the session", async () => {
|
it("stop() closes the Agent Window and forgets the session", async () => {
|
||||||
const aw = fakeAgentWindow();
|
const aw = fakeAgentWindow();
|
||||||
const sm = new SessionManager({ agentWindow: aw });
|
const sm = new SessionManager({ agentWindow: aw });
|
||||||
|
|||||||
@@ -33,4 +33,19 @@ describe("RefStore", () => {
|
|||||||
expect(s.resolve("@e11")).toBe(11);
|
expect(s.resolve("@e11")).toBe(11);
|
||||||
expect(s.size()).toBe(2);
|
expect(s.size()).toBe(2);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("preserves the child CDP session as part of ref identity", () => {
|
||||||
|
const s = new RefStore();
|
||||||
|
s.set("e1", 42, {
|
||||||
|
tabId: 7,
|
||||||
|
frameId: "child-frame",
|
||||||
|
cdpSessionId: "child-session",
|
||||||
|
});
|
||||||
|
expect(s.resolveEntry("e1")).toMatchObject({
|
||||||
|
backendNodeId: 42,
|
||||||
|
tabId: 7,
|
||||||
|
frameId: "child-frame",
|
||||||
|
cdpSessionId: "child-session",
|
||||||
|
});
|
||||||
|
});
|
||||||
});
|
});
|
||||||
@@ -8,7 +8,7 @@
|
|||||||
*/
|
*/
|
||||||
|
|
||||||
export interface AgentWindowApi {
|
export interface AgentWindowApi {
|
||||||
create(url: string, size?: { width: number; height: number }): Promise<number>;
|
create(url: string, opts?: AgentWindowCreateOptions): Promise<number>;
|
||||||
remove(windowId: number): Promise<void>;
|
remove(windowId: number): Promise<void>;
|
||||||
/**
|
/**
|
||||||
* Guarantee the Agent Window has an active, CDP-navigable tab.
|
* Guarantee the Agent Window has an active, CDP-navigable tab.
|
||||||
@@ -18,16 +18,24 @@ export interface AgentWindowApi {
|
|||||||
ensureActiveTab(windowId: number, url: string): Promise<void>;
|
ensureActiveTab(windowId: number, url: string): Promise<void>;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Creation hints for a new Agent Window. */
|
||||||
|
export interface AgentWindowCreateOptions {
|
||||||
|
/** Optional outer size in CSS pixels. */
|
||||||
|
size?: { width: number; height: number };
|
||||||
|
/** Defaults to true so existing sessions keep visible Agent Windows. */
|
||||||
|
focused?: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
/** Initial tab URL for every new session's Agent Window. */
|
/** Initial tab URL for every new session's Agent Window. */
|
||||||
export const AGENT_WINDOW_HOME = "about:blank";
|
export const AGENT_WINDOW_HOME = "about:blank";
|
||||||
|
|
||||||
export const chromeAgentWindowApi: AgentWindowApi = {
|
export const chromeAgentWindowApi: AgentWindowApi = {
|
||||||
async create(url: string, size?: { width: number; height: number }): Promise<number> {
|
async create(url: string, opts: AgentWindowCreateOptions = {}): Promise<number> {
|
||||||
const win = await chrome.windows.create({
|
const win = await chrome.windows.create({
|
||||||
type: "normal",
|
type: "normal",
|
||||||
focused: true,
|
focused: opts.focused ?? true,
|
||||||
url,
|
url,
|
||||||
...(size ? { width: size.width, height: size.height } : {}),
|
...(opts.size ? { width: opts.size.width, height: opts.size.height } : {}),
|
||||||
});
|
});
|
||||||
if (typeof win?.id !== "number") {
|
if (typeof win?.id !== "number") {
|
||||||
throw new Error("[bh] chrome.windows.create returned no window id");
|
throw new Error("[bh] chrome.windows.create returned no window id");
|
||||||
@@ -35,12 +43,11 @@ export const chromeAgentWindowApi: AgentWindowApi = {
|
|||||||
return win.id;
|
return win.id;
|
||||||
},
|
},
|
||||||
async remove(windowId: number): Promise<void> {
|
async remove(windowId: number): Promise<void> {
|
||||||
try {
|
// Callers decide whether a missing/failed removal is benign. In
|
||||||
await chrome.windows.remove(windowId);
|
// particular, transactional session-start cleanup must be able to
|
||||||
} catch (err) {
|
// surface a window it could not remove instead of reporting a false
|
||||||
// Window may have been closed by the user already; ignore.
|
// cancellation success while the Agent Window remains open.
|
||||||
console.debug("[bh] chrome.windows.remove failed", err);
|
await chrome.windows.remove(windowId);
|
||||||
}
|
|
||||||
},
|
},
|
||||||
async ensureActiveTab(windowId: number, url: string): Promise<void> {
|
async ensureActiveTab(windowId: number, url: string): Promise<void> {
|
||||||
const tabs = await chrome.tabs.query({ windowId });
|
const tabs = await chrome.tabs.query({ windowId });
|
||||||
|
|||||||
@@ -0,0 +1,75 @@
|
|||||||
|
import { handleSessionStop, type SessionStopDeps } from "@/tools/session";
|
||||||
|
import type { SessionManager } from "./manager";
|
||||||
|
|
||||||
|
export interface DisconnectCleanupFailure {
|
||||||
|
sessionId: string;
|
||||||
|
message: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface DisconnectCleanupReport {
|
||||||
|
stoppedSessionIds: string[];
|
||||||
|
failures: DisconnectCleanupFailure[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface DisconnectCleanupOptions {
|
||||||
|
manager: SessionManager;
|
||||||
|
sessionStopDeps?: SessionStopDeps;
|
||||||
|
onSessionsChanged?: () => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Build a coalesced cleanup operation for transport loss.
|
||||||
|
*
|
||||||
|
* The daemon purges its registry when the extension socket disappears. To
|
||||||
|
* keep the extension-side mirror consistent, every local session must follow
|
||||||
|
* the normal safe stop path as well: return borrowed tabs, clear refs, detach
|
||||||
|
* CDP, and only then close the Agent Window.
|
||||||
|
*/
|
||||||
|
export function createDisconnectCleanup(options: DisconnectCleanupOptions) {
|
||||||
|
let inFlight: Promise<DisconnectCleanupReport> | null = null;
|
||||||
|
|
||||||
|
return (): Promise<DisconnectCleanupReport> => {
|
||||||
|
if (inFlight) return inFlight;
|
||||||
|
inFlight = cleanupSessions(options).finally(() => {
|
||||||
|
inFlight = null;
|
||||||
|
});
|
||||||
|
return inFlight;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async function cleanupSessions(
|
||||||
|
options: DisconnectCleanupOptions,
|
||||||
|
): Promise<DisconnectCleanupReport> {
|
||||||
|
const stoppedSessionIds: string[] = [];
|
||||||
|
const failures: DisconnectCleanupFailure[] = [];
|
||||||
|
|
||||||
|
for (const ctx of options.manager.list()) {
|
||||||
|
try {
|
||||||
|
const result = await handleSessionStop(
|
||||||
|
options.manager,
|
||||||
|
{ session_id: ctx.sessionId },
|
||||||
|
options.sessionStopDeps,
|
||||||
|
);
|
||||||
|
if ("code" in result) {
|
||||||
|
failures.push({ sessionId: ctx.sessionId, message: result.message });
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (result.return_failures?.length) {
|
||||||
|
failures.push({
|
||||||
|
sessionId: ctx.sessionId,
|
||||||
|
message: result.return_failures.map((failure) => failure.message).join("; "),
|
||||||
|
});
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
stoppedSessionIds.push(ctx.sessionId);
|
||||||
|
} catch (err) {
|
||||||
|
failures.push({
|
||||||
|
sessionId: ctx.sessionId,
|
||||||
|
message: err instanceof Error ? err.message : String(err),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
options.onSessionsChanged?.();
|
||||||
|
return { stoppedSessionIds, failures };
|
||||||
|
}
|
||||||
@@ -25,6 +25,46 @@ export interface SessionManagerOptions {
|
|||||||
now?: () => number;
|
now?: () => number;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Options for starting a session's Agent Window. */
|
||||||
|
export interface SessionStartOptions {
|
||||||
|
/** Optional Agent Window outer size in CSS pixels. */
|
||||||
|
size?: { width: number; height: number };
|
||||||
|
/** Defaults to true so existing clients keep visible Agent Windows. */
|
||||||
|
focused?: boolean;
|
||||||
|
/** Cancellation for the transactional Agent Window startup sequence. */
|
||||||
|
signal?: AbortSignal;
|
||||||
|
}
|
||||||
|
|
||||||
|
export class SessionStartCleanupError extends Error {
|
||||||
|
readonly windowId: number;
|
||||||
|
readonly startupError: unknown;
|
||||||
|
readonly cleanupError: unknown;
|
||||||
|
|
||||||
|
constructor(windowId: number, startupError: unknown, cleanupError: unknown) {
|
||||||
|
const startupMessage =
|
||||||
|
startupError instanceof Error ? startupError.message : String(startupError);
|
||||||
|
const cleanupMessage =
|
||||||
|
cleanupError instanceof Error ? cleanupError.message : String(cleanupError);
|
||||||
|
super(
|
||||||
|
`session_start failed (${startupMessage}) and cleanup of Agent Window ${windowId} failed: ${cleanupMessage}`,
|
||||||
|
);
|
||||||
|
this.name = "SessionStartCleanupError";
|
||||||
|
this.windowId = windowId;
|
||||||
|
this.startupError = startupError;
|
||||||
|
this.cleanupError = cleanupError;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function sessionStartAbortError(): Error {
|
||||||
|
const error = new Error("session_start aborted");
|
||||||
|
error.name = "AbortError";
|
||||||
|
return error;
|
||||||
|
}
|
||||||
|
|
||||||
|
function throwIfSessionStartAborted(signal: AbortSignal | undefined): void {
|
||||||
|
if (signal?.aborted) throw sessionStartAbortError();
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Owner of all live agent sessions inside the extension.
|
* Owner of all live agent sessions inside the extension.
|
||||||
*
|
*
|
||||||
@@ -128,29 +168,40 @@ export class SessionManager {
|
|||||||
* Returns the created window id so callers can echo it back to the
|
* Returns the created window id so callers can echo it back to the
|
||||||
* daemon in the `tool.session_start` reply.
|
* daemon in the `tool.session_start` reply.
|
||||||
*/
|
*/
|
||||||
async start(
|
async start(sessionId: string, opts: SessionStartOptions = {}): Promise<SessionContext> {
|
||||||
sessionId: string,
|
|
||||||
size?: { width: number; height: number },
|
|
||||||
): Promise<SessionContext> {
|
|
||||||
if (this.sessions.has(sessionId)) {
|
if (this.sessions.has(sessionId)) {
|
||||||
throw new Error(`[bh] session ${sessionId} already exists`);
|
throw new Error(`[bh] session ${sessionId} already exists`);
|
||||||
}
|
}
|
||||||
// Only pass `size` when given so the no-size call shape (and its
|
throwIfSessionStartAborted(opts.signal);
|
||||||
// chrome.windows.create payload) stays exactly as before.
|
|
||||||
const windowId = size
|
let windowId: number | null = null;
|
||||||
? await this.agentWindow.create(AGENT_WINDOW_HOME, size)
|
try {
|
||||||
: await this.agentWindow.create(AGENT_WINDOW_HOME);
|
const { signal: _signal, ...createOptions } = opts;
|
||||||
await this.agentWindow.ensureActiveTab(windowId, AGENT_WINDOW_HOME);
|
windowId = await this.agentWindow.create(AGENT_WINDOW_HOME, createOptions);
|
||||||
const ctx: SessionContext = {
|
throwIfSessionStartAborted(opts.signal);
|
||||||
sessionId,
|
await this.agentWindow.ensureActiveTab(windowId, AGENT_WINDOW_HOME);
|
||||||
agentWindowId: windowId,
|
throwIfSessionStartAborted(opts.signal);
|
||||||
refStore: new RefStore(),
|
|
||||||
borrowedTabs: new Map(),
|
const ctx: SessionContext = {
|
||||||
createdAtMs: this.now(),
|
sessionId,
|
||||||
};
|
agentWindowId: windowId,
|
||||||
this.sessions.set(sessionId, ctx);
|
refStore: new RefStore(),
|
||||||
this.windowIndex.set(windowId, sessionId);
|
borrowedTabs: new Map(),
|
||||||
return ctx;
|
createdAtMs: this.now(),
|
||||||
|
};
|
||||||
|
this.sessions.set(sessionId, ctx);
|
||||||
|
this.windowIndex.set(windowId, sessionId);
|
||||||
|
return ctx;
|
||||||
|
} catch (startupError) {
|
||||||
|
if (windowId !== null) {
|
||||||
|
try {
|
||||||
|
await this.agentWindow.remove(windowId);
|
||||||
|
} catch (cleanupError) {
|
||||||
|
throw new SessionStartCleanupError(windowId, startupError, cleanupError);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
throw startupError;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
@@ -1,9 +1,10 @@
|
|||||||
/**
|
/**
|
||||||
* Per-session map from `@e<N>` snapshot refs to CDP `backendNodeId`.
|
* Per-session map from `@e<N>` snapshot refs to a CDP node address.
|
||||||
*
|
*
|
||||||
* Each fresh `tool.snapshot` resets the store: M6 will call
|
* Each fresh `tool.snapshot` resets the store: M6 will call
|
||||||
* `replace(...)` with the new ref → backendNodeId pairs. Tools like
|
* `replace(...)` with the new ref → node address pairs. A node address
|
||||||
* `tool.click` consume the store via `resolve("@e1")`.
|
* includes the owning flat CDP session/frame when the element lives in
|
||||||
|
* an OOPIF; tools resolve live geometry from that identity at call time.
|
||||||
*
|
*
|
||||||
* Refs are session-scoped (§7): looking up a ref in the wrong session
|
* Refs are session-scoped (§7): looking up a ref in the wrong session
|
||||||
* returns `null`, never silently leaks. Storing values in different
|
* returns `null`, never silently leaks. Storing values in different
|
||||||
@@ -15,10 +16,19 @@ export type BackendNodeId = number;
|
|||||||
export interface RefEntry {
|
export interface RefEntry {
|
||||||
backendNodeId: BackendNodeId;
|
backendNodeId: BackendNodeId;
|
||||||
tabId: number | null;
|
tabId: number | null;
|
||||||
|
frameId?: string;
|
||||||
|
cdpSessionId?: string;
|
||||||
generation: number;
|
generation: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
type RefInput = BackendNodeId | { backendNodeId: BackendNodeId; tabId: number };
|
export type RefInput =
|
||||||
|
| BackendNodeId
|
||||||
|
| {
|
||||||
|
backendNodeId: BackendNodeId;
|
||||||
|
tabId: number;
|
||||||
|
frameId?: string;
|
||||||
|
cdpSessionId?: string;
|
||||||
|
};
|
||||||
|
|
||||||
export class RefStore {
|
export class RefStore {
|
||||||
private readonly map = new Map<string, RefEntry>();
|
private readonly map = new Map<string, RefEntry>();
|
||||||
@@ -44,7 +54,7 @@ export class RefStore {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Replace the entire store with a new ref → backendNodeId mapping.
|
* Replace the entire store with a new ref → CDP node identity mapping.
|
||||||
* Used after every fresh `tool.snapshot`.
|
* Used after every fresh `tool.snapshot`.
|
||||||
*/
|
*/
|
||||||
replace(entries: Iterable<readonly [string, RefInput]>): void {
|
replace(entries: Iterable<readonly [string, RefInput]>): void {
|
||||||
@@ -53,10 +63,20 @@ export class RefStore {
|
|||||||
for (const [ref, input] of entries) this.map.set(normaliseRef(ref), this.entry(input));
|
for (const [ref, input] of entries) this.map.set(normaliseRef(ref), this.entry(input));
|
||||||
}
|
}
|
||||||
|
|
||||||
set(ref: string, id: BackendNodeId, opts: { tabId?: number } = {}): void {
|
set(
|
||||||
|
ref: string,
|
||||||
|
id: BackendNodeId,
|
||||||
|
opts: {
|
||||||
|
tabId?: number;
|
||||||
|
frameId?: string;
|
||||||
|
cdpSessionId?: string;
|
||||||
|
} = {},
|
||||||
|
): void {
|
||||||
this.map.set(normaliseRef(ref), {
|
this.map.set(normaliseRef(ref), {
|
||||||
backendNodeId: id,
|
backendNodeId: id,
|
||||||
tabId: opts.tabId ?? null,
|
tabId: opts.tabId ?? null,
|
||||||
|
...(opts.frameId ? { frameId: opts.frameId } : {}),
|
||||||
|
...(opts.cdpSessionId ? { cdpSessionId: opts.cdpSessionId } : {}),
|
||||||
generation: this.generation,
|
generation: this.generation,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
@@ -80,6 +100,8 @@ export class RefStore {
|
|||||||
return {
|
return {
|
||||||
backendNodeId: input.backendNodeId,
|
backendNodeId: input.backendNodeId,
|
||||||
tabId: input.tabId,
|
tabId: input.tabId,
|
||||||
|
...(input.frameId ? { frameId: input.frameId } : {}),
|
||||||
|
...(input.cdpSessionId ? { cdpSessionId: input.cdpSessionId } : {}),
|
||||||
generation: this.generation,
|
generation: this.generation,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -612,4 +612,14 @@ describe("isInjectableContentScriptUrl", () => {
|
|||||||
).toBe(false);
|
).toBe(false);
|
||||||
expect(isInjectableContentScriptUrl("https://chromewebstore.google.com/")).toBe(false);
|
expect(isInjectableContentScriptUrl("https://chromewebstore.google.com/")).toBe(false);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("rejects the Edge Add-ons store", () => {
|
||||||
|
expect(
|
||||||
|
isInjectableContentScriptUrl(
|
||||||
|
"https://microsoftedge.microsoft.com/addons/detail/browserskill/emacgiaaaiojkkpkddmmdfhmokgmnikg",
|
||||||
|
),
|
||||||
|
).toBe(false);
|
||||||
|
// Non-store paths on the same host stay injectable.
|
||||||
|
expect(isInjectableContentScriptUrl("https://microsoftedge.microsoft.com/")).toBe(true);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
@@ -9,6 +9,8 @@ import type {
|
|||||||
} from "@/transport/types";
|
} from "@/transport/types";
|
||||||
import { ToolDispatcher } from "../dispatcher";
|
import { ToolDispatcher } from "../dispatcher";
|
||||||
|
|
||||||
|
type TestDispatcherCdp = NonNullable<ConstructorParameters<typeof ToolDispatcher>[0]["cdp"]>;
|
||||||
|
|
||||||
function fakeTransport() {
|
function fakeTransport() {
|
||||||
const handlers = new Set<FrameHandler>();
|
const handlers = new Set<FrameHandler>();
|
||||||
const stateHandlers = new Set<ConnectionStateHandler>();
|
const stateHandlers = new Set<ConnectionStateHandler>();
|
||||||
@@ -67,6 +69,25 @@ describe("ToolDispatcher", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("forwards an unfocused session start to the Agent Window", async () => {
|
||||||
|
const { transport, deliver } = fakeTransport();
|
||||||
|
const create = vi.fn(async () => 4242);
|
||||||
|
const sessions = new SessionManager({
|
||||||
|
agentWindow: {
|
||||||
|
create,
|
||||||
|
remove: vi.fn(),
|
||||||
|
ensureActiveTab: vi.fn(async () => {}),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const dispatcher = new ToolDispatcher({ transport, sessions });
|
||||||
|
dispatcher.start();
|
||||||
|
|
||||||
|
deliver(makeRequest("tool.session_start", { session_id: "aa11", focused: false }));
|
||||||
|
await flushMicrotasks();
|
||||||
|
|
||||||
|
expect(create).toHaveBeenCalledWith("about:blank", { focused: false });
|
||||||
|
});
|
||||||
|
|
||||||
it("routes tool.session_stop and replies with empty result", async () => {
|
it("routes tool.session_stop and replies with empty result", async () => {
|
||||||
const { transport, sent, deliver } = fakeTransport();
|
const { transport, sent, deliver } = fakeTransport();
|
||||||
const sessions = new SessionManager({
|
const sessions = new SessionManager({
|
||||||
@@ -129,7 +150,7 @@ describe("ToolDispatcher", () => {
|
|||||||
setUserAgentOverride: vi.fn(async () => {}),
|
setUserAgentOverride: vi.fn(async () => {}),
|
||||||
setTouchEmulationEnabled: vi.fn(async () => {}),
|
setTouchEmulationEnabled: vi.fn(async () => {}),
|
||||||
};
|
};
|
||||||
const dispatcher = new ToolDispatcher({ transport, sessions, cdp });
|
const dispatcher = new ToolDispatcher({ transport, sessions, cdp: cdp as TestDispatcherCdp });
|
||||||
dispatcher.start();
|
dispatcher.start();
|
||||||
|
|
||||||
deliver(makeRequest("tool.console", { session_id: "aa11" }));
|
deliver(makeRequest("tool.console", { session_id: "aa11" }));
|
||||||
@@ -172,7 +193,7 @@ describe("ToolDispatcher", () => {
|
|||||||
setUserAgentOverride: vi.fn(async () => {}),
|
setUserAgentOverride: vi.fn(async () => {}),
|
||||||
setTouchEmulationEnabled: vi.fn(async () => {}),
|
setTouchEmulationEnabled: vi.fn(async () => {}),
|
||||||
};
|
};
|
||||||
const dispatcher = new ToolDispatcher({ transport, sessions, cdp });
|
const dispatcher = new ToolDispatcher({ transport, sessions, cdp: cdp as TestDispatcherCdp });
|
||||||
dispatcher.start();
|
dispatcher.start();
|
||||||
|
|
||||||
deliver(makeRequest("tool.session_stop", { session_id: "aa11" }));
|
deliver(makeRequest("tool.session_stop", { session_id: "aa11" }));
|
||||||
@@ -182,6 +203,55 @@ describe("ToolDispatcher", () => {
|
|||||||
expect(sent[0]).toEqual({ id: "r-1", result: {} });
|
expect(sent[0]).toEqual({ id: "r-1", result: {} });
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("forwards daemon cancellation into an in-flight session stop", async () => {
|
||||||
|
const { transport, sent, deliver } = fakeTransport();
|
||||||
|
const remove = vi.fn(async () => {});
|
||||||
|
const sessions = new SessionManager({
|
||||||
|
agentWindow: {
|
||||||
|
create: vi.fn(async () => 4242),
|
||||||
|
remove,
|
||||||
|
ensureActiveTab: vi.fn(async () => {}),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
await sessions.start("aa11");
|
||||||
|
let finishDetach: () => void = () => {};
|
||||||
|
const detach = new Promise<void>((resolve) => {
|
||||||
|
finishDetach = resolve;
|
||||||
|
});
|
||||||
|
const cdp = {
|
||||||
|
send: vi.fn(),
|
||||||
|
detachSession: vi.fn(() => detach),
|
||||||
|
ensureNetworkCapture: vi.fn(async () => {}),
|
||||||
|
networkEntriesSince: vi.fn(() => ({
|
||||||
|
tab_id: 7,
|
||||||
|
entries: [],
|
||||||
|
next_since: 0,
|
||||||
|
truncated: false,
|
||||||
|
})),
|
||||||
|
setDeviceMetricsOverride: vi.fn(async () => {}),
|
||||||
|
clearDeviceMetricsOverride: vi.fn(async () => {}),
|
||||||
|
setUserAgentOverride: vi.fn(async () => {}),
|
||||||
|
setTouchEmulationEnabled: vi.fn(async () => {}),
|
||||||
|
};
|
||||||
|
const dispatcher = new ToolDispatcher({ transport, sessions, cdp: cdp as TestDispatcherCdp });
|
||||||
|
dispatcher.start();
|
||||||
|
|
||||||
|
deliver(makeRequest("tool.session_stop", { session_id: "aa11" }));
|
||||||
|
await vi.waitFor(() => expect(cdp.detachSession).toHaveBeenCalledWith("aa11"));
|
||||||
|
deliver({ id: "cancel-stop", method: "cancel", params: { rpc_id: "r-1" } });
|
||||||
|
await flushMicrotasks();
|
||||||
|
finishDetach();
|
||||||
|
await flushMicrotasks();
|
||||||
|
|
||||||
|
expect(sent).toContainEqual({ id: "cancel-stop", result: { cancelled: true } });
|
||||||
|
expect(sent).toContainEqual({
|
||||||
|
id: "r-1",
|
||||||
|
error: expect.objectContaining({ code: "cancelled" }),
|
||||||
|
});
|
||||||
|
expect(sessions.has("aa11")).toBe(true);
|
||||||
|
expect(remove).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
it("returns not_found when stopping an unknown session", async () => {
|
it("returns not_found when stopping an unknown session", async () => {
|
||||||
const { transport, sent, deliver } = fakeTransport();
|
const { transport, sent, deliver } = fakeTransport();
|
||||||
const sessions = new SessionManager({
|
const sessions = new SessionManager({
|
||||||
@@ -311,6 +381,277 @@ describe("ToolDispatcher", () => {
|
|||||||
expect(onBrowserControlResumed).toHaveBeenCalledWith("aa11");
|
expect(onBrowserControlResumed).toHaveBeenCalledWith("aa11");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("reasserts remembered hover before follow-up work and releases only after actions", async () => {
|
||||||
|
vi.stubGlobal("chrome", {
|
||||||
|
tabs: {
|
||||||
|
sendMessage: vi.fn(async () => undefined),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const { transport } = fakeTransport();
|
||||||
|
const sessions = new SessionManager({
|
||||||
|
agentWindow: {
|
||||||
|
create: vi.fn(async () => 1),
|
||||||
|
remove: vi.fn(async () => {}),
|
||||||
|
ensureActiveTab: vi.fn(async () => {}),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const order: string[] = [];
|
||||||
|
const cdp = {
|
||||||
|
send: vi.fn(async <T>() => {
|
||||||
|
order.push("hover");
|
||||||
|
return {} as T;
|
||||||
|
}),
|
||||||
|
detachSession: vi.fn(async () => {}),
|
||||||
|
ensureNetworkCapture: vi.fn(async () => {}),
|
||||||
|
networkEntriesSince: vi.fn(() => ({
|
||||||
|
tab_id: 7,
|
||||||
|
entries: [],
|
||||||
|
next_since: 0,
|
||||||
|
truncated: false,
|
||||||
|
})),
|
||||||
|
setDeviceMetricsOverride: vi.fn(async () => {}),
|
||||||
|
clearDeviceMetricsOverride: vi.fn(async () => {}),
|
||||||
|
setUserAgentOverride: vi.fn(async () => {}),
|
||||||
|
setTouchEmulationEnabled: vi.fn(async () => {}),
|
||||||
|
};
|
||||||
|
const dispatcher = new ToolDispatcher({ transport, sessions, cdp: cdp as TestDispatcherCdp });
|
||||||
|
|
||||||
|
(
|
||||||
|
dispatcher as unknown as { rememberHover: (sessionId: string, result: unknown) => unknown }
|
||||||
|
).rememberHover("aa11", {
|
||||||
|
tab_id: 7,
|
||||||
|
x: 10,
|
||||||
|
y: 20,
|
||||||
|
});
|
||||||
|
const helpers = dispatcher as unknown as {
|
||||||
|
withHoverReassert: <T>(
|
||||||
|
params: { session_id: string; tab_id?: number },
|
||||||
|
work: () => Promise<T>,
|
||||||
|
options?: { releaseAfter?: boolean },
|
||||||
|
) => Promise<T>;
|
||||||
|
setHoverBypass: (sessionId: string, tabId: number, enabled: boolean) => Promise<void>;
|
||||||
|
};
|
||||||
|
|
||||||
|
await helpers.withHoverReassert({ session_id: "aa11", tab_id: 7 }, async () => {
|
||||||
|
order.push("observe");
|
||||||
|
return {};
|
||||||
|
});
|
||||||
|
expect(order).toEqual(["hover", "observe"]);
|
||||||
|
expect(chrome.tabs.sendMessage).not.toHaveBeenCalledWith(
|
||||||
|
7,
|
||||||
|
expect.objectContaining({ enabled: false }),
|
||||||
|
);
|
||||||
|
|
||||||
|
await helpers.setHoverBypass("aa11", 7, true);
|
||||||
|
await helpers.withHoverReassert(
|
||||||
|
{ session_id: "aa11", tab_id: 7 },
|
||||||
|
async () => {
|
||||||
|
order.push("click");
|
||||||
|
return {};
|
||||||
|
},
|
||||||
|
{ releaseAfter: true },
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(cdp.send).toHaveBeenLastCalledWith(7, "Input.dispatchMouseEvent", {
|
||||||
|
type: "mouseMoved",
|
||||||
|
x: 10,
|
||||||
|
y: 20,
|
||||||
|
});
|
||||||
|
expect(chrome.tabs.sendMessage).toHaveBeenCalledWith(
|
||||||
|
7,
|
||||||
|
expect.objectContaining({ enabled: false }),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not reassert or disable hover latches from another session", async () => {
|
||||||
|
vi.stubGlobal("chrome", {
|
||||||
|
tabs: {
|
||||||
|
sendMessage: vi.fn(async () => undefined),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const { transport } = fakeTransport();
|
||||||
|
const sessions = new SessionManager({
|
||||||
|
agentWindow: {
|
||||||
|
create: vi.fn(async () => 1),
|
||||||
|
remove: vi.fn(async () => {}),
|
||||||
|
ensureActiveTab: vi.fn(async () => {}),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const cdp = {
|
||||||
|
send: vi.fn(async <T>() => ({}) as T),
|
||||||
|
detachSession: vi.fn(async () => {}),
|
||||||
|
ensureNetworkCapture: vi.fn(async () => {}),
|
||||||
|
networkEntriesSince: vi.fn(() => ({
|
||||||
|
tab_id: 7,
|
||||||
|
entries: [],
|
||||||
|
next_since: 0,
|
||||||
|
truncated: false,
|
||||||
|
})),
|
||||||
|
setDeviceMetricsOverride: vi.fn(async () => {}),
|
||||||
|
clearDeviceMetricsOverride: vi.fn(async () => {}),
|
||||||
|
setUserAgentOverride: vi.fn(async () => {}),
|
||||||
|
setTouchEmulationEnabled: vi.fn(async () => {}),
|
||||||
|
};
|
||||||
|
const dispatcher = new ToolDispatcher({ transport, sessions, cdp: cdp as TestDispatcherCdp });
|
||||||
|
const helpers = dispatcher as unknown as {
|
||||||
|
rememberHover: (sessionId: string, result: unknown) => unknown;
|
||||||
|
setHoverBypass: (sessionId: string, tabId: number, enabled: boolean) => Promise<void>;
|
||||||
|
withHoverReassert: <T>(
|
||||||
|
params: { session_id: string; tab_id?: number },
|
||||||
|
work: () => Promise<T>,
|
||||||
|
options?: { releaseAfter?: boolean },
|
||||||
|
) => Promise<T>;
|
||||||
|
releaseHoverLatch: (sessionId?: string, tabId?: number) => Promise<void>;
|
||||||
|
};
|
||||||
|
|
||||||
|
helpers.rememberHover("aa11", { tab_id: 7, x: 10, y: 20 });
|
||||||
|
await helpers.setHoverBypass("aa11", 7, true);
|
||||||
|
|
||||||
|
await helpers.withHoverReassert({ session_id: "bb22", tab_id: 7 }, async () => ({}), {
|
||||||
|
releaseAfter: true,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(cdp.send).not.toHaveBeenCalled();
|
||||||
|
expect(chrome.tabs.sendMessage).not.toHaveBeenCalledWith(
|
||||||
|
7,
|
||||||
|
expect.objectContaining({ enabled: false }),
|
||||||
|
);
|
||||||
|
|
||||||
|
await helpers.releaseHoverLatch("aa11", 7);
|
||||||
|
expect(chrome.tabs.sendMessage).toHaveBeenCalledWith(
|
||||||
|
7,
|
||||||
|
expect.objectContaining({ enabled: false }),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("transfers hover overlay bypass ownership between sessions", async () => {
|
||||||
|
vi.stubGlobal("chrome", {
|
||||||
|
tabs: {
|
||||||
|
sendMessage: vi.fn(async () => undefined),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const { transport } = fakeTransport();
|
||||||
|
const sessions = new SessionManager({
|
||||||
|
agentWindow: {
|
||||||
|
create: vi.fn(async () => 1),
|
||||||
|
remove: vi.fn(async () => {}),
|
||||||
|
ensureActiveTab: vi.fn(async () => {}),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const dispatcher = new ToolDispatcher({ transport, sessions });
|
||||||
|
const helpers = dispatcher as unknown as {
|
||||||
|
setHoverBypass: (sessionId: string, tabId: number, enabled: boolean) => Promise<void>;
|
||||||
|
};
|
||||||
|
|
||||||
|
await helpers.setHoverBypass("aa11", 7, true);
|
||||||
|
await helpers.setHoverBypass("bb22", 7, true);
|
||||||
|
await helpers.setHoverBypass("aa11", 7, false);
|
||||||
|
expect(chrome.tabs.sendMessage).not.toHaveBeenCalledWith(
|
||||||
|
7,
|
||||||
|
expect.objectContaining({ enabled: false }),
|
||||||
|
);
|
||||||
|
|
||||||
|
await helpers.setHoverBypass("bb22", 7, false);
|
||||||
|
expect(chrome.tabs.sendMessage).toHaveBeenCalledWith(
|
||||||
|
7,
|
||||||
|
expect.objectContaining({ enabled: false }),
|
||||||
|
);
|
||||||
|
expect(chrome.tabs.sendMessage).toHaveBeenCalledTimes(2);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("limits default-target hover reassertion to the active tab", async () => {
|
||||||
|
vi.stubGlobal("chrome", {
|
||||||
|
tabs: {
|
||||||
|
query: vi.fn(async () => [{ id: 9, windowId: 4242, active: true }]),
|
||||||
|
get: vi.fn(async (tabId: number) => ({ id: tabId, windowId: 4242, active: tabId === 9 })),
|
||||||
|
sendMessage: vi.fn(async () => undefined),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const { transport } = fakeTransport();
|
||||||
|
const sessions = new SessionManager({
|
||||||
|
agentWindow: {
|
||||||
|
create: vi.fn(async () => 4242),
|
||||||
|
remove: vi.fn(async () => {}),
|
||||||
|
ensureActiveTab: vi.fn(async () => {}),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
await sessions.start("aa11");
|
||||||
|
const cdp = {
|
||||||
|
send: vi.fn(async <T>() => ({}) as T),
|
||||||
|
detachSession: vi.fn(async () => {}),
|
||||||
|
ensureNetworkCapture: vi.fn(async () => {}),
|
||||||
|
networkEntriesSince: vi.fn(() => ({
|
||||||
|
tab_id: 9,
|
||||||
|
entries: [],
|
||||||
|
next_since: 0,
|
||||||
|
truncated: false,
|
||||||
|
})),
|
||||||
|
setDeviceMetricsOverride: vi.fn(async () => {}),
|
||||||
|
clearDeviceMetricsOverride: vi.fn(async () => {}),
|
||||||
|
setUserAgentOverride: vi.fn(async () => {}),
|
||||||
|
setTouchEmulationEnabled: vi.fn(async () => {}),
|
||||||
|
};
|
||||||
|
const dispatcher = new ToolDispatcher({ transport, sessions, cdp: cdp as TestDispatcherCdp });
|
||||||
|
const helpers = dispatcher as unknown as {
|
||||||
|
rememberHover: (sessionId: string, result: unknown) => unknown;
|
||||||
|
withHoverReassert: <T>(
|
||||||
|
params: { session_id: string; tab_id?: number },
|
||||||
|
work: () => Promise<T>,
|
||||||
|
) => Promise<T>;
|
||||||
|
};
|
||||||
|
|
||||||
|
helpers.rememberHover("aa11", { tab_id: 7, x: 10, y: 20 });
|
||||||
|
helpers.rememberHover("aa11", { tab_id: 9, x: 30, y: 40 });
|
||||||
|
|
||||||
|
await helpers.withHoverReassert({ session_id: "aa11" }, async () => ({}));
|
||||||
|
|
||||||
|
expect(cdp.send).toHaveBeenCalledTimes(1);
|
||||||
|
expect(cdp.send).toHaveBeenCalledWith(9, "Input.dispatchMouseEvent", {
|
||||||
|
type: "mouseMoved",
|
||||||
|
x: 30,
|
||||||
|
y: 40,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("releases the current session hover latch before navigation-style work", async () => {
|
||||||
|
vi.stubGlobal("chrome", {
|
||||||
|
tabs: {
|
||||||
|
sendMessage: vi.fn(async () => undefined),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const { transport } = fakeTransport();
|
||||||
|
const sessions = new SessionManager({
|
||||||
|
agentWindow: {
|
||||||
|
create: vi.fn(async () => 1),
|
||||||
|
remove: vi.fn(async () => {}),
|
||||||
|
ensureActiveTab: vi.fn(async () => {}),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const dispatcher = new ToolDispatcher({ transport, sessions });
|
||||||
|
const helpers = dispatcher as unknown as {
|
||||||
|
rememberHover: (sessionId: string, result: unknown) => unknown;
|
||||||
|
setHoverBypass: (sessionId: string, tabId: number, enabled: boolean) => Promise<void>;
|
||||||
|
withHoverReleaseForRequest: <T>(
|
||||||
|
params: { session_id: string; tab_id?: number },
|
||||||
|
work: () => Promise<T>,
|
||||||
|
) => Promise<T>;
|
||||||
|
withHoverReassert: <T>(
|
||||||
|
params: { session_id: string; tab_id?: number },
|
||||||
|
work: () => Promise<T>,
|
||||||
|
) => Promise<T>;
|
||||||
|
};
|
||||||
|
helpers.rememberHover("aa11", { tab_id: 7, x: 10, y: 20 });
|
||||||
|
await helpers.setHoverBypass("aa11", 7, true);
|
||||||
|
|
||||||
|
await helpers.withHoverReleaseForRequest({ session_id: "aa11", tab_id: 7 }, async () => ({}));
|
||||||
|
await helpers.withHoverReassert({ session_id: "aa11", tab_id: 7 }, async () => ({}));
|
||||||
|
|
||||||
|
expect(chrome.tabs.sendMessage).toHaveBeenCalledWith(
|
||||||
|
7,
|
||||||
|
expect.objectContaining({ enabled: false }),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
it("disconnects the transport when send() fails so keepalive can rebuild", async () => {
|
it("disconnects the transport when send() fails so keepalive can rebuild", async () => {
|
||||||
const { transport, deliver } = fakeTransport();
|
const { transport, deliver } = fakeTransport();
|
||||||
const sessions = new SessionManager({
|
const sessions = new SessionManager({
|
||||||
@@ -385,23 +726,31 @@ describe("ToolDispatcher", () => {
|
|||||||
await flushMicrotasks();
|
await flushMicrotasks();
|
||||||
expect(ac?.signal.aborted).toBe(true);
|
expect(ac?.signal.aborted).toBe(true);
|
||||||
|
|
||||||
// Cancel ack arrived synchronously; the slow tool replies with
|
// Cancel ack arrives synchronously, but the original RPC must not reply
|
||||||
// `cancelled` once the dispatcher's race observes the abort.
|
// until the in-progress window creation has completed and been rolled back.
|
||||||
const ack = sent.find(
|
const ack = sent.find(
|
||||||
(m) =>
|
(m) =>
|
||||||
typeof (m as { id?: string }).id === "string" && (m as { id: string }).id === "cancel-1",
|
typeof (m as { id?: string }).id === "string" && (m as { id: string }).id === "cancel-1",
|
||||||
);
|
);
|
||||||
expect(ack).toEqual({ id: "cancel-1", result: { cancelled: true } });
|
expect(ack).toEqual({ id: "cancel-1", result: { cancelled: true } });
|
||||||
|
|
||||||
|
expect(
|
||||||
|
sent.find(
|
||||||
|
(m) => typeof (m as { id?: string }).id === "string" && (m as { id: string }).id === "r-1",
|
||||||
|
),
|
||||||
|
).toBeUndefined();
|
||||||
|
expect(dispatcher.inflightAbortControllers.has("r-1")).toBe(true);
|
||||||
|
|
||||||
|
resolveCreate(9999);
|
||||||
|
await flushMicrotasks();
|
||||||
|
|
||||||
const slow = sent.find(
|
const slow = sent.find(
|
||||||
(m) => typeof (m as { id?: string }).id === "string" && (m as { id: string }).id === "r-1",
|
(m) => typeof (m as { id?: string }).id === "string" && (m as { id: string }).id === "r-1",
|
||||||
);
|
);
|
||||||
expect(slow).toMatchObject({ id: "r-1", error: { code: "cancelled" } });
|
expect(slow).toMatchObject({ id: "r-1", error: { code: "cancelled" } });
|
||||||
expect(dispatcher.inflightAbortControllers.has("r-1")).toBe(false);
|
expect(dispatcher.inflightAbortControllers.has("r-1")).toBe(false);
|
||||||
|
expect(sessions.has("aa44")).toBe(false);
|
||||||
// Drain the dangling create promise so vitest does not warn.
|
expect(sessions.list()).toEqual([]);
|
||||||
resolveCreate(9999);
|
|
||||||
await flushMicrotasks();
|
|
||||||
});
|
});
|
||||||
|
|
||||||
it("cancel for an unknown rpc_id replies with cancelled=false", async () => {
|
it("cancel for an unknown rpc_id replies with cancelled=false", async () => {
|
||||||
|
|||||||
@@ -1,24 +1,54 @@
|
|||||||
import { describe, expect, it } from "vitest";
|
import { describe, expect, it } from "vitest";
|
||||||
import { nodeBoundingRect, quadBoundingRect } from "../element-geometry";
|
import { nodeContentRegion } from "../element-geometry";
|
||||||
import type { CdpRunner } from "../shared";
|
import type { CdpRunner } from "../shared";
|
||||||
|
|
||||||
describe("quadBoundingRect", () => {
|
describe("nodeContentRegion", () => {
|
||||||
it("computes axis-aligned bounds from an 8-double quad", () => {
|
it("rejects invalid content quads before using the box model fallback", async () => {
|
||||||
expect(quadBoundingRect([10, 20, 110, 20, 110, 60, 10, 60])).toEqual({
|
const send = async (_tabId: number, method: string) => {
|
||||||
x: 10,
|
if (method === "DOM.getContentQuads") return { quads: [[0, 0, 0, 0, 0, 0, 0, 0]] };
|
||||||
y: 20,
|
if (method === "DOM.getBoxModel") {
|
||||||
width: 100,
|
return { model: { content: [5, 10, 25, 10, 25, 30, 5, 30] } };
|
||||||
height: 40,
|
}
|
||||||
|
throw new Error(`unexpected CDP call ${method}`);
|
||||||
|
};
|
||||||
|
|
||||||
|
await expect(nodeContentRegion({ send: send as CdpRunner["send"] }, 7, 555)).resolves.toEqual([
|
||||||
|
[
|
||||||
|
{ x: 5, y: 10 },
|
||||||
|
{ x: 25, y: 10 },
|
||||||
|
{ x: 25, y: 30 },
|
||||||
|
{ x: 5, y: 30 },
|
||||||
|
],
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("preserves every visible content fragment", async () => {
|
||||||
|
const send = async () => ({
|
||||||
|
quads: [
|
||||||
|
[10, 20, 60, 20, 60, 40, 10, 40],
|
||||||
|
[15, 50, 90, 50, 90, 80, 15, 80],
|
||||||
|
],
|
||||||
});
|
});
|
||||||
|
const cdp: CdpRunner = {
|
||||||
|
send: send as CdpRunner["send"],
|
||||||
|
};
|
||||||
|
|
||||||
|
await expect(nodeContentRegion(cdp, 7, 555)).resolves.toEqual([
|
||||||
|
[
|
||||||
|
{ x: 10, y: 20 },
|
||||||
|
{ x: 60, y: 20 },
|
||||||
|
{ x: 60, y: 40 },
|
||||||
|
{ x: 10, y: 40 },
|
||||||
|
],
|
||||||
|
[
|
||||||
|
{ x: 15, y: 50 },
|
||||||
|
{ x: 90, y: 50 },
|
||||||
|
{ x: 90, y: 80 },
|
||||||
|
{ x: 15, y: 80 },
|
||||||
|
],
|
||||||
|
]);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("returns null for degenerate quads", () => {
|
|
||||||
expect(quadBoundingRect([0, 0, 0, 0, 0, 0, 0, 0])).toBeNull();
|
|
||||||
expect(quadBoundingRect([1, 2, 3])).toBeNull();
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
describe("nodeBoundingRect", () => {
|
|
||||||
it("falls back to visible descendant bounds for zero-size containers", async () => {
|
it("falls back to visible descendant bounds for zero-size containers", async () => {
|
||||||
const calls: Array<{ method: string; params?: object }> = [];
|
const calls: Array<{ method: string; params?: object }> = [];
|
||||||
const send = async (_tabId: number, method: string, params?: object) => {
|
const send = async (_tabId: number, method: string, params?: object) => {
|
||||||
@@ -38,12 +68,14 @@ describe("nodeBoundingRect", () => {
|
|||||||
};
|
};
|
||||||
const cdp = { send: send as CdpRunner["send"] };
|
const cdp = { send: send as CdpRunner["send"] };
|
||||||
|
|
||||||
await expect(nodeBoundingRect(cdp, 7, 555)).resolves.toEqual({
|
await expect(nodeContentRegion(cdp, 7, 555)).resolves.toEqual([
|
||||||
x: 242,
|
[
|
||||||
y: 468,
|
{ x: 242, y: 468 },
|
||||||
width: 769,
|
{ x: 1011, y: 468 },
|
||||||
height: 180,
|
{ x: 1011, y: 648 },
|
||||||
});
|
{ x: 242, y: 648 },
|
||||||
|
],
|
||||||
|
]);
|
||||||
expect(calls.map((c) => c.method)).toEqual([
|
expect(calls.map((c) => c.method)).toEqual([
|
||||||
"DOM.getContentQuads",
|
"DOM.getContentQuads",
|
||||||
"DOM.getBoxModel",
|
"DOM.getBoxModel",
|
||||||
|
|||||||
@@ -0,0 +1,286 @@
|
|||||||
|
import { describe, expect, it, vi } from "vitest";
|
||||||
|
import { resolveFrameProjection, resolveNodeGeometry } from "../frame-geometry";
|
||||||
|
import {
|
||||||
|
clipPolygon,
|
||||||
|
polygonArea,
|
||||||
|
polygonBounds,
|
||||||
|
projectAndClipRegion,
|
||||||
|
projectUnitPoint,
|
||||||
|
type Quad,
|
||||||
|
rectPolygon,
|
||||||
|
regionBounds,
|
||||||
|
} from "../geometry";
|
||||||
|
import type { CdpRunner } from "../shared";
|
||||||
|
|
||||||
|
describe("frame geometry projection", () => {
|
||||||
|
it("keeps region bounds separate from polygon area", () => {
|
||||||
|
const region = [
|
||||||
|
rectPolygon({ x: 0, y: 0, w: 10, h: 10 }),
|
||||||
|
rectPolygon({ x: 20, y: 5, w: 5, h: 5 }),
|
||||||
|
];
|
||||||
|
expect(regionBounds(region)).toEqual({ x: 0, y: 0, width: 25, height: 10 });
|
||||||
|
expect(polygonArea(region[0])).toBe(100);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("maps through the iframe content box instead of its border box", () => {
|
||||||
|
const contentQuad: Quad = [
|
||||||
|
{ x: 204, y: 306 },
|
||||||
|
{ x: 604, y: 306 },
|
||||||
|
{ x: 604, y: 506 },
|
||||||
|
{ x: 204, y: 506 },
|
||||||
|
];
|
||||||
|
const projected = projectAndClipRegion(
|
||||||
|
[rectPolygon({ x: 10, y: 20, w: 100, h: 40 })],
|
||||||
|
[{ sourceViewport: { width: 200, height: 100 }, destinationQuad: contentQuad }],
|
||||||
|
{ width: 1000, height: 800 },
|
||||||
|
);
|
||||||
|
expect(polygonBounds(projected.flat())).toEqual({ x: 224, y: 346, width: 200, height: 80 });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("uses a projective mapping for perspective-transformed iframe quads", () => {
|
||||||
|
const quad: Quad = [
|
||||||
|
{ x: 0, y: 0 },
|
||||||
|
{ x: 200, y: 20 },
|
||||||
|
{ x: 180, y: 140 },
|
||||||
|
{ x: 20, y: 100 },
|
||||||
|
];
|
||||||
|
expect(projectUnitPoint({ x: 0, y: 0 }, quad)).toEqual(quad[0]);
|
||||||
|
expect(projectUnitPoint({ x: 1, y: 0 }, quad)).toEqual(quad[1]);
|
||||||
|
expect(projectUnitPoint({ x: 1, y: 1 }, quad)).toEqual(quad[2]);
|
||||||
|
expect(projectUnitPoint({ x: 0, y: 1 }, quad)).toEqual(quad[3]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("clips against the iframe polygon rather than its bounding box", () => {
|
||||||
|
const frameQuad: Quad = [
|
||||||
|
{ x: 50, y: 0 },
|
||||||
|
{ x: 100, y: 50 },
|
||||||
|
{ x: 50, y: 100 },
|
||||||
|
{ x: 0, y: 50 },
|
||||||
|
];
|
||||||
|
expect(clipPolygon(rectPolygon({ x: 0, y: 0, w: 10, h: 10 }), frameQuad)).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not reapply same-target frame transforms around an OOPIF", async () => {
|
||||||
|
const send = vi.fn(async (_tabId, method, params) => {
|
||||||
|
if (method === "Page.getLayoutMetrics") {
|
||||||
|
return { cssLayoutViewport: { clientWidth: 800, clientHeight: 600 } };
|
||||||
|
}
|
||||||
|
if (method === "DOM.getBoxModel") {
|
||||||
|
const backendNodeId = (params as { backendNodeId?: number }).backendNodeId;
|
||||||
|
return backendNodeId === 20
|
||||||
|
? { model: { content: [110, 70, 310, 70, 310, 170, 110, 170] } }
|
||||||
|
: { model: { content: [100, 50, 400, 50, 400, 250, 100, 250] } };
|
||||||
|
}
|
||||||
|
throw new Error(`unexpected root command ${method}`);
|
||||||
|
});
|
||||||
|
const cdp: CdpRunner = {
|
||||||
|
send: send as CdpRunner["send"],
|
||||||
|
sendToTarget: vi.fn(async (_target, method) => {
|
||||||
|
if (method === "Page.getLayoutMetrics") {
|
||||||
|
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
|
||||||
|
}
|
||||||
|
throw new Error(`unexpected child command ${method}`);
|
||||||
|
}) as CdpRunner["sendToTarget"],
|
||||||
|
};
|
||||||
|
const projection = await resolveFrameProjection(
|
||||||
|
cdp,
|
||||||
|
{
|
||||||
|
rootFrameId: "main",
|
||||||
|
frames: [
|
||||||
|
{ frameId: "main", target: { tabId: 4 } },
|
||||||
|
{
|
||||||
|
frameId: "same-process-parent",
|
||||||
|
parentFrameId: "main",
|
||||||
|
ownerBackendNodeId: 10,
|
||||||
|
target: { tabId: 4 },
|
||||||
|
},
|
||||||
|
{
|
||||||
|
frameId: "oopif",
|
||||||
|
parentFrameId: "same-process-parent",
|
||||||
|
ownerBackendNodeId: 20,
|
||||||
|
target: { tabId: 4, sessionId: "oopif-session" },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
"oopif",
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(projection?.edges).toEqual([
|
||||||
|
{
|
||||||
|
sourceViewport: { width: 200, height: 100 },
|
||||||
|
destinationQuad: [
|
||||||
|
{ x: 110, y: 70 },
|
||||||
|
{ x: 310, y: 70 },
|
||||||
|
{ x: 310, y: 170 },
|
||||||
|
{ x: 110, y: 170 },
|
||||||
|
],
|
||||||
|
destinationClips: [
|
||||||
|
[
|
||||||
|
{ x: 100, y: 50 },
|
||||||
|
{ x: 400, y: 50 },
|
||||||
|
{ x: 400, y: 250 },
|
||||||
|
{ x: 100, y: 250 },
|
||||||
|
],
|
||||||
|
],
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("clips live OOPIF geometry at every frame boundary", async () => {
|
||||||
|
const cdp: CdpRunner = {
|
||||||
|
send: vi.fn(async (_tabId, method) => {
|
||||||
|
if (method === "Page.getLayoutMetrics") {
|
||||||
|
return { cssLayoutViewport: { clientWidth: 800, clientHeight: 600 } };
|
||||||
|
}
|
||||||
|
if (method === "DOM.getBoxModel") {
|
||||||
|
return { model: { content: [100, 100, 300, 100, 300, 200, 100, 200] } };
|
||||||
|
}
|
||||||
|
throw new Error(`unexpected root command ${method}`);
|
||||||
|
}) as CdpRunner["send"],
|
||||||
|
sendToTarget: vi.fn(async (_target, method) => {
|
||||||
|
if (method === "Page.getLayoutMetrics") {
|
||||||
|
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
|
||||||
|
}
|
||||||
|
if (method === "DOM.getContentQuads") {
|
||||||
|
return { quads: [[-50, -20, 250, -20, 250, 120, -50, 120]] };
|
||||||
|
}
|
||||||
|
throw new Error(`unexpected child command ${method}`);
|
||||||
|
}) as CdpRunner["sendToTarget"],
|
||||||
|
getFrameGraph: vi.fn(async () => ({
|
||||||
|
rootFrameId: "main",
|
||||||
|
frames: [
|
||||||
|
{ frameId: "main", target: { tabId: 4 } },
|
||||||
|
{
|
||||||
|
frameId: "child",
|
||||||
|
parentFrameId: "main",
|
||||||
|
ownerBackendNodeId: 10,
|
||||||
|
target: { tabId: 4, sessionId: "child-session" },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
})),
|
||||||
|
};
|
||||||
|
|
||||||
|
const geometry = await resolveNodeGeometry(cdp, 4, {
|
||||||
|
target: { tabId: 4, sessionId: "child-session" },
|
||||||
|
frameId: "child",
|
||||||
|
backendNodeId: 101,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(geometry).toMatchObject({
|
||||||
|
topBounds: { x: 100, y: 100, width: 200, height: 100 },
|
||||||
|
actionPoint: { x: 200, y: 150 },
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("reuses one frame graph while scrolling and projecting live geometry", async () => {
|
||||||
|
const getFrameGraph = vi.fn(async () => ({
|
||||||
|
rootFrameId: "main",
|
||||||
|
frames: [
|
||||||
|
{ frameId: "main", target: { tabId: 4 } },
|
||||||
|
{
|
||||||
|
frameId: "child",
|
||||||
|
parentFrameId: "main",
|
||||||
|
ownerBackendNodeId: 10,
|
||||||
|
target: { tabId: 4, sessionId: "child-session" },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
}));
|
||||||
|
const cdp: CdpRunner = {
|
||||||
|
send: vi.fn(async (_tabId, method) => {
|
||||||
|
if (method === "DOM.scrollIntoViewIfNeeded") return {};
|
||||||
|
if (method === "Page.getLayoutMetrics") {
|
||||||
|
return { cssLayoutViewport: { clientWidth: 800, clientHeight: 600 } };
|
||||||
|
}
|
||||||
|
if (method === "DOM.getBoxModel") {
|
||||||
|
return { model: { content: [100, 100, 300, 100, 300, 200, 100, 200] } };
|
||||||
|
}
|
||||||
|
throw new Error(`unexpected root command ${method}`);
|
||||||
|
}) as CdpRunner["send"],
|
||||||
|
sendToTarget: vi.fn(async (_target, method) => {
|
||||||
|
if (method === "DOM.scrollIntoViewIfNeeded") return {};
|
||||||
|
if (method === "Page.getLayoutMetrics") {
|
||||||
|
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
|
||||||
|
}
|
||||||
|
if (method === "DOM.getContentQuads") {
|
||||||
|
return { quads: [[0, 0, 20, 0, 20, 20, 0, 20]] };
|
||||||
|
}
|
||||||
|
throw new Error(`unexpected child command ${method}`);
|
||||||
|
}) as CdpRunner["sendToTarget"],
|
||||||
|
getFrameGraph,
|
||||||
|
};
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
resolveNodeGeometry(
|
||||||
|
cdp,
|
||||||
|
4,
|
||||||
|
{
|
||||||
|
target: { tabId: 4, sessionId: "child-session" },
|
||||||
|
frameId: "child",
|
||||||
|
backendNodeId: 101,
|
||||||
|
},
|
||||||
|
{ scrollIntoView: true },
|
||||||
|
),
|
||||||
|
).resolves.not.toMatchObject({ code: expect.any(String) });
|
||||||
|
expect(getFrameGraph).toHaveBeenCalledOnce();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("clips live geometry to same-target iframe ancestors", async () => {
|
||||||
|
const cdp: CdpRunner = {
|
||||||
|
send: vi.fn(async (_tabId, method) => {
|
||||||
|
if (method === "Page.getLayoutMetrics") {
|
||||||
|
return { cssLayoutViewport: { clientWidth: 800, clientHeight: 600 } };
|
||||||
|
}
|
||||||
|
if (method === "DOM.getBoxModel") {
|
||||||
|
return { model: { content: [100, 100, 300, 100, 300, 200, 100, 200] } };
|
||||||
|
}
|
||||||
|
if (method === "DOM.getContentQuads") {
|
||||||
|
return { quads: [[50, 50, 350, 50, 350, 250, 50, 250]] };
|
||||||
|
}
|
||||||
|
throw new Error(`unexpected ${method}`);
|
||||||
|
}) as CdpRunner["send"],
|
||||||
|
getFrameGraph: vi.fn(async () => ({
|
||||||
|
rootFrameId: "main",
|
||||||
|
frames: [
|
||||||
|
{ frameId: "main", target: { tabId: 4 } },
|
||||||
|
{
|
||||||
|
frameId: "child",
|
||||||
|
parentFrameId: "main",
|
||||||
|
ownerBackendNodeId: 10,
|
||||||
|
target: { tabId: 4 },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
})),
|
||||||
|
};
|
||||||
|
|
||||||
|
const geometry = await resolveNodeGeometry(cdp, 4, {
|
||||||
|
target: { tabId: 4 },
|
||||||
|
frameId: "child",
|
||||||
|
backendNodeId: 101,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(geometry).toMatchObject({
|
||||||
|
topBounds: { x: 100, y: 100, width: 200, height: 100 },
|
||||||
|
actionPoint: { x: 200, y: 150 },
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("fails closed when an OOPIF frame graph is unavailable", async () => {
|
||||||
|
const cdp: CdpRunner = {
|
||||||
|
send: vi.fn(),
|
||||||
|
sendToTarget: vi.fn(async (_target, method) => {
|
||||||
|
if (method === "DOM.getContentQuads") {
|
||||||
|
return { quads: [[0, 0, 20, 0, 20, 20, 0, 20]] };
|
||||||
|
}
|
||||||
|
throw new Error(`unexpected ${method}`);
|
||||||
|
}) as CdpRunner["sendToTarget"],
|
||||||
|
};
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
resolveNodeGeometry(cdp, 4, {
|
||||||
|
target: { tabId: 4, sessionId: "child-session" },
|
||||||
|
frameId: "child",
|
||||||
|
backendNodeId: 101,
|
||||||
|
}),
|
||||||
|
).resolves.toMatchObject({ code: "cdp_failed" });
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -1,5 +1,6 @@
|
|||||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||||
import { SessionManager } from "@/session-manager/manager";
|
import { SessionManager } from "@/session-manager/manager";
|
||||||
|
import { RefStore } from "@/session-manager/ref-store";
|
||||||
import type { RequestHelpParams } from "@/transport/types";
|
import type { RequestHelpParams } from "@/transport/types";
|
||||||
import { handleRequestHelp, type RequestHelpDeps, resetHelpLifecycleForTests } from "../human-loop";
|
import { handleRequestHelp, type RequestHelpDeps, resetHelpLifecycleForTests } from "../human-loop";
|
||||||
|
|
||||||
@@ -50,11 +51,10 @@ function installHelpLifecycleChrome() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function fakeManager(sessionId: string, agentWindowId: number, tabId: number) {
|
function fakeManager(sessionId: string, agentWindowId: number, tabId: number) {
|
||||||
|
const refStore = new RefStore();
|
||||||
const mgr = {
|
const mgr = {
|
||||||
get: (id: string) =>
|
get: (id: string) =>
|
||||||
id === sessionId
|
id === sessionId ? { sessionId, agentWindowId, refStore, borrowedTabs: new Map() } : null,
|
||||||
? { sessionId, agentWindowId, refStore: { resolve: () => null }, borrowedTabs: new Map() }
|
|
||||||
: null,
|
|
||||||
findByWindowId: (wid: number) => (wid === agentWindowId ? { sessionId } : null),
|
findByWindowId: (wid: number) => (wid === agentWindowId ? { sessionId } : null),
|
||||||
} as unknown as SessionManager;
|
} as unknown as SessionManager;
|
||||||
return mgr;
|
return mgr;
|
||||||
@@ -339,26 +339,23 @@ describe("handleRequestHelp", () => {
|
|||||||
|
|
||||||
ac.abort();
|
ac.abort();
|
||||||
await expect(pending).resolves.toMatchObject({ code: "cancelled" });
|
await expect(pending).resolves.toMatchObject({ code: "cancelled" });
|
||||||
expect(cdpSend).toHaveBeenCalledWith(
|
expect(cdpSend).not.toHaveBeenCalledWith(
|
||||||
5,
|
expect.anything(),
|
||||||
"DOM.querySelectorAll",
|
"DOM.removeAttribute",
|
||||||
expect.objectContaining({ selector: "[data-bsk-help]" }),
|
expect.anything(),
|
||||||
);
|
|
||||||
expect(cdpSend).toHaveBeenCalledWith(
|
|
||||||
6,
|
|
||||||
"DOM.querySelectorAll",
|
|
||||||
expect.objectContaining({ selector: "[data-bsk-help]" }),
|
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("tags ref targets via CDP and reports them matched", async () => {
|
it("resolves ref targets to explicit viewport rectangles without mutating the page", async () => {
|
||||||
|
const refStore = new RefStore();
|
||||||
|
refStore.set("e1", 42, { tabId: 5 });
|
||||||
const mgr = {
|
const mgr = {
|
||||||
get: (id: string) =>
|
get: (id: string) =>
|
||||||
id === "abcd"
|
id === "abcd"
|
||||||
? {
|
? {
|
||||||
sessionId: "abcd",
|
sessionId: "abcd",
|
||||||
agentWindowId: 99,
|
agentWindowId: 99,
|
||||||
refStore: { resolve: () => 42 },
|
refStore,
|
||||||
borrowedTabs: new Map(),
|
borrowedTabs: new Map(),
|
||||||
}
|
}
|
||||||
: null,
|
: null,
|
||||||
@@ -366,7 +363,17 @@ describe("handleRequestHelp", () => {
|
|||||||
} as unknown as SessionManager;
|
} as unknown as SessionManager;
|
||||||
const deps = baseDeps({
|
const deps = baseDeps({
|
||||||
cdp: {
|
cdp: {
|
||||||
send: vi.fn(async () => ({ object: { objectId: "obj-1" } })),
|
send: vi.fn(async (_tabId, method) => {
|
||||||
|
if (method === "DOM.getDocument") return { root: { nodeId: 1 } };
|
||||||
|
if (method === "DOM.scrollIntoViewIfNeeded") return {};
|
||||||
|
if (method === "DOM.getContentQuads") {
|
||||||
|
return { quads: [[10, 20, 110, 20, 110, 60, 10, 60]] };
|
||||||
|
}
|
||||||
|
if (method === "Page.getLayoutMetrics") {
|
||||||
|
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
|
||||||
|
}
|
||||||
|
throw new Error(`unexpected ${method}`);
|
||||||
|
}),
|
||||||
} as unknown as RequestHelpDeps["cdp"],
|
} as unknown as RequestHelpDeps["cdp"],
|
||||||
});
|
});
|
||||||
const res = await handleRequestHelp(
|
const res = await handleRequestHelp(
|
||||||
@@ -375,14 +382,97 @@ describe("handleRequestHelp", () => {
|
|||||||
deps,
|
deps,
|
||||||
);
|
);
|
||||||
const sentMsg = (deps.sendToTab as ReturnType<typeof vi.fn>).mock.calls[0][1];
|
const sentMsg = (deps.sendToTab as ReturnType<typeof vi.fn>).mock.calls[0][1];
|
||||||
expect(sentMsg.selectors).toContain('[data-bsk-help="0"]');
|
expect(sentMsg.selectors).toEqual([]);
|
||||||
|
expect(sentMsg.rects).toEqual([{ top: 20, left: 10, width: 100, height: 40 }]);
|
||||||
expect(res).toMatchObject({
|
expect(res).toMatchObject({
|
||||||
outcome: "continued",
|
outcome: "continued",
|
||||||
tab_id: 5,
|
tab_id: 5,
|
||||||
resolved_targets: [{ matched: true, ref: "@e1" }],
|
resolved_targets: [
|
||||||
|
{
|
||||||
|
matched: true,
|
||||||
|
ref: "@e1",
|
||||||
|
},
|
||||||
|
],
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("projects an OOPIF ref through its live content quad for the help overlay", async () => {
|
||||||
|
const mgr = {
|
||||||
|
get: (id: string) =>
|
||||||
|
id === "abcd"
|
||||||
|
? {
|
||||||
|
sessionId: "abcd",
|
||||||
|
agentWindowId: 99,
|
||||||
|
refStore: {
|
||||||
|
resolveEntry: () => ({
|
||||||
|
backendNodeId: 42,
|
||||||
|
tabId: 5,
|
||||||
|
frameId: "child",
|
||||||
|
cdpSessionId: "child-session",
|
||||||
|
generation: 1,
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
borrowedTabs: new Map(),
|
||||||
|
}
|
||||||
|
: null,
|
||||||
|
findByWindowId: (wid: number) => (wid === 99 ? { sessionId: "abcd" } : null),
|
||||||
|
} as unknown as SessionManager;
|
||||||
|
const send = vi.fn(async (_tabId, method) => {
|
||||||
|
if (method === "DOM.getDocument") return { root: { nodeId: 1 } };
|
||||||
|
if (method === "DOM.scrollIntoViewIfNeeded") return {};
|
||||||
|
if (method === "DOM.getBoxModel") {
|
||||||
|
return { model: { content: [204, 306, 604, 306, 604, 506, 204, 506] } };
|
||||||
|
}
|
||||||
|
if (method === "Page.getLayoutMetrics") {
|
||||||
|
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
|
||||||
|
}
|
||||||
|
throw new Error(`unexpected root ${method}`);
|
||||||
|
});
|
||||||
|
const sendToTarget = vi.fn(async (_target, method) => {
|
||||||
|
if (method === "DOM.scrollIntoViewIfNeeded") return {};
|
||||||
|
if (method === "DOM.getContentQuads") {
|
||||||
|
return { quads: [[10, 20, 110, 20, 110, 60, 10, 60]] };
|
||||||
|
}
|
||||||
|
if (method === "Page.getLayoutMetrics") {
|
||||||
|
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
|
||||||
|
}
|
||||||
|
throw new Error(`unexpected child ${method}`);
|
||||||
|
});
|
||||||
|
const deps = baseDeps({
|
||||||
|
cdp: {
|
||||||
|
send,
|
||||||
|
sendToTarget,
|
||||||
|
getFrameGraph: vi.fn(async () => ({
|
||||||
|
rootFrameId: "main",
|
||||||
|
frames: [
|
||||||
|
{ frameId: "main", target: { tabId: 5 } },
|
||||||
|
{
|
||||||
|
frameId: "child",
|
||||||
|
parentFrameId: "main",
|
||||||
|
ownerBackendNodeId: 99,
|
||||||
|
target: { tabId: 5, sessionId: "child-session" },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
})),
|
||||||
|
} as unknown as NonNullable<RequestHelpDeps["cdp"]>,
|
||||||
|
});
|
||||||
|
|
||||||
|
const res = await handleRequestHelp(
|
||||||
|
mgr,
|
||||||
|
baseParams({ tab_id: 5, targets: [{ ref: "@e1" }] }),
|
||||||
|
deps,
|
||||||
|
);
|
||||||
|
|
||||||
|
const sentMsg = (deps.sendToTab as ReturnType<typeof vi.fn>).mock.calls[0][1];
|
||||||
|
expect(sentMsg.rects).toEqual([{ top: 346, left: 224, width: 200, height: 80 }]);
|
||||||
|
expect(res).toMatchObject({ resolved_targets: [{ matched: true, ref: "@e1" }] });
|
||||||
|
expect(sendToTarget).toHaveBeenCalledWith(
|
||||||
|
{ tabId: 5, sessionId: "child-session" },
|
||||||
|
"DOM.getContentQuads",
|
||||||
|
{ backendNodeId: 42 },
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
it("reports ref target unmatched when ref does not resolve", async () => {
|
it("reports ref target unmatched when ref does not resolve", async () => {
|
||||||
const deps = baseDeps();
|
const deps = baseDeps();
|
||||||
const res = await handleRequestHelp(
|
const res = await handleRequestHelp(
|
||||||
@@ -398,16 +488,15 @@ describe("handleRequestHelp", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it("reports ref target unmatched when ref is for another tab", async () => {
|
it("reports ref target unmatched when ref is for another tab", async () => {
|
||||||
|
const refStore = new RefStore();
|
||||||
|
refStore.set("e1", 42, { tabId: 4 });
|
||||||
const mgr = {
|
const mgr = {
|
||||||
get: (id: string) =>
|
get: (id: string) =>
|
||||||
id === "abcd"
|
id === "abcd"
|
||||||
? {
|
? {
|
||||||
sessionId: "abcd",
|
sessionId: "abcd",
|
||||||
agentWindowId: 99,
|
agentWindowId: 99,
|
||||||
refStore: {
|
refStore,
|
||||||
resolve: (ref: string, opts: { tabId?: number }) =>
|
|
||||||
ref === "e1" && opts.tabId === 4 ? 42 : null,
|
|
||||||
},
|
|
||||||
borrowedTabs: new Map(),
|
borrowedTabs: new Map(),
|
||||||
}
|
}
|
||||||
: null,
|
: null,
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import type { CdpRunner } from "@/tools/shared";
|
|||||||
import {
|
import {
|
||||||
handleClick,
|
handleClick,
|
||||||
handleFill,
|
handleFill,
|
||||||
|
handleHover,
|
||||||
handlePress,
|
handlePress,
|
||||||
handleSelect,
|
handleSelect,
|
||||||
modifiersBitfield,
|
modifiersBitfield,
|
||||||
@@ -29,6 +30,9 @@ function makeFakeCdp(handlers: Record<string, (params: unknown) => unknown>) {
|
|||||||
const sendImpl = async (tabId: number, method: string, params?: object) => {
|
const sendImpl = async (tabId: number, method: string, params?: object) => {
|
||||||
sent.push({ tabId, method, params });
|
sent.push({ tabId, method, params });
|
||||||
const h = handlers[method];
|
const h = handlers[method];
|
||||||
|
if (!h && method === "Page.getLayoutMetrics") {
|
||||||
|
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
|
||||||
|
}
|
||||||
if (!h) throw new Error(`unexpected CDP call ${method}`);
|
if (!h) throw new Error(`unexpected CDP call ${method}`);
|
||||||
return h(params);
|
return h(params);
|
||||||
};
|
};
|
||||||
@@ -163,6 +167,63 @@ describe("handleClick", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("resolves frame refs in their CDP session and dispatches input in top coordinates", async () => {
|
||||||
|
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||||
|
const ctx = await sm.start("aa11");
|
||||||
|
ctx.refStore.set("e3", 1234, {
|
||||||
|
tabId: 4,
|
||||||
|
frameId: "child-frame",
|
||||||
|
cdpSessionId: "child-session",
|
||||||
|
});
|
||||||
|
const fake = makeFakeCdp({
|
||||||
|
"Input.dispatchMouseEvent": () => ({}),
|
||||||
|
"DOM.scrollIntoViewIfNeeded": () => ({}),
|
||||||
|
"DOM.getBoxModel": () => ({
|
||||||
|
model: { content: [204, 306, 604, 306, 604, 506, 204, 506] },
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
fake.cdp.getFrameGraph = vi.fn(async () => ({
|
||||||
|
rootFrameId: "main",
|
||||||
|
frames: [
|
||||||
|
{ frameId: "main", target: { tabId: 4 } },
|
||||||
|
{
|
||||||
|
frameId: "child-frame",
|
||||||
|
parentFrameId: "main",
|
||||||
|
ownerBackendNodeId: 99,
|
||||||
|
target: { tabId: 4, sessionId: "child-session" },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
}));
|
||||||
|
const targetCalls: Array<{ sessionId?: string; method: string }> = [];
|
||||||
|
fake.cdp.sendToTarget = vi.fn(async (target, method) => {
|
||||||
|
targetCalls.push({ sessionId: target.sessionId, method });
|
||||||
|
if (method === "DOM.scrollIntoViewIfNeeded") return {};
|
||||||
|
if (method === "DOM.getContentQuads") {
|
||||||
|
return { quads: [[10, 20, 110, 20, 110, 60, 10, 60]] };
|
||||||
|
}
|
||||||
|
if (method === "Page.getLayoutMetrics") {
|
||||||
|
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
|
||||||
|
}
|
||||||
|
throw new Error(`unexpected child CDP call ${method}`);
|
||||||
|
}) as CdpRunner["sendToTarget"];
|
||||||
|
|
||||||
|
const res = await handleClick(
|
||||||
|
sm,
|
||||||
|
{ session_id: "aa11", ref: "@e3" },
|
||||||
|
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
||||||
|
);
|
||||||
|
|
||||||
|
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||||
|
// The iframe content box starts at (204, 306) after its border and is scaled 2x.
|
||||||
|
expect(res).toMatchObject({ x: 324, y: 386 });
|
||||||
|
expect(targetCalls).toEqual([
|
||||||
|
{ sessionId: "child-session", method: "DOM.scrollIntoViewIfNeeded" },
|
||||||
|
{ sessionId: "child-session", method: "DOM.getContentQuads" },
|
||||||
|
{ sessionId: "child-session", method: "Page.getLayoutMetrics" },
|
||||||
|
]);
|
||||||
|
expect(fake.sent.filter((call) => call.method === "Input.dispatchMouseEvent")).toHaveLength(3);
|
||||||
|
});
|
||||||
|
|
||||||
it("enables overlay bypass before mouse events when overlay blocks the click point", async () => {
|
it("enables overlay bypass before mouse events when overlay blocks the click point", async () => {
|
||||||
const order: string[] = [];
|
const order: string[] = [];
|
||||||
const bypassOverlay = vi.fn(async (_tabId: number, enabled: boolean) => {
|
const bypassOverlay = vi.fn(async (_tabId: number, enabled: boolean) => {
|
||||||
@@ -435,6 +496,50 @@ describe("handleClick", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe("handleHover", () => {
|
||||||
|
it("keeps overlay bypass enabled after a successful hover when requested", async () => {
|
||||||
|
const bypassOverlay = vi.fn().mockResolvedValue(undefined);
|
||||||
|
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||||
|
const ctx = await sm.start("aa11");
|
||||||
|
ctx.refStore.set("e3", 1234, { tabId: 4 });
|
||||||
|
const fake = makeFakeCdp({
|
||||||
|
"DOM.scrollIntoViewIfNeeded": () => ({}),
|
||||||
|
"DOM.getContentQuads": () => ({ quads: [[10, 20, 110, 20, 110, 60, 10, 60]] }),
|
||||||
|
"Runtime.evaluate": (params: unknown) => {
|
||||||
|
const expr = String((params as { expression?: string })?.expression ?? "");
|
||||||
|
if (expr.includes("overlayHostPresent") && !expr.includes("hitIndex")) {
|
||||||
|
return { result: { value: { overlayHostPresent: true, overlayHostConnected: true } } };
|
||||||
|
}
|
||||||
|
if (expr.includes("hitIndex")) {
|
||||||
|
return {
|
||||||
|
result: {
|
||||||
|
value: { overlayHostPresent: true, overlayHostConnected: true, hitIndex: 0 },
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
throw new Error(`unexpected Runtime.evaluate: ${expr.slice(0, 80)}`);
|
||||||
|
},
|
||||||
|
"Input.dispatchMouseEvent": () => ({}),
|
||||||
|
});
|
||||||
|
|
||||||
|
const res = await handleHover(
|
||||||
|
sm,
|
||||||
|
{ session_id: "aa11", ref: "@e3", settle_ms: 0 },
|
||||||
|
{
|
||||||
|
cdp: fake.cdp,
|
||||||
|
tabsApi: fake.tabsApi,
|
||||||
|
bypassOverlay,
|
||||||
|
keepOverlayBypassAfterHover: true,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||||
|
expect(res).toMatchObject({ tab_id: 4, used_ref: "e3", x: 60, y: 40 });
|
||||||
|
expect(bypassOverlay).toHaveBeenCalledTimes(1);
|
||||||
|
expect(bypassOverlay).toHaveBeenCalledWith(4, true);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
describe("handleFill", () => {
|
describe("handleFill", () => {
|
||||||
it("returns not_found for unknown ref", async () => {
|
it("returns not_found for unknown ref", async () => {
|
||||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||||
|
|||||||
@@ -49,6 +49,8 @@ function makeFakeCdp(opts?: {
|
|||||||
beforeLoaderId?: string;
|
beforeLoaderId?: string;
|
||||||
readyState?: string;
|
readyState?: string;
|
||||||
readyStateSequence?: string[];
|
readyStateSequence?: string[];
|
||||||
|
/** Make the readyState probe's `Runtime.evaluate` throw (returns null). */
|
||||||
|
evaluateThrows?: boolean;
|
||||||
}) {
|
}) {
|
||||||
const readyStates = opts?.readyStateSequence ?? (opts?.readyState ? [opts.readyState] : []);
|
const readyStates = opts?.readyStateSequence ?? (opts?.readyState ? [opts.readyState] : []);
|
||||||
let readyStateIdx = 0;
|
let readyStateIdx = 0;
|
||||||
@@ -94,7 +96,10 @@ function makeFakeCdp(opts?: {
|
|||||||
return {};
|
return {};
|
||||||
},
|
},
|
||||||
"Runtime.enable": () => ({}),
|
"Runtime.enable": () => ({}),
|
||||||
"Runtime.evaluate": () => ({ result: { value: nextReadyState() } }),
|
"Runtime.evaluate": () => {
|
||||||
|
if (opts?.evaluateThrows) throw new Error("Cannot execute in a destroyed execution context");
|
||||||
|
return { result: { value: nextReadyState() } };
|
||||||
|
},
|
||||||
"Page.getNavigationHistory": () => ({
|
"Page.getNavigationHistory": () => ({
|
||||||
currentIndex: opts?.historyIndex ?? 1,
|
currentIndex: opts?.historyIndex ?? 1,
|
||||||
entries: opts?.historyEntries ?? [
|
entries: opts?.historyEntries ?? [
|
||||||
@@ -168,8 +173,13 @@ function makeFakeCdp(opts?: {
|
|||||||
const payload = { name, frameId, loaderId };
|
const payload = { name, frameId, loaderId };
|
||||||
for (const listener of [...events]) listener({ tabId: 4 }, "Page.lifecycleEvent", payload);
|
for (const listener of [...events]) listener({ tabId: 4 }, "Page.lifecycleEvent", payload);
|
||||||
},
|
},
|
||||||
fireFrameNavigated(frameId = opts?.navigateFrameId ?? "frame-1") {
|
fireFrameNavigated(
|
||||||
const payload = { frame: { id: frameId, url: opts?.finalUrl ?? "https://example.com/" } };
|
frameId = opts?.navigateFrameId ?? "frame-1",
|
||||||
|
loaderId = opts?.navigateLoaderId ?? "loader-after",
|
||||||
|
) {
|
||||||
|
const payload = {
|
||||||
|
frame: { id: frameId, loaderId, url: opts?.finalUrl ?? "https://example.com/" },
|
||||||
|
};
|
||||||
for (const listener of [...events]) listener({ tabId: 4 }, "Page.frameNavigated", payload);
|
for (const listener of [...events]) listener({ tabId: 4 }, "Page.frameNavigated", payload);
|
||||||
},
|
},
|
||||||
listeners: events,
|
listeners: events,
|
||||||
@@ -457,6 +467,34 @@ describe("handleNavigate", () => {
|
|||||||
expect(res.reached).toBe("commit");
|
expect(res.reached).toBe("commit");
|
||||||
expect(fake.listeners.length).toBe(0);
|
expect(fake.listeners.length).toBe(0);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("resolves wait_until=commit from Page.frameNavigated when the readyState probe fails", async () => {
|
||||||
|
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||||
|
await sm.start("aa11");
|
||||||
|
// Cross-origin-style navigation: the readyState probe can't read the
|
||||||
|
// (already destroyed) old execution context, so commit must come from
|
||||||
|
// the Page.frameNavigated event. Before the fix this timed out because
|
||||||
|
// the handler dropped the frame's loaderId and the loader guard then
|
||||||
|
// rejected the event.
|
||||||
|
const fake = makeFakeCdp({ evaluateThrows: true });
|
||||||
|
const navP = handleNavigate(
|
||||||
|
sm,
|
||||||
|
{
|
||||||
|
session_id: "aa11",
|
||||||
|
url: "https://example.com/",
|
||||||
|
wait_until: "commit",
|
||||||
|
timeout_ms: 1_000,
|
||||||
|
},
|
||||||
|
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
||||||
|
);
|
||||||
|
|
||||||
|
await new Promise((r) => setTimeout(r, 5));
|
||||||
|
fake.fireFrameNavigated("frame-1", "loader-after");
|
||||||
|
const res = await navP;
|
||||||
|
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||||
|
expect(res.reached).toBe("commit");
|
||||||
|
expect(fake.listeners.length).toBe(0);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
describe("handleNavigateBack / Forward", () => {
|
describe("handleNavigateBack / Forward", () => {
|
||||||
|
|||||||
File diff suppressed because it is too large.
Load diff
@@ -60,11 +60,58 @@ function makeApis(
|
|||||||
}),
|
}),
|
||||||
getLastFocused: vi.fn(async () => ({ id: 500 }) as chrome.windows.Window),
|
getLastFocused: vi.fn(async () => ({ id: 500 }) as chrome.windows.Window),
|
||||||
create: vi.fn(async () => ({ id: 999 }) as chrome.windows.Window),
|
create: vi.fn(async () => ({ id: 999 }) as chrome.windows.Window),
|
||||||
|
remove: vi.fn(async () => {}),
|
||||||
};
|
};
|
||||||
return { tabs, windows };
|
return { tabs, windows };
|
||||||
}
|
}
|
||||||
|
|
||||||
describe("handleSessionStop with auto-return", () => {
|
describe("handleSessionStop with auto-return", () => {
|
||||||
|
it("leaves the session untouched when cancellation arrived before teardown", async () => {
|
||||||
|
const aw = fakeAgentWindow([100]);
|
||||||
|
const sm = new SessionManager({ agentWindow: aw });
|
||||||
|
await sm.start("aa11");
|
||||||
|
const cdp = { detachSession: vi.fn(async () => {}) };
|
||||||
|
const controller = new AbortController();
|
||||||
|
controller.abort();
|
||||||
|
|
||||||
|
const res = await handleSessionStop(
|
||||||
|
sm,
|
||||||
|
{ session_id: "aa11" },
|
||||||
|
{ cdp, signal: controller.signal },
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(res).toMatchObject({ code: "cancelled" });
|
||||||
|
expect(sm.has("aa11")).toBe(true);
|
||||||
|
expect(cdp.detachSession).not.toHaveBeenCalled();
|
||||||
|
expect(aw.remove).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not close the Agent Window when cancellation lands during CDP detach", async () => {
|
||||||
|
const aw = fakeAgentWindow([100]);
|
||||||
|
const sm = new SessionManager({ agentWindow: aw });
|
||||||
|
await sm.start("aa11");
|
||||||
|
const controller = new AbortController();
|
||||||
|
let finishDetach: () => void = () => {};
|
||||||
|
const detach = new Promise<void>((resolve) => {
|
||||||
|
finishDetach = resolve;
|
||||||
|
});
|
||||||
|
const cdp = { detachSession: vi.fn(() => detach) };
|
||||||
|
|
||||||
|
const stopping = handleSessionStop(
|
||||||
|
sm,
|
||||||
|
{ session_id: "aa11" },
|
||||||
|
{ cdp, signal: controller.signal },
|
||||||
|
);
|
||||||
|
await vi.waitFor(() => expect(cdp.detachSession).toHaveBeenCalledWith("aa11"));
|
||||||
|
controller.abort();
|
||||||
|
finishDetach();
|
||||||
|
|
||||||
|
const res = await stopping;
|
||||||
|
expect(res).toMatchObject({ code: "cancelled" });
|
||||||
|
expect(sm.has("aa11")).toBe(true);
|
||||||
|
expect(aw.remove).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
it("returns every borrowed tab and closes the Agent Window in the right order", async () => {
|
it("returns every borrowed tab and closes the Agent Window in the right order", async () => {
|
||||||
const aw = fakeAgentWindow([100]);
|
const aw = fakeAgentWindow([100]);
|
||||||
const sm = new SessionManager({ agentWindow: aw });
|
const sm = new SessionManager({ agentWindow: aw });
|
||||||
|
|||||||
@@ -46,6 +46,25 @@ describe("lookupSnapshotRef", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
describe("resolveSnapshotRef", () => {
|
describe("resolveSnapshotRef", () => {
|
||||||
|
it("preserves the frame and child session needed to route iframe refs", async () => {
|
||||||
|
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||||
|
const ctx = await sm.start("aa11");
|
||||||
|
ctx.refStore.set("e3", 1234, {
|
||||||
|
tabId: 4,
|
||||||
|
frameId: "child-frame",
|
||||||
|
cdpSessionId: "child-session",
|
||||||
|
});
|
||||||
|
|
||||||
|
const expected = {
|
||||||
|
backendNodeId: 1234,
|
||||||
|
refKey: "e3",
|
||||||
|
frameId: "child-frame",
|
||||||
|
cdpSessionId: "child-session",
|
||||||
|
};
|
||||||
|
expect(lookupSnapshotRef(ctx, "@e3", 4)).toEqual(expected);
|
||||||
|
expect(resolveSnapshotRef(ctx, "@e3", 4)).toEqual(expected);
|
||||||
|
});
|
||||||
|
|
||||||
it("returns not_found for unknown ref", async () => {
|
it("returns not_found for unknown ref", async () => {
|
||||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||||
const ctx = await sm.start("aa11");
|
const ctx = await sm.start("aa11");
|
||||||
|
|||||||
@@ -179,6 +179,7 @@ function makeWindowsApi(
|
|||||||
get: ReturnType<typeof vi.fn>;
|
get: ReturnType<typeof vi.fn>;
|
||||||
lastFocused: ReturnType<typeof vi.fn>;
|
lastFocused: ReturnType<typeof vi.fn>;
|
||||||
create: ReturnType<typeof vi.fn>;
|
create: ReturnType<typeof vi.fn>;
|
||||||
|
remove: ReturnType<typeof vi.fn>;
|
||||||
};
|
};
|
||||||
} {
|
} {
|
||||||
const get = vi.fn(async (windowId: number) => {
|
const get = vi.fn(async (windowId: number) => {
|
||||||
@@ -197,7 +198,13 @@ function makeWindowsApi(
|
|||||||
const id = opts?.createWindowId ?? 999;
|
const id = opts?.createWindowId ?? 999;
|
||||||
return { id } as chrome.windows.Window;
|
return { id } as chrome.windows.Window;
|
||||||
});
|
});
|
||||||
return { api: { get, getLastFocused: lastFocused, create }, spies: { get, lastFocused, create } };
|
const remove = vi.fn(async (windowId: number) => {
|
||||||
|
state.windowsClosed.add(windowId);
|
||||||
|
});
|
||||||
|
return {
|
||||||
|
api: { get, getLastFocused: lastFocused, create, remove },
|
||||||
|
spies: { get, lastFocused, create, remove },
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
describe("handleTabCreate", () => {
|
describe("handleTabCreate", () => {
|
||||||
@@ -601,6 +608,86 @@ describe("handleTabReturn", () => {
|
|||||||
expect(winSpies.create).toHaveBeenCalledOnce();
|
expect(winSpies.create).toHaveBeenCalledOnce();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("closes a newly created fallback window when cancellation wins before the move", async () => {
|
||||||
|
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||||
|
const ctx = await sm.start("aa11");
|
||||||
|
ctx.borrowedTabs.set(7, { tabId: 7, originalWindowId: 200, originalIndex: 4 });
|
||||||
|
const state: FakeTabState = {
|
||||||
|
tabs: new Map([[7, { id: 7, windowId: 100 } as chrome.tabs.Tab]]),
|
||||||
|
nextTabId: 50,
|
||||||
|
windowsClosed: new Set([200]),
|
||||||
|
};
|
||||||
|
const { api, spies } = makeTabMutationApi(state);
|
||||||
|
const { api: windowsApi, spies: winSpies } = makeWindowsApi(state, {
|
||||||
|
lastFocused: 100,
|
||||||
|
createWindowId: 777,
|
||||||
|
});
|
||||||
|
let resolveCreate: (window: chrome.windows.Window) => void = () => {};
|
||||||
|
winSpies.create.mockImplementationOnce(
|
||||||
|
() =>
|
||||||
|
new Promise((resolve) => {
|
||||||
|
resolveCreate = resolve;
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
const controller = new AbortController();
|
||||||
|
|
||||||
|
const pending = handleTabReturn(
|
||||||
|
sm,
|
||||||
|
{ session_id: "aa11", tab_id: 7 },
|
||||||
|
{ tabs: api, windows: windowsApi, signal: controller.signal },
|
||||||
|
);
|
||||||
|
await vi.waitFor(() => expect(winSpies.create).toHaveBeenCalledOnce());
|
||||||
|
controller.abort();
|
||||||
|
resolveCreate({ id: 777 } as chrome.windows.Window);
|
||||||
|
|
||||||
|
await expect(pending).resolves.toMatchObject({ code: "cancelled" });
|
||||||
|
expect(winSpies.remove).toHaveBeenCalledWith(777);
|
||||||
|
expect(spies.move).not.toHaveBeenCalled();
|
||||||
|
expect(ctx.borrowedTabs.has(7)).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("surfaces the fallback window id when cancellation cleanup fails", async () => {
|
||||||
|
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||||
|
const ctx = await sm.start("aa11");
|
||||||
|
ctx.borrowedTabs.set(7, { tabId: 7, originalWindowId: 200, originalIndex: 4 });
|
||||||
|
const state: FakeTabState = {
|
||||||
|
tabs: new Map([[7, { id: 7, windowId: 100 } as chrome.tabs.Tab]]),
|
||||||
|
nextTabId: 50,
|
||||||
|
windowsClosed: new Set([200]),
|
||||||
|
};
|
||||||
|
const { api, spies } = makeTabMutationApi(state);
|
||||||
|
const { api: windowsApi, spies: winSpies } = makeWindowsApi(state, {
|
||||||
|
lastFocused: 100,
|
||||||
|
createWindowId: 777,
|
||||||
|
});
|
||||||
|
let resolveCreate: (window: chrome.windows.Window) => void = () => {};
|
||||||
|
winSpies.create.mockImplementationOnce(
|
||||||
|
() =>
|
||||||
|
new Promise((resolve) => {
|
||||||
|
resolveCreate = resolve;
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
winSpies.remove.mockRejectedValueOnce(new Error("window removal denied"));
|
||||||
|
const controller = new AbortController();
|
||||||
|
|
||||||
|
const pending = handleTabReturn(
|
||||||
|
sm,
|
||||||
|
{ session_id: "aa11", tab_id: 7 },
|
||||||
|
{ tabs: api, windows: windowsApi, signal: controller.signal },
|
||||||
|
);
|
||||||
|
await vi.waitFor(() => expect(winSpies.create).toHaveBeenCalledOnce());
|
||||||
|
controller.abort();
|
||||||
|
resolveCreate({ id: 777 } as chrome.windows.Window);
|
||||||
|
|
||||||
|
await expect(pending).resolves.toMatchObject({
|
||||||
|
code: "protocol_error",
|
||||||
|
data: { reason: "cleanup_failed", resource_type: "window", resource_id: 777 },
|
||||||
|
message: expect.stringMatching(/fallback window 777.*window removal denied/),
|
||||||
|
});
|
||||||
|
expect(spies.move).not.toHaveBeenCalled();
|
||||||
|
expect(ctx.borrowedTabs.has(7)).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
it("falls back to a new window when getLastFocused only returns the Agent Window", async () => {
|
it("falls back to a new window when getLastFocused only returns the Agent Window", async () => {
|
||||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||||
const ctx = await sm.start("aa11");
|
const ctx = await sm.start("aa11");
|
||||||
|
|||||||
@@ -107,7 +107,7 @@ describe("handleSessionStart window size", () => {
|
|||||||
const sm = new SessionManager({ agentWindow: aw });
|
const sm = new SessionManager({ agentWindow: aw });
|
||||||
const result = await handleSessionStart(sm, { session_id: "aa11", width: 1280, height: 800 });
|
const result = await handleSessionStart(sm, { session_id: "aa11", width: 1280, height: 800 });
|
||||||
expect(result).toEqual({ agent_window_id: 100 });
|
expect(result).toEqual({ agent_window_id: 100 });
|
||||||
expect(aw.create).toHaveBeenCalledWith("about:blank", { width: 1280, height: 800 });
|
expect(aw.create).toHaveBeenCalledWith("about:blank", { size: { width: 1280, height: 800 } });
|
||||||
});
|
});
|
||||||
|
|
||||||
it("creates the window without size when width/height are omitted", async () => {
|
it("creates the window without size when width/height are omitted", async () => {
|
||||||
@@ -115,7 +115,15 @@ describe("handleSessionStart window size", () => {
|
|||||||
const sm = new SessionManager({ agentWindow: aw });
|
const sm = new SessionManager({ agentWindow: aw });
|
||||||
const result = await handleSessionStart(sm, { session_id: "aa11" });
|
const result = await handleSessionStart(sm, { session_id: "aa11" });
|
||||||
expect(result).toEqual({ agent_window_id: 100 });
|
expect(result).toEqual({ agent_window_id: 100 });
|
||||||
expect(aw.create).toHaveBeenCalledWith("about:blank");
|
expect(aw.create).toHaveBeenCalledWith("about:blank", {});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("forwards focused: false to the Agent Window", async () => {
|
||||||
|
const aw = fakeAgentWindow([100]);
|
||||||
|
const sm = new SessionManager({ agentWindow: aw });
|
||||||
|
const result = await handleSessionStart(sm, { session_id: "aa11", focused: false });
|
||||||
|
expect(result).toEqual({ agent_window_id: 100 });
|
||||||
|
expect(aw.create).toHaveBeenCalledWith("about:blank", { focused: false });
|
||||||
});
|
});
|
||||||
|
|
||||||
it("rejects a lone width without height", async () => {
|
it("rejects a lone width without height", async () => {
|
||||||
|
|||||||
@@ -250,9 +250,10 @@ const DEFAULT_NOTIFICATION_COPY: BorrowNotificationCopy = {
|
|||||||
// URL injection-eligibility test
|
// URL injection-eligibility test
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
const CHROME_WEB_STORE_RES = [
|
const EXTENSION_STORE_RES = [
|
||||||
/^https:\/\/chrome\.google\.com\/webstore/i,
|
/^https:\/\/chrome\.google\.com\/webstore/i,
|
||||||
/^https:\/\/chromewebstore\.google\.com/i,
|
/^https:\/\/chromewebstore\.google\.com/i,
|
||||||
|
/^https:\/\/microsoftedge\.microsoft\.com\/addons/i,
|
||||||
];
|
];
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -264,14 +265,15 @@ const CHROME_WEB_STORE_RES = [
|
|||||||
* - `ftp://` no longer hosts content scripts reliably in modern Chrome.
|
* - `ftp://` no longer hosts content scripts reliably in modern Chrome.
|
||||||
* - `about:` / `chrome:` / `chrome-extension:` / `edge:` / `devtools:` /
|
* - `about:` / `chrome:` / `chrome-extension:` / `edge:` / `devtools:` /
|
||||||
* `view-source:` / `data:` / `blob:` are blocked by the platform.
|
* `view-source:` / `data:` / `blob:` are blocked by the platform.
|
||||||
* - The Chrome Web Store (both legacy and new domains) is also blocked.
|
* - Extension storefronts are blocked by their own browser: the Chrome Web
|
||||||
|
* Store (legacy and new domains) under Chrome, and Edge Add-ons under Edge.
|
||||||
*
|
*
|
||||||
* Mirrors the implicit scheme list documented in
|
* Mirrors the implicit scheme list documented in
|
||||||
* https://developer.chrome.com/docs/extensions/develop/concepts/match-patterns.
|
* https://developer.chrome.com/docs/extensions/develop/concepts/match-patterns.
|
||||||
*/
|
*/
|
||||||
export function isInjectableContentScriptUrl(url: string | undefined): boolean {
|
export function isInjectableContentScriptUrl(url: string | undefined): boolean {
|
||||||
if (!url) return false;
|
if (!url) return false;
|
||||||
for (const re of CHROME_WEB_STORE_RES) {
|
for (const re of EXTENSION_STORE_RES) {
|
||||||
if (re.test(url)) return false;
|
if (re.test(url)) return false;
|
||||||
}
|
}
|
||||||
return /^https?:\/\//i.test(url);
|
return /^https?:\/\//i.test(url);
|
||||||
|
|||||||
@@ -0,0 +1,9 @@
|
|||||||
|
export {
|
||||||
|
type CaptureVomObservationOptions,
|
||||||
|
captureVomObservation,
|
||||||
|
} from "./observation";
|
||||||
|
export {
|
||||||
|
type CaptureVomFrame,
|
||||||
|
type CaptureVomMatchNode,
|
||||||
|
type CaptureVomObservationResult,
|
||||||
|
} from "./vom/record-safe-observation";
|
||||||
@@ -27,19 +27,23 @@ export async function handleConsole(
|
|||||||
manager: SessionManager,
|
manager: SessionManager,
|
||||||
params: ConsoleParams,
|
params: ConsoleParams,
|
||||||
deps: ConsoleDeps = defaultConsoleDeps(),
|
deps: ConsoleDeps = defaultConsoleDeps(),
|
||||||
|
signal?: AbortSignal,
|
||||||
): Promise<ConsoleResult | RpcError> {
|
): Promise<ConsoleResult | RpcError> {
|
||||||
|
if (signal?.aborted) return { code: "cancelled", message: "console aborted" };
|
||||||
const ctxOrErr = lookupSession(manager, params, "console");
|
const ctxOrErr = lookupSession(manager, params, "console");
|
||||||
if (isRpcError(ctxOrErr)) return ctxOrErr;
|
if (isRpcError(ctxOrErr)) return ctxOrErr;
|
||||||
const bounds = parseBufferedReadBounds(params);
|
const bounds = parseBufferedReadBounds(params);
|
||||||
if (isRpcError(bounds)) return bounds;
|
if (isRpcError(bounds)) return bounds;
|
||||||
const target = await resolveTargetTab(manager, ctxOrErr, params.tab_id, deps.tabsApi);
|
const target = await resolveTargetTab(manager, ctxOrErr, params.tab_id, deps.tabsApi);
|
||||||
if (isRpcError(target)) return target;
|
if (isRpcError(target)) return target;
|
||||||
|
if (signal?.aborted) return { code: "cancelled", message: "console aborted" };
|
||||||
if (!deps.cdp.ensureConsoleCapture || !deps.cdp.consoleEntriesSince) {
|
if (!deps.cdp.ensureConsoleCapture || !deps.cdp.consoleEntriesSince) {
|
||||||
return { code: "cdp_failed", message: "console capture requires CDP console support" };
|
return { code: "cdp_failed", message: "console capture requires CDP console support" };
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
await deps.cdp.ensureConsoleCapture(target.tabId);
|
await deps.cdp.ensureConsoleCapture(target.tabId);
|
||||||
|
if (signal?.aborted) return { code: "cancelled", message: "console aborted" };
|
||||||
deps.cdp.trackSessionTab?.(ctxOrErr.sessionId, target.tabId);
|
deps.cdp.trackSessionTab?.(ctxOrErr.sessionId, target.tabId);
|
||||||
return deps.cdp.consoleEntriesSince(
|
return deps.cdp.consoleEntriesSince(
|
||||||
target.tabId,
|
target.tabId,
|
||||||
|
|||||||
@@ -8,6 +8,8 @@ import type {
|
|||||||
EvaluateParams,
|
EvaluateParams,
|
||||||
FillParams,
|
FillParams,
|
||||||
GetHtmlParams,
|
GetHtmlParams,
|
||||||
|
HoverParams,
|
||||||
|
HoverResult,
|
||||||
NavigateBackParams,
|
NavigateBackParams,
|
||||||
NavigateForwardParams,
|
NavigateForwardParams,
|
||||||
NavigateParams,
|
NavigateParams,
|
||||||
@@ -33,7 +35,7 @@ import { handleConsole } from "./console";
|
|||||||
import { type EmulateCdpRunner, handleEmulate } from "./emulate";
|
import { type EmulateCdpRunner, handleEmulate } from "./emulate";
|
||||||
import { handleEvaluate } from "./evaluate";
|
import { handleEvaluate } from "./evaluate";
|
||||||
import { handleRequestHelp } from "./human-loop";
|
import { handleRequestHelp } from "./human-loop";
|
||||||
import { handleClick, handleFill, handlePress, handleSelect } from "./interaction";
|
import { handleClick, handleFill, handleHover, handlePress, handleSelect } from "./interaction";
|
||||||
import {
|
import {
|
||||||
handleNavigate,
|
handleNavigate,
|
||||||
handleNavigateBack,
|
handleNavigateBack,
|
||||||
@@ -56,7 +58,7 @@ import {
|
|||||||
type SessionStartParams,
|
type SessionStartParams,
|
||||||
type SessionStopParams,
|
type SessionStopParams,
|
||||||
} from "./session";
|
} from "./session";
|
||||||
import { chromeTabsApi } from "./shared";
|
import { chromeTabsApi, lookupSession, resolveTargetTab } from "./shared";
|
||||||
import {
|
import {
|
||||||
type BorrowConfirmationApprover,
|
type BorrowConfirmationApprover,
|
||||||
handleTabBorrow,
|
handleTabBorrow,
|
||||||
@@ -81,6 +83,18 @@ type DispatcherCdpRunner = CdpRunner &
|
|||||||
detachSession(sessionId: string): Promise<void>;
|
detachSession(sessionId: string): Promise<void>;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
interface HoverLatch {
|
||||||
|
sessionId: string;
|
||||||
|
tabId: number;
|
||||||
|
x: number;
|
||||||
|
y: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface HoverLatchScope {
|
||||||
|
session_id: string;
|
||||||
|
tab_id?: number;
|
||||||
|
}
|
||||||
|
|
||||||
export interface DispatcherDeps {
|
export interface DispatcherDeps {
|
||||||
transport: Transport;
|
transport: Transport;
|
||||||
sessions: SessionManager;
|
sessions: SessionManager;
|
||||||
@@ -111,11 +125,10 @@ export interface DispatcherDeps {
|
|||||||
* `AbortController` keyed by its wire `id` in
|
* `AbortController` keyed by its wire `id` in
|
||||||
* [`inflightAbortControllers`]. When the daemon pushes a `cancel`
|
* [`inflightAbortControllers`]. When the daemon pushes a `cancel`
|
||||||
* request the dispatcher trips the matching controller; tool
|
* request the dispatcher trips the matching controller; tool
|
||||||
* handlers that already accept a `signal` (waits, navigation,
|
* handlers observe that signal between awaited operations. The
|
||||||
* interaction, evaluate, tabs) react in line, and the dispatcher
|
* original RPC remains pending until its handler has stopped or
|
||||||
* additionally races the in-flight invocation against the abort
|
* completed compensation; only the separate cancel acknowledgement
|
||||||
* promise so handlers without explicit signal plumbing still respond
|
* takes the fast path.
|
||||||
* promptly with `cancelled`.
|
|
||||||
*/
|
*/
|
||||||
export class ToolDispatcher {
|
export class ToolDispatcher {
|
||||||
private readonly transport: Transport;
|
private readonly transport: Transport;
|
||||||
@@ -126,6 +139,8 @@ export class ToolDispatcher {
|
|||||||
private readonly approveBorrow?: BorrowConfirmationApprover;
|
private readonly approveBorrow?: BorrowConfirmationApprover;
|
||||||
private readonly helpNotificationCopy?: () => { title: string; body: string };
|
private readonly helpNotificationCopy?: () => { title: string; body: string };
|
||||||
private subscription: { dispose(): void } | null = null;
|
private subscription: { dispose(): void } | null = null;
|
||||||
|
private readonly hoverBypassTabs = new Map<number, string>();
|
||||||
|
private readonly hoverLatches = new Map<number, HoverLatch>();
|
||||||
/**
|
/**
|
||||||
* Per-rpc-id `AbortController` registry. Populated inside
|
* Per-rpc-id `AbortController` registry. Populated inside
|
||||||
* [`dispatch`] before we await the tool handler and torn down in
|
* [`dispatch`] before we await the tool handler and torn down in
|
||||||
@@ -205,7 +220,7 @@ export class ToolDispatcher {
|
|||||||
try {
|
try {
|
||||||
const sessionId = sessionIdForBrowserControlMethod(req);
|
const sessionId = sessionIdForBrowserControlMethod(req);
|
||||||
if (sessionId) this.onBrowserControlResumed?.(sessionId);
|
if (sessionId) this.onBrowserControlResumed?.(sessionId);
|
||||||
const result = await Promise.race([this.invoke(req, ac.signal), abortPromise(ac.signal)]);
|
const result = await this.invoke(req, ac.signal);
|
||||||
if (isRpcError(result)) {
|
if (isRpcError(result)) {
|
||||||
body = { id: req.id, error: result };
|
body = { id: req.id, error: result };
|
||||||
} else {
|
} else {
|
||||||
@@ -270,33 +285,45 @@ export class ToolDispatcher {
|
|||||||
private async invoke(req: RequestFrame, signal: AbortSignal): Promise<unknown | RpcError> {
|
private async invoke(req: RequestFrame, signal: AbortSignal): Promise<unknown | RpcError> {
|
||||||
switch (req.method) {
|
switch (req.method) {
|
||||||
case "tool.session_start":
|
case "tool.session_start":
|
||||||
return handleSessionStart(this.sessions, req.params as SessionStartParams);
|
return handleSessionStart(this.sessions, req.params as SessionStartParams, { signal });
|
||||||
case "tool.session_stop":
|
case "tool.session_stop": {
|
||||||
|
await this.releaseHoverLatch((req.params as SessionStopParams).session_id);
|
||||||
return handleSessionStop(this.sessions, req.params as SessionStopParams, {
|
return handleSessionStop(this.sessions, req.params as SessionStopParams, {
|
||||||
cdp: this.cdp,
|
cdp: this.cdp,
|
||||||
|
signal,
|
||||||
});
|
});
|
||||||
|
}
|
||||||
case "tool.tab_list":
|
case "tool.tab_list":
|
||||||
return handleTabList(this.sessions, req.params as TabListParams);
|
return handleTabList(this.sessions, req.params as TabListParams, chromeTabsApi, signal);
|
||||||
case "tool.tab_create":
|
case "tool.tab_create":
|
||||||
return handleTabCreate(this.sessions, req.params as TabCreateParams);
|
return handleTabCreate(this.sessions, req.params as TabCreateParams, { signal });
|
||||||
case "tool.tab_close":
|
case "tool.tab_close":
|
||||||
return handleTabClose(this.sessions, req.params as TabCloseParams);
|
return this.withHoverReleaseForRequest(
|
||||||
|
req.params as TabCloseParams,
|
||||||
|
() => handleTabClose(this.sessions, req.params as TabCloseParams, { signal }),
|
||||||
|
signal,
|
||||||
|
);
|
||||||
case "tool.tab_select":
|
case "tool.tab_select":
|
||||||
return handleTabSelect(this.sessions, req.params as TabSelectParams);
|
return handleTabSelect(this.sessions, req.params as TabSelectParams, { signal });
|
||||||
case "tool.tab_borrow":
|
case "tool.tab_borrow":
|
||||||
return handleTabBorrow(this.sessions, req.params as TabBorrowParams, {
|
return handleTabBorrow(this.sessions, req.params as TabBorrowParams, {
|
||||||
signal,
|
signal,
|
||||||
approveBorrow: this.approveBorrow,
|
approveBorrow: this.approveBorrow,
|
||||||
});
|
});
|
||||||
case "tool.tab_return":
|
case "tool.tab_return":
|
||||||
return handleTabReturn(this.sessions, req.params as TabReturnParams);
|
return handleTabReturn(this.sessions, req.params as TabReturnParams, { signal });
|
||||||
case "tool.window_resize":
|
case "tool.window_resize":
|
||||||
return handleWindowResize(this.sessions, req.params as WindowResizeParams);
|
return handleWindowResize(
|
||||||
|
this.sessions,
|
||||||
|
req.params as WindowResizeParams,
|
||||||
|
undefined,
|
||||||
|
signal,
|
||||||
|
);
|
||||||
case "tool.emulate":
|
case "tool.emulate":
|
||||||
return handleEmulate(
|
return handleEmulate(
|
||||||
this.sessions,
|
this.sessions,
|
||||||
req.params as EmulateParams,
|
req.params as EmulateParams,
|
||||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi } : undefined,
|
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||||
);
|
);
|
||||||
case "tool.screenshot":
|
case "tool.screenshot":
|
||||||
return handleScreenshot(
|
return handleScreenshot(
|
||||||
@@ -305,100 +332,178 @@ export class ToolDispatcher {
|
|||||||
this.cdp
|
this.cdp
|
||||||
? { cdp: this.cdp, tabsApi: chromeTabsCaptureApi, captureApi: chromeTabsCaptureApi }
|
? { cdp: this.cdp, tabsApi: chromeTabsCaptureApi, captureApi: chromeTabsCaptureApi }
|
||||||
: undefined,
|
: undefined,
|
||||||
|
signal,
|
||||||
);
|
);
|
||||||
case "tool.console":
|
case "tool.console":
|
||||||
return handleConsole(
|
return handleConsole(
|
||||||
this.sessions,
|
this.sessions,
|
||||||
req.params as ConsoleParams,
|
req.params as ConsoleParams,
|
||||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi } : undefined,
|
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi } : undefined,
|
||||||
|
signal,
|
||||||
);
|
);
|
||||||
case "tool.network":
|
case "tool.network":
|
||||||
return handleNetwork(
|
return handleNetwork(
|
||||||
this.sessions,
|
this.sessions,
|
||||||
req.params as NetworkParams,
|
req.params as NetworkParams,
|
||||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi } : undefined,
|
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi } : undefined,
|
||||||
|
signal,
|
||||||
);
|
);
|
||||||
case "tool.snapshot":
|
case "tool.snapshot":
|
||||||
return handleSnapshot(
|
return this.withHoverReassert(
|
||||||
this.sessions,
|
|
||||||
req.params as SnapshotParams,
|
req.params as SnapshotParams,
|
||||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsCaptureApi } : undefined,
|
() =>
|
||||||
|
handleSnapshot(
|
||||||
|
this.sessions,
|
||||||
|
req.params as SnapshotParams,
|
||||||
|
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsCaptureApi } : undefined,
|
||||||
|
signal,
|
||||||
|
),
|
||||||
|
{},
|
||||||
|
signal,
|
||||||
);
|
);
|
||||||
case "tool.observe":
|
case "tool.observe": {
|
||||||
return handleObserve(
|
const params = req.params as ObserveParams;
|
||||||
this.sessions,
|
const hoverScope = await this.resolveHoverLatchScope(params);
|
||||||
req.params as ObserveParams,
|
throwIfDispatchAborted(signal);
|
||||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsCaptureApi } : undefined,
|
return this.withHoverReassert(
|
||||||
|
params,
|
||||||
|
() =>
|
||||||
|
handleObserve(
|
||||||
|
this.sessions,
|
||||||
|
params,
|
||||||
|
this.cdp
|
||||||
|
? {
|
||||||
|
cdp: this.cdp,
|
||||||
|
tabsApi: chromeTabsCaptureApi,
|
||||||
|
conditionalSurfaceProbe: !this.hasHoverLatchForScope(hoverScope),
|
||||||
|
hoverProbeBypassOverlay: bypassOverlay,
|
||||||
|
}
|
||||||
|
: undefined,
|
||||||
|
signal,
|
||||||
|
),
|
||||||
|
{},
|
||||||
|
signal,
|
||||||
);
|
);
|
||||||
|
}
|
||||||
case "tool.get_html":
|
case "tool.get_html":
|
||||||
return handleGetHtml(
|
return handleGetHtml(
|
||||||
this.sessions,
|
this.sessions,
|
||||||
req.params as GetHtmlParams,
|
req.params as GetHtmlParams,
|
||||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsCaptureApi } : undefined,
|
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsCaptureApi } : undefined,
|
||||||
|
signal,
|
||||||
);
|
);
|
||||||
case "tool.navigate":
|
case "tool.navigate":
|
||||||
return handleNavigate(
|
return this.withHoverReleaseForRequest(
|
||||||
this.sessions,
|
|
||||||
req.params as NavigateParams,
|
req.params as NavigateParams,
|
||||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
() =>
|
||||||
|
handleNavigate(
|
||||||
|
this.sessions,
|
||||||
|
req.params as NavigateParams,
|
||||||
|
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||||
|
),
|
||||||
|
signal,
|
||||||
);
|
);
|
||||||
case "tool.navigate_back":
|
case "tool.navigate_back":
|
||||||
return handleNavigateBack(
|
return this.withHoverReleaseForRequest(
|
||||||
this.sessions,
|
|
||||||
req.params as NavigateBackParams,
|
req.params as NavigateBackParams,
|
||||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
() =>
|
||||||
|
handleNavigateBack(
|
||||||
|
this.sessions,
|
||||||
|
req.params as NavigateBackParams,
|
||||||
|
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||||
|
),
|
||||||
|
signal,
|
||||||
);
|
);
|
||||||
case "tool.navigate_forward":
|
case "tool.navigate_forward":
|
||||||
return handleNavigateForward(
|
return this.withHoverReleaseForRequest(
|
||||||
this.sessions,
|
|
||||||
req.params as NavigateForwardParams,
|
req.params as NavigateForwardParams,
|
||||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
() =>
|
||||||
|
handleNavigateForward(
|
||||||
|
this.sessions,
|
||||||
|
req.params as NavigateForwardParams,
|
||||||
|
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||||
|
),
|
||||||
|
signal,
|
||||||
);
|
);
|
||||||
case "tool.reload":
|
case "tool.reload":
|
||||||
return handleReload(
|
return this.withHoverReleaseForRequest(
|
||||||
this.sessions,
|
|
||||||
req.params as ReloadParams,
|
req.params as ReloadParams,
|
||||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
() =>
|
||||||
|
handleReload(
|
||||||
|
this.sessions,
|
||||||
|
req.params as ReloadParams,
|
||||||
|
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||||
|
),
|
||||||
|
signal,
|
||||||
);
|
);
|
||||||
case "tool.click":
|
case "tool.click":
|
||||||
return handleClick(
|
return this.withHoverReassert(
|
||||||
this.sessions,
|
|
||||||
req.params as ClickParams,
|
req.params as ClickParams,
|
||||||
|
() =>
|
||||||
|
handleClick(
|
||||||
|
this.sessions,
|
||||||
|
req.params as ClickParams,
|
||||||
|
this.cdp
|
||||||
|
? {
|
||||||
|
cdp: this.cdp,
|
||||||
|
tabsApi: chromeTabsApi,
|
||||||
|
signal,
|
||||||
|
bypassOverlay,
|
||||||
|
}
|
||||||
|
: undefined,
|
||||||
|
),
|
||||||
|
{ releaseAfter: true },
|
||||||
|
signal,
|
||||||
|
);
|
||||||
|
case "tool.hover": {
|
||||||
|
const result = await handleHover(
|
||||||
|
this.sessions,
|
||||||
|
req.params as HoverParams,
|
||||||
this.cdp
|
this.cdp
|
||||||
? {
|
? {
|
||||||
cdp: this.cdp,
|
cdp: this.cdp,
|
||||||
tabsApi: chromeTabsApi,
|
tabsApi: chromeTabsApi,
|
||||||
signal,
|
signal,
|
||||||
bypassOverlay: async (tabId, enabled) => {
|
bypassOverlay: (tabId, enabled) =>
|
||||||
try {
|
this.setHoverBypass((req.params as HoverParams).session_id, tabId, enabled),
|
||||||
await chrome.tabs.sendMessage(tabId, {
|
keepOverlayBypassAfterHover: true,
|
||||||
type: OVERLAY_AUTOMATION_BYPASS,
|
|
||||||
enabled,
|
|
||||||
});
|
|
||||||
} catch {
|
|
||||||
// Content script may be unavailable on restricted pages.
|
|
||||||
}
|
|
||||||
},
|
|
||||||
}
|
}
|
||||||
: undefined,
|
: undefined,
|
||||||
);
|
);
|
||||||
|
return this.rememberHover((req.params as HoverParams).session_id, result);
|
||||||
|
}
|
||||||
case "tool.fill":
|
case "tool.fill":
|
||||||
return handleFill(
|
return this.withHoverReleaseForRequest(
|
||||||
this.sessions,
|
|
||||||
req.params as FillParams,
|
req.params as FillParams,
|
||||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
() =>
|
||||||
|
handleFill(
|
||||||
|
this.sessions,
|
||||||
|
req.params as FillParams,
|
||||||
|
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||||
|
),
|
||||||
|
signal,
|
||||||
);
|
);
|
||||||
case "tool.press":
|
case "tool.press":
|
||||||
return handlePress(
|
return this.withHoverReleaseForRequest(
|
||||||
this.sessions,
|
|
||||||
req.params as PressParams,
|
req.params as PressParams,
|
||||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
() =>
|
||||||
|
handlePress(
|
||||||
|
this.sessions,
|
||||||
|
req.params as PressParams,
|
||||||
|
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||||
|
),
|
||||||
|
signal,
|
||||||
);
|
);
|
||||||
case "tool.select":
|
case "tool.select":
|
||||||
return handleSelect(
|
return this.withHoverReleaseForRequest(
|
||||||
this.sessions,
|
|
||||||
req.params as SelectParams,
|
req.params as SelectParams,
|
||||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
() =>
|
||||||
|
handleSelect(
|
||||||
|
this.sessions,
|
||||||
|
req.params as SelectParams,
|
||||||
|
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||||
|
),
|
||||||
|
signal,
|
||||||
);
|
);
|
||||||
case "tool.evaluate":
|
case "tool.evaluate":
|
||||||
return handleEvaluate(
|
return handleEvaluate(
|
||||||
@@ -471,6 +576,122 @@ export class ToolDispatcher {
|
|||||||
} satisfies RpcError;
|
} satisfies RpcError;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private async setHoverBypass(sessionId: string, tabId: number, enabled: boolean): Promise<void> {
|
||||||
|
const owner = this.hoverBypassTabs.get(tabId);
|
||||||
|
if (enabled) {
|
||||||
|
if (owner === sessionId) return;
|
||||||
|
if (owner === undefined) await bypassOverlay(tabId, true);
|
||||||
|
this.hoverBypassTabs.set(tabId, sessionId);
|
||||||
|
} else {
|
||||||
|
if (owner !== sessionId) return;
|
||||||
|
await bypassOverlay(tabId, false);
|
||||||
|
this.hoverBypassTabs.delete(tabId);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private rememberHover(sessionId: string, result: HoverResult | RpcError): HoverResult | RpcError {
|
||||||
|
if (!isRpcError(result)) {
|
||||||
|
this.hoverLatches.set(result.tab_id, {
|
||||||
|
sessionId,
|
||||||
|
tabId: result.tab_id,
|
||||||
|
x: result.x,
|
||||||
|
y: result.y,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
private hasHoverLatchForScope(scope: HoverLatchScope): boolean {
|
||||||
|
return this.hoverLatchesForRequest(scope).length > 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
private async withHoverReassert<T>(
|
||||||
|
params: { session_id: string; tab_id?: number },
|
||||||
|
work: () => Promise<T>,
|
||||||
|
options: { releaseAfter?: boolean } = {},
|
||||||
|
signal?: AbortSignal,
|
||||||
|
): Promise<T> {
|
||||||
|
throwIfDispatchAborted(signal);
|
||||||
|
const scope = await this.resolveHoverLatchScope(params);
|
||||||
|
throwIfDispatchAborted(signal);
|
||||||
|
await this.reassertHover(scope);
|
||||||
|
throwIfDispatchAborted(signal);
|
||||||
|
try {
|
||||||
|
return await work();
|
||||||
|
} finally {
|
||||||
|
if (options.releaseAfter) {
|
||||||
|
await this.releaseHoverLatch(scope.session_id, scope.tab_id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async withHoverReleaseForRequest<T>(
|
||||||
|
params: { session_id: string; tab_id?: number },
|
||||||
|
work: () => Promise<T>,
|
||||||
|
signal?: AbortSignal,
|
||||||
|
): Promise<T> {
|
||||||
|
throwIfDispatchAborted(signal);
|
||||||
|
const scope = await this.resolveHoverLatchScope(params);
|
||||||
|
throwIfDispatchAborted(signal);
|
||||||
|
await this.releaseHoverLatch(scope.session_id, scope.tab_id);
|
||||||
|
throwIfDispatchAborted(signal);
|
||||||
|
return work();
|
||||||
|
}
|
||||||
|
|
||||||
|
private async resolveHoverLatchScope(params: {
|
||||||
|
session_id: string;
|
||||||
|
tab_id?: number;
|
||||||
|
}): Promise<HoverLatchScope> {
|
||||||
|
if (params.tab_id !== undefined) return params;
|
||||||
|
const ctx = lookupSession(this.sessions, params, "hover latch");
|
||||||
|
if (isRpcError(ctx)) return params;
|
||||||
|
const target = await resolveTargetTab(this.sessions, ctx, undefined, chromeTabsApi);
|
||||||
|
if (isRpcError(target)) return params;
|
||||||
|
return { session_id: params.session_id, tab_id: target.tabId };
|
||||||
|
}
|
||||||
|
|
||||||
|
private hoverLatchesForRequest(params: { session_id: string; tab_id?: number }): HoverLatch[] {
|
||||||
|
return [...this.hoverLatches.values()].filter((latch) => {
|
||||||
|
if (latch.sessionId !== params.session_id) return false;
|
||||||
|
return params.tab_id === undefined || latch.tabId === params.tab_id;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
private async reassertHover(params: { session_id: string; tab_id?: number }): Promise<void> {
|
||||||
|
if (!this.cdp) return;
|
||||||
|
await Promise.all(
|
||||||
|
this.hoverLatchesForRequest(params).map((latch) =>
|
||||||
|
this.cdp!.send(latch.tabId, "Input.dispatchMouseEvent", {
|
||||||
|
type: "mouseMoved",
|
||||||
|
x: latch.x,
|
||||||
|
y: latch.y,
|
||||||
|
}).catch((err) => {
|
||||||
|
console.debug("[bsk dispatcher] hover reassert failed", err);
|
||||||
|
this.hoverLatches.delete(latch.tabId);
|
||||||
|
}),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async releaseHoverLatch(sessionId?: string, tabId?: number): Promise<void> {
|
||||||
|
const matchesScope = (entrySessionId: string, entryTabId: number): boolean => {
|
||||||
|
if (sessionId !== undefined && entrySessionId !== sessionId) return false;
|
||||||
|
return tabId === undefined || entryTabId === tabId;
|
||||||
|
};
|
||||||
|
const tabs = new Set<number>();
|
||||||
|
for (const [bypassTabId, bypassSessionId] of this.hoverBypassTabs) {
|
||||||
|
if (!matchesScope(bypassSessionId, bypassTabId)) continue;
|
||||||
|
tabs.add(bypassTabId);
|
||||||
|
this.hoverBypassTabs.delete(bypassTabId);
|
||||||
|
}
|
||||||
|
for (const latch of this.hoverLatches.values()) {
|
||||||
|
if (!matchesScope(latch.sessionId, latch.tabId)) continue;
|
||||||
|
tabs.add(latch.tabId);
|
||||||
|
this.hoverLatches.delete(latch.tabId);
|
||||||
|
}
|
||||||
|
await Promise.all([...tabs].map((tabId) => bypassOverlay(tabId, false)));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
function isRpcError(v: unknown): v is RpcError {
|
function isRpcError(v: unknown): v is RpcError {
|
||||||
@@ -497,6 +718,7 @@ function sessionIdForBrowserControlMethod(req: RequestFrame): string | null {
|
|||||||
case "tool.navigate_forward":
|
case "tool.navigate_forward":
|
||||||
case "tool.reload":
|
case "tool.reload":
|
||||||
case "tool.click":
|
case "tool.click":
|
||||||
|
case "tool.hover":
|
||||||
case "tool.fill":
|
case "tool.fill":
|
||||||
case "tool.press":
|
case "tool.press":
|
||||||
case "tool.select":
|
case "tool.select":
|
||||||
@@ -512,41 +734,25 @@ function sessionIdForBrowserControlMethod(req: RequestFrame): string | null {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
async function bypassOverlay(tabId: number, enabled: boolean): Promise<void> {
|
||||||
* Resolves never; rejects with `AbortLikeError` as soon as the signal
|
try {
|
||||||
* fires (or immediately if it is already aborted). Used by the
|
await chrome.tabs.sendMessage(tabId, {
|
||||||
* dispatcher to race the tool invocation so handlers without explicit
|
type: OVERLAY_AUTOMATION_BYPASS,
|
||||||
* signal plumbing still surface a `cancelled` reply promptly.
|
enabled,
|
||||||
*/
|
});
|
||||||
function abortPromise(signal: AbortSignal): Promise<never> {
|
} catch {
|
||||||
return new Promise<never>((_, reject) => {
|
// Content script may be unavailable on restricted pages.
|
||||||
if (signal.aborted) {
|
|
||||||
reject(new AbortLikeError());
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
signal.addEventListener(
|
|
||||||
"abort",
|
|
||||||
() => {
|
|
||||||
reject(new AbortLikeError());
|
|
||||||
},
|
|
||||||
{ once: true },
|
|
||||||
);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Sentinel error class so [`isAbortLikeError`] can recognise our own
|
|
||||||
* race-rejection without confusing it with a real CDP failure.
|
|
||||||
*/
|
|
||||||
class AbortLikeError extends Error {
|
|
||||||
constructor() {
|
|
||||||
super("rpc aborted by daemon cancel");
|
|
||||||
this.name = "BhAbortError";
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function throwIfDispatchAborted(signal: AbortSignal | undefined): void {
|
||||||
|
if (!signal?.aborted) return;
|
||||||
|
const error = new Error("rpc aborted by daemon cancel");
|
||||||
|
error.name = "AbortError";
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
|
||||||
function isAbortLikeError(err: unknown): boolean {
|
function isAbortLikeError(err: unknown): boolean {
|
||||||
if (err instanceof AbortLikeError) return true;
|
|
||||||
if (err instanceof DOMException && err.name === "AbortError") return true;
|
if (err instanceof DOMException && err.name === "AbortError") return true;
|
||||||
if (typeof err === "object" && err !== null && (err as { name?: string }).name === "AbortError") {
|
if (typeof err === "object" && err !== null && (err as { name?: string }).name === "AbortError") {
|
||||||
return true;
|
return true;
|
||||||
|
|||||||
@@ -1,9 +1,15 @@
|
|||||||
// Shared CDP helpers for scrolling nodes into view and computing
|
// Resolve and scroll nodes inside one CDP target. Cross-frame projection
|
||||||
// viewport-space bounding rectangles. Used by interaction tools (click
|
// belongs to frame-geometry.ts.
|
||||||
// centre) and observation tools (element screenshot clip).
|
|
||||||
|
|
||||||
import type { RpcError } from "@/transport/types";
|
import type { RpcError } from "@/transport/types";
|
||||||
import { rpcError } from "./errors";
|
import { rpcError } from "./errors";
|
||||||
|
import {
|
||||||
|
parseCdpQuad,
|
||||||
|
polygonBounds,
|
||||||
|
type Region,
|
||||||
|
rectPolygon,
|
||||||
|
type ViewportRect,
|
||||||
|
} from "./geometry";
|
||||||
import { type CdpRunner, isRpcError } from "./shared";
|
import { type CdpRunner, isRpcError } from "./shared";
|
||||||
|
|
||||||
const ELEMENT_NOT_VISIBLE_MESSAGE =
|
const ELEMENT_NOT_VISIBLE_MESSAGE =
|
||||||
@@ -13,45 +19,6 @@ function elementNotVisibleError(): RpcError {
|
|||||||
return rpcError("permission_denied", "element_not_visible", ELEMENT_NOT_VISIBLE_MESSAGE);
|
return rpcError("permission_denied", "element_not_visible", ELEMENT_NOT_VISIBLE_MESSAGE);
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Viewport-space axis-aligned bounding box for CDP `Page.captureScreenshot` clip. */
|
|
||||||
export interface ViewportRect {
|
|
||||||
x: number;
|
|
||||||
y: number;
|
|
||||||
width: number;
|
|
||||||
height: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Compute the axis-aligned bounding box of an 8-double CDP content
|
|
||||||
* quad / box-model polygon. Exported for unit tests.
|
|
||||||
*/
|
|
||||||
export function quadBoundingRect(quad: number[]): ViewportRect | null {
|
|
||||||
if (quad.length !== 8) return null;
|
|
||||||
const xs = [quad[0], quad[2], quad[4], quad[6]];
|
|
||||||
const ys = [quad[1], quad[3], quad[5], quad[7]];
|
|
||||||
const minX = Math.min(...xs);
|
|
||||||
const maxX = Math.max(...xs);
|
|
||||||
const minY = Math.min(...ys);
|
|
||||||
const maxY = Math.max(...ys);
|
|
||||||
const width = maxX - minX;
|
|
||||||
const height = maxY - minY;
|
|
||||||
if (width <= 0 || height <= 0) return null;
|
|
||||||
return { x: minX, y: minY, width, height };
|
|
||||||
}
|
|
||||||
|
|
||||||
function rectToQuad(rect: ViewportRect): number[] {
|
|
||||||
return [
|
|
||||||
rect.x,
|
|
||||||
rect.y,
|
|
||||||
rect.x + rect.width,
|
|
||||||
rect.y,
|
|
||||||
rect.x + rect.width,
|
|
||||||
rect.y + rect.height,
|
|
||||||
rect.x,
|
|
||||||
rect.y + rect.height,
|
|
||||||
];
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Resolve `backendNodeId` → CDP `objectId` so we can invoke
|
* Resolve `backendNodeId` → CDP `objectId` so we can invoke
|
||||||
* `Runtime.callFunctionOn` against the live JS object.
|
* `Runtime.callFunctionOn` against the live JS object.
|
||||||
@@ -112,48 +79,24 @@ export async function scrollNodeIntoView(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
function validCdpRegion(quads: number[][] | undefined): Region {
|
||||||
* Viewport-space bounding box for `backendNodeId`. Prefers
|
return (quads ?? []).flatMap((raw) => {
|
||||||
* `DOM.getContentQuads`, then `DOM.getBoxModel`, then the union of
|
const polygon = parseCdpQuad(raw);
|
||||||
* visible descendant client rects for zero-size overflow containers.
|
return polygon && polygonBounds(polygon) ? [polygon] : [];
|
||||||
*/
|
});
|
||||||
export async function nodeBoundingRect(
|
}
|
||||||
|
|
||||||
|
async function resolveVisibleContentRegion(
|
||||||
cdp: CdpRunner,
|
cdp: CdpRunner,
|
||||||
tabId: number,
|
tabId: number,
|
||||||
backendNodeId: number,
|
backendNodeId: number,
|
||||||
): Promise<ViewportRect | RpcError> {
|
): Promise<Region | RpcError> {
|
||||||
const polygon = await visibleContentPolygon(cdp, tabId, backendNodeId);
|
|
||||||
if (isRpcError(polygon)) return polygon;
|
|
||||||
// visibleContentPolygon only returns quads with positive area.
|
|
||||||
return quadBoundingRect(polygon)!;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Compute the centroid of a CDP content quad. Quads are reported as
|
|
||||||
* 8 doubles in clockwise order: (x1, y1, x2, y2, x3, y3, x4, y4).
|
|
||||||
*/
|
|
||||||
export function quadCentre(quad: number[]): { x: number; y: number } {
|
|
||||||
const x = (quad[0] + quad[4]) / 2;
|
|
||||||
const y = (quad[1] + quad[5]) / 2;
|
|
||||||
return { x, y };
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Centre of a `DOM.getBoxModel` `content` polygon (also 8 doubles). */
|
|
||||||
export function boxCentre(box: number[]): { x: number; y: number } {
|
|
||||||
return quadCentre(box);
|
|
||||||
}
|
|
||||||
|
|
||||||
async function visibleContentPolygon(
|
|
||||||
cdp: CdpRunner,
|
|
||||||
tabId: number,
|
|
||||||
backendNodeId: number,
|
|
||||||
): Promise<number[] | RpcError> {
|
|
||||||
try {
|
try {
|
||||||
const quads = await cdp.send<{ quads?: number[][] }>(tabId, "DOM.getContentQuads", {
|
const quads = await cdp.send<{ quads?: number[][] }>(tabId, "DOM.getContentQuads", {
|
||||||
backendNodeId,
|
backendNodeId,
|
||||||
});
|
});
|
||||||
const quad = quads.quads?.find((q) => q.length === 8 && quadBoundingRect(q) !== null);
|
const visible = validCdpRegion(quads.quads);
|
||||||
if (quad) return quad;
|
if (visible.length > 0) return visible;
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
console.debug("[bsk element-geometry] getContentQuads failed", err);
|
console.debug("[bsk element-geometry] getContentQuads failed", err);
|
||||||
}
|
}
|
||||||
@@ -162,15 +105,34 @@ async function visibleContentPolygon(
|
|||||||
backendNodeId,
|
backendNodeId,
|
||||||
});
|
});
|
||||||
const content = box.model?.content;
|
const content = box.model?.content;
|
||||||
if (content && content.length === 8 && quadBoundingRect(content)) return content;
|
const fallback = validCdpRegion(content ? [content] : undefined);
|
||||||
|
if (fallback.length > 0) return fallback;
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
console.debug("[bsk element-geometry] getBoxModel failed", err);
|
console.debug("[bsk element-geometry] getBoxModel failed", err);
|
||||||
}
|
}
|
||||||
const descendantRect = await descendantBoundingRect(cdp, tabId, backendNodeId);
|
const descendantRect = await descendantBoundingRect(cdp, tabId, backendNodeId);
|
||||||
if (descendantRect) return rectToQuad(descendantRect);
|
if (descendantRect) {
|
||||||
|
return [
|
||||||
|
rectPolygon({
|
||||||
|
x: descendantRect.x,
|
||||||
|
y: descendantRect.y,
|
||||||
|
w: descendantRect.width,
|
||||||
|
h: descendantRect.height,
|
||||||
|
}),
|
||||||
|
];
|
||||||
|
}
|
||||||
return elementNotVisibleError();
|
return elementNotVisibleError();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Resolve every visible content region in the node's target-local viewport. */
|
||||||
|
export async function nodeContentRegion(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
tabId: number,
|
||||||
|
backendNodeId: number,
|
||||||
|
): Promise<Region | RpcError> {
|
||||||
|
return resolveVisibleContentRegion(cdp, tabId, backendNodeId);
|
||||||
|
}
|
||||||
|
|
||||||
async function descendantBoundingRect(
|
async function descendantBoundingRect(
|
||||||
cdp: CdpRunner,
|
cdp: CdpRunner,
|
||||||
tabId: number,
|
tabId: number,
|
||||||
@@ -241,20 +203,3 @@ function parseViewportRect(value: unknown): ViewportRect | null {
|
|||||||
if (![x, y, width, height].every(Number.isFinite) || width <= 0 || height <= 0) return null;
|
if (![x, y, width, height].every(Number.isFinite) || width <= 0 || height <= 0) return null;
|
||||||
return { x, y, width, height };
|
return { x, y, width, height };
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Get the click point for `backendNodeId`. Prefers
|
|
||||||
* `DOM.getContentQuads` (richer for rotated / transformed elements);
|
|
||||||
* falls back to `DOM.getBoxModel`, then visible descendant bounds.
|
|
||||||
* Returns `permission_denied` with "element not visible" when every
|
|
||||||
* geometry path comes back empty / fails.
|
|
||||||
*/
|
|
||||||
export async function nodeCentre(
|
|
||||||
cdp: CdpRunner,
|
|
||||||
tabId: number,
|
|
||||||
backendNodeId: number,
|
|
||||||
): Promise<{ x: number; y: number } | RpcError> {
|
|
||||||
const polygon = await visibleContentPolygon(cdp, tabId, backendNodeId);
|
|
||||||
if (isRpcError(polygon)) return polygon;
|
|
||||||
return quadCentre(polygon);
|
|
||||||
}
|
|
||||||
@@ -44,6 +44,7 @@ export interface EmulateCdpRunner {
|
|||||||
export interface EmulateDeps {
|
export interface EmulateDeps {
|
||||||
cdp: EmulateCdpRunner;
|
cdp: EmulateCdpRunner;
|
||||||
tabsApi: ChromeTabsApi;
|
tabsApi: ChromeTabsApi;
|
||||||
|
signal?: AbortSignal;
|
||||||
}
|
}
|
||||||
|
|
||||||
function defaultEmulateDeps(): EmulateDeps {
|
function defaultEmulateDeps(): EmulateDeps {
|
||||||
@@ -210,11 +211,13 @@ export async function handleEmulate(
|
|||||||
params: EmulateParams,
|
params: EmulateParams,
|
||||||
deps: EmulateDeps = defaultEmulateDeps(),
|
deps: EmulateDeps = defaultEmulateDeps(),
|
||||||
): Promise<EmulateResult | RpcError> {
|
): Promise<EmulateResult | RpcError> {
|
||||||
|
if (deps.signal?.aborted) return { code: "cancelled", message: "emulate aborted" };
|
||||||
const ctxOrErr = lookupSession(manager, params, "emulate");
|
const ctxOrErr = lookupSession(manager, params, "emulate");
|
||||||
if (isRpcError(ctxOrErr)) return ctxOrErr;
|
if (isRpcError(ctxOrErr)) return ctxOrErr;
|
||||||
const ctx = ctxOrErr;
|
const ctx = ctxOrErr;
|
||||||
const target = await resolveTargetTab(manager, ctx, params?.tab_id, deps.tabsApi);
|
const target = await resolveTargetTab(manager, ctx, params?.tab_id, deps.tabsApi);
|
||||||
if (isRpcError(target)) return target;
|
if (isRpcError(target)) return target;
|
||||||
|
if (deps.signal?.aborted) return { code: "cancelled", message: "emulate aborted" };
|
||||||
const denied = enforceAgentWindow(ctx, target, "emulate");
|
const denied = enforceAgentWindow(ctx, target, "emulate");
|
||||||
if (denied) return denied;
|
if (denied) return denied;
|
||||||
|
|
||||||
@@ -223,9 +226,16 @@ export async function handleEmulate(
|
|||||||
return invalidParams("emulate off cannot be combined with overrides");
|
return invalidParams("emulate off cannot be combined with overrides");
|
||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
|
if (deps.signal?.aborted) return { code: "cancelled", message: "emulate aborted" };
|
||||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||||
await deps.cdp.clearDeviceMetricsOverride(target.tabId);
|
await deps.cdp.clearDeviceMetricsOverride(target.tabId);
|
||||||
|
if (deps.signal?.aborted) {
|
||||||
|
return { code: "cancelled", message: "emulate aborted after clearing device metrics" };
|
||||||
|
}
|
||||||
await deps.cdp.setTouchEmulationEnabled(target.tabId, false);
|
await deps.cdp.setTouchEmulationEnabled(target.tabId, false);
|
||||||
|
if (deps.signal?.aborted) {
|
||||||
|
return { code: "cancelled", message: "emulate aborted after clearing touch emulation" };
|
||||||
|
}
|
||||||
// An empty userAgent string clears the override (CDP convention).
|
// An empty userAgent string clears the override (CDP convention).
|
||||||
await deps.cdp.setUserAgentOverride(target.tabId, { userAgent: "" });
|
await deps.cdp.setUserAgentOverride(target.tabId, { userAgent: "" });
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
@@ -246,6 +256,7 @@ export async function handleEmulate(
|
|||||||
// values; the merged state is what gets applied (and echoed back).
|
// values; the merged state is what gets applied (and echoed back).
|
||||||
const merged = mergeEmulateOverrides(tabEmulationStates.get(target.tabId), overrides);
|
const merged = mergeEmulateOverrides(tabEmulationStates.get(target.tabId), overrides);
|
||||||
try {
|
try {
|
||||||
|
if (deps.signal?.aborted) return { code: "cancelled", message: "emulate aborted" };
|
||||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||||
if (merged.width !== undefined && merged.height !== undefined) {
|
if (merged.width !== undefined && merged.height !== undefined) {
|
||||||
await deps.cdp.setDeviceMetricsOverride(target.tabId, {
|
await deps.cdp.setDeviceMetricsOverride(target.tabId, {
|
||||||
@@ -254,6 +265,9 @@ export async function handleEmulate(
|
|||||||
deviceScaleFactor: merged.device_scale_factor ?? 0,
|
deviceScaleFactor: merged.device_scale_factor ?? 0,
|
||||||
mobile: merged.mobile ?? false,
|
mobile: merged.mobile ?? false,
|
||||||
});
|
});
|
||||||
|
if (deps.signal?.aborted) {
|
||||||
|
return { code: "cancelled", message: "emulate aborted after applying device metrics" };
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if (merged.user_agent !== undefined) {
|
if (merged.user_agent !== undefined) {
|
||||||
await deps.cdp.setUserAgentOverride(target.tabId, {
|
await deps.cdp.setUserAgentOverride(target.tabId, {
|
||||||
@@ -263,6 +277,9 @@ export async function handleEmulate(
|
|||||||
? { userAgentMetadata: toCdpUserAgentMetadata(merged.user_agent_metadata) }
|
? { userAgentMetadata: toCdpUserAgentMetadata(merged.user_agent_metadata) }
|
||||||
: {}),
|
: {}),
|
||||||
});
|
});
|
||||||
|
if (deps.signal?.aborted) {
|
||||||
|
return { code: "cancelled", message: "emulate aborted after applying user agent" };
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if (merged.touch !== undefined || merged.max_touch_points !== undefined) {
|
if (merged.touch !== undefined || merged.max_touch_points !== undefined) {
|
||||||
await deps.cdp.setTouchEmulationEnabled(
|
await deps.cdp.setTouchEmulationEnabled(
|
||||||
|
|||||||
@@ -0,0 +1,290 @@
|
|||||||
|
import { type CdpFrame, type CdpFrameGraph, type CdpTarget } from "@/browser-driver/frame-graph";
|
||||||
|
import type { RpcError } from "@/transport/types";
|
||||||
|
import { nodeContentRegion, scrollNodeIntoView } from "./element-geometry";
|
||||||
|
import {
|
||||||
|
clipPolygon,
|
||||||
|
type GeometryProjection,
|
||||||
|
type Point,
|
||||||
|
type Polygon,
|
||||||
|
type ProjectiveEdge,
|
||||||
|
parseCdpQuad,
|
||||||
|
polygonArea,
|
||||||
|
polygonCentroid,
|
||||||
|
projectRegionToViewport,
|
||||||
|
type Quad,
|
||||||
|
type Region,
|
||||||
|
rectPolygon,
|
||||||
|
regionBounds,
|
||||||
|
type Size,
|
||||||
|
type ViewportRect,
|
||||||
|
} from "./geometry";
|
||||||
|
import { type CdpRunner, cdpRunnerForTarget, isRpcError, sendToCdpTarget } from "./shared";
|
||||||
|
|
||||||
|
export interface NodeAddress {
|
||||||
|
target: CdpTarget;
|
||||||
|
backendNodeId: number;
|
||||||
|
frameId?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ResolvedNodeGeometry {
|
||||||
|
topVisibleRegions: Region;
|
||||||
|
topBounds: ViewportRect;
|
||||||
|
actionPoint: Point;
|
||||||
|
}
|
||||||
|
|
||||||
|
function geometryError(message: string): RpcError {
|
||||||
|
return { code: "cdp_failed", message };
|
||||||
|
}
|
||||||
|
|
||||||
|
function frameMap(graph: CdpFrameGraph): Map<string, CdpFrame> {
|
||||||
|
return new Map(graph.frames.map((frame) => [frame.frameId, frame]));
|
||||||
|
}
|
||||||
|
|
||||||
|
function targetRootFrame(byId: Map<string, CdpFrame>, frame: CdpFrame): CdpFrame | null {
|
||||||
|
const seen = new Set<string>();
|
||||||
|
let current = frame;
|
||||||
|
while (current.parentFrameId) {
|
||||||
|
if (seen.has(current.frameId)) return null;
|
||||||
|
seen.add(current.frameId);
|
||||||
|
const parent = byId.get(current.parentFrameId);
|
||||||
|
if (!parent || parent.target.sessionId !== current.target.sessionId) break;
|
||||||
|
current = parent;
|
||||||
|
}
|
||||||
|
return current;
|
||||||
|
}
|
||||||
|
|
||||||
|
function frameAncestry(graph: CdpFrameGraph, frameId: string): CdpFrame[] | null {
|
||||||
|
const byId = frameMap(graph);
|
||||||
|
const path: CdpFrame[] = [];
|
||||||
|
const seen = new Set<string>();
|
||||||
|
let current = byId.get(frameId);
|
||||||
|
if (!current) return null;
|
||||||
|
while (current.parentFrameId) {
|
||||||
|
if (seen.has(current.frameId)) return null;
|
||||||
|
seen.add(current.frameId);
|
||||||
|
path.push(current);
|
||||||
|
const parent = byId.get(current.parentFrameId);
|
||||||
|
if (!parent) return null;
|
||||||
|
current = parent;
|
||||||
|
}
|
||||||
|
return path;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function targetViewport(cdp: CdpRunner, target: CdpTarget): Promise<Size | null> {
|
||||||
|
const metrics = await sendToCdpTarget<{
|
||||||
|
cssLayoutViewport?: { clientWidth?: number; clientHeight?: number };
|
||||||
|
layoutViewport?: { clientWidth?: number; clientHeight?: number };
|
||||||
|
}>(cdp, target, "Page.getLayoutMetrics", {});
|
||||||
|
const viewport = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
|
||||||
|
const width = viewport.clientWidth ?? 0;
|
||||||
|
const height = viewport.clientHeight ?? 0;
|
||||||
|
return width > 0 && height > 0 ? { width, height } : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function ownerContentQuad(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
parent: CdpFrame,
|
||||||
|
ownerBackendNodeId: number,
|
||||||
|
): Promise<Quad | null> {
|
||||||
|
const result = await sendToCdpTarget<{ model?: { content?: number[] } }>(
|
||||||
|
cdp,
|
||||||
|
parent.target,
|
||||||
|
"DOM.getBoxModel",
|
||||||
|
{ backendNodeId: ownerBackendNodeId },
|
||||||
|
);
|
||||||
|
return parseCdpQuad(result.model?.content);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function sameTargetFrameClips(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
byId: Map<string, CdpFrame>,
|
||||||
|
frame: CdpFrame,
|
||||||
|
targetRoot: CdpFrame,
|
||||||
|
): Promise<Polygon[] | null> {
|
||||||
|
const clips: Polygon[] = [];
|
||||||
|
let current = frame;
|
||||||
|
const seen = new Set<string>();
|
||||||
|
while (current.frameId !== targetRoot.frameId) {
|
||||||
|
if (seen.has(current.frameId) || !current.parentFrameId) return null;
|
||||||
|
seen.add(current.frameId);
|
||||||
|
const parent = byId.get(current.parentFrameId);
|
||||||
|
if (!parent || current.ownerBackendNodeId === undefined) return null;
|
||||||
|
const clip = await ownerContentQuad(cdp, parent, current.ownerBackendNodeId);
|
||||||
|
if (!clip) return null;
|
||||||
|
clips.push(clip);
|
||||||
|
current = parent;
|
||||||
|
}
|
||||||
|
return clips;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function resolveFrameProjection(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
graph: CdpFrameGraph,
|
||||||
|
frameId: string,
|
||||||
|
): Promise<GeometryProjection | null> {
|
||||||
|
const byId = frameMap(graph);
|
||||||
|
const frame = byId.get(frameId);
|
||||||
|
if (!frame) return null;
|
||||||
|
let targetRoot = targetRootFrame(byId, frame);
|
||||||
|
if (!targetRoot) return null;
|
||||||
|
const sourceViewport = await targetViewport(cdp, targetRoot.target);
|
||||||
|
if (!sourceViewport) return null;
|
||||||
|
const sourceClips = await sameTargetFrameClips(cdp, byId, frame, targetRoot);
|
||||||
|
if (!sourceClips) return null;
|
||||||
|
|
||||||
|
const edges: ProjectiveEdge[] = [];
|
||||||
|
while (targetRoot.parentFrameId) {
|
||||||
|
const parent = byId.get(targetRoot.parentFrameId);
|
||||||
|
if (!parent || targetRoot.ownerBackendNodeId === undefined) return null;
|
||||||
|
const destinationQuad = await ownerContentQuad(cdp, parent, targetRoot.ownerBackendNodeId);
|
||||||
|
if (!destinationQuad) return null;
|
||||||
|
const source =
|
||||||
|
edges.length === 0 ? sourceViewport : await targetViewport(cdp, targetRoot.target);
|
||||||
|
if (!source) return null;
|
||||||
|
const parentTargetRoot = targetRootFrame(byId, parent);
|
||||||
|
if (!parentTargetRoot) return null;
|
||||||
|
const destinationClips = await sameTargetFrameClips(cdp, byId, parent, parentTargetRoot);
|
||||||
|
if (!destinationClips) return null;
|
||||||
|
edges.push({ sourceViewport: source, destinationQuad, destinationClips });
|
||||||
|
targetRoot = parentTargetRoot;
|
||||||
|
}
|
||||||
|
|
||||||
|
const topViewport =
|
||||||
|
edges.length === 0 ? sourceViewport : await targetViewport(cdp, targetRoot.target);
|
||||||
|
return topViewport ? { sourceClips, edges, topViewport } : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function loadFrameGraph(cdp: CdpRunner, tabId: number): Promise<CdpFrameGraph | null> {
|
||||||
|
if (!cdp.getFrameGraph) return null;
|
||||||
|
try {
|
||||||
|
return await cdp.getFrameGraph(tabId);
|
||||||
|
} catch (error) {
|
||||||
|
console.debug("[bsk frame-geometry] frame graph resolution failed", error);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function scrollFrameOwners(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
tabId: number,
|
||||||
|
graph: CdpFrameGraph,
|
||||||
|
frameId: string,
|
||||||
|
): Promise<RpcError | null> {
|
||||||
|
const byId = frameMap(graph);
|
||||||
|
const ancestry = frameAncestry(graph, frameId);
|
||||||
|
if (!ancestry) return geometryError(`could not resolve frame ancestry for ${frameId}`);
|
||||||
|
for (const child of [...ancestry].reverse()) {
|
||||||
|
const parent = child.parentFrameId ? byId.get(child.parentFrameId) : undefined;
|
||||||
|
if (!parent || child.ownerBackendNodeId === undefined) {
|
||||||
|
return geometryError(`could not resolve frame owner for ${child.frameId}`);
|
||||||
|
}
|
||||||
|
const error = await scrollNodeIntoView(
|
||||||
|
cdpRunnerForTarget(cdp, parent.target),
|
||||||
|
tabId,
|
||||||
|
child.ownerBackendNodeId,
|
||||||
|
);
|
||||||
|
if (error) return error;
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function scrollElementWithFrameGraph(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
tabId: number,
|
||||||
|
target: CdpTarget,
|
||||||
|
backendNodeId: number,
|
||||||
|
frameId: string | undefined,
|
||||||
|
graph: CdpFrameGraph | null,
|
||||||
|
): Promise<RpcError | null> {
|
||||||
|
if (frameId) {
|
||||||
|
if (!graph) return geometryError(`could not resolve frame graph for ${frameId}`);
|
||||||
|
const error = await scrollFrameOwners(cdp, tabId, graph, frameId);
|
||||||
|
if (error) return error;
|
||||||
|
} else if (target.sessionId) {
|
||||||
|
return geometryError("an OOPIF node address requires frameId");
|
||||||
|
}
|
||||||
|
return scrollNodeIntoView(cdpRunnerForTarget(cdp, target), tabId, backendNodeId);
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function scrollElementAndFramesIntoView(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
tabId: number,
|
||||||
|
target: CdpTarget,
|
||||||
|
backendNodeId: number,
|
||||||
|
frameId?: string,
|
||||||
|
): Promise<RpcError | null> {
|
||||||
|
const graph = frameId ? await loadFrameGraph(cdp, tabId) : null;
|
||||||
|
return scrollElementWithFrameGraph(cdp, tabId, target, backendNodeId, frameId, graph);
|
||||||
|
}
|
||||||
|
|
||||||
|
function largestRegion(regions: Region): Polygon | null {
|
||||||
|
let largest: { polygon: Polygon; area: number } | null = null;
|
||||||
|
for (const polygon of regions) {
|
||||||
|
const area = polygonArea(polygon);
|
||||||
|
if (area <= 0) continue;
|
||||||
|
if (!largest || area > largest.area) largest = { polygon, area };
|
||||||
|
}
|
||||||
|
return largest?.polygon ?? null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function resolveNodeGeometry(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
tabId: number,
|
||||||
|
address: NodeAddress,
|
||||||
|
options: { scrollIntoView?: boolean } = {},
|
||||||
|
): Promise<ResolvedNodeGeometry | RpcError> {
|
||||||
|
try {
|
||||||
|
if (address.target.sessionId && !address.frameId) {
|
||||||
|
return geometryError("an OOPIF node address requires frameId");
|
||||||
|
}
|
||||||
|
const graph = address.frameId ? await loadFrameGraph(cdp, tabId) : null;
|
||||||
|
if (address.frameId && !graph) {
|
||||||
|
return geometryError(`could not resolve frame graph for ${address.frameId}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (options.scrollIntoView) {
|
||||||
|
const scrollError = await scrollElementWithFrameGraph(
|
||||||
|
cdp,
|
||||||
|
tabId,
|
||||||
|
address.target,
|
||||||
|
address.backendNodeId,
|
||||||
|
address.frameId,
|
||||||
|
graph,
|
||||||
|
);
|
||||||
|
if (scrollError) return scrollError;
|
||||||
|
}
|
||||||
|
|
||||||
|
const localRegion = await nodeContentRegion(
|
||||||
|
cdpRunnerForTarget(cdp, address.target),
|
||||||
|
tabId,
|
||||||
|
address.backendNodeId,
|
||||||
|
);
|
||||||
|
if (isRpcError(localRegion)) return localRegion;
|
||||||
|
|
||||||
|
let topVisibleRegions: Region;
|
||||||
|
if (address.frameId && graph) {
|
||||||
|
const projection = await resolveFrameProjection(cdp, graph, address.frameId);
|
||||||
|
if (!projection)
|
||||||
|
return geometryError(`could not resolve frame geometry for ${address.frameId}`);
|
||||||
|
topVisibleRegions = projectRegionToViewport(localRegion, projection);
|
||||||
|
} else {
|
||||||
|
const viewport = await targetViewport(cdp, address.target);
|
||||||
|
if (!viewport) return geometryError("could not resolve top viewport geometry");
|
||||||
|
topVisibleRegions = localRegion
|
||||||
|
.map((polygon) =>
|
||||||
|
clipPolygon(polygon, rectPolygon({ x: 0, y: 0, w: viewport.width, h: viewport.height })),
|
||||||
|
)
|
||||||
|
.filter((polygon) => polygon.length >= 3);
|
||||||
|
}
|
||||||
|
|
||||||
|
const topBounds = regionBounds(topVisibleRegions);
|
||||||
|
const actionRegion = largestRegion(topVisibleRegions);
|
||||||
|
const actionPoint = actionRegion ? polygonCentroid(actionRegion) : null;
|
||||||
|
if (!topBounds || !actionPoint) {
|
||||||
|
return { code: "permission_denied", message: "element not visible" };
|
||||||
|
}
|
||||||
|
return { topVisibleRegions, topBounds, actionPoint };
|
||||||
|
} catch (error) {
|
||||||
|
return geometryError(error instanceof Error ? error.message : String(error));
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,266 @@
|
|||||||
|
export interface Point {
|
||||||
|
x: number;
|
||||||
|
y: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export type Polygon = Point[];
|
||||||
|
export type Region = Polygon[];
|
||||||
|
export type Quad = [Point, Point, Point, Point];
|
||||||
|
|
||||||
|
export interface Size {
|
||||||
|
width: number;
|
||||||
|
height: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ViewportRect {
|
||||||
|
x: number;
|
||||||
|
y: number;
|
||||||
|
width: number;
|
||||||
|
height: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ProjectiveEdge {
|
||||||
|
sourceViewport: Size;
|
||||||
|
destinationQuad: Quad;
|
||||||
|
destinationClips?: Polygon[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface GeometryProjection {
|
||||||
|
sourceClips: Polygon[];
|
||||||
|
edges: ProjectiveEdge[];
|
||||||
|
topViewport: Size;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function parseCdpQuad(raw: number[] | undefined): Quad | null {
|
||||||
|
if (!raw || raw.length !== 8 || !raw.every(Number.isFinite)) return null;
|
||||||
|
return [
|
||||||
|
{ x: raw[0], y: raw[1] },
|
||||||
|
{ x: raw[2], y: raw[3] },
|
||||||
|
{ x: raw[4], y: raw[5] },
|
||||||
|
{ x: raw[6], y: raw[7] },
|
||||||
|
];
|
||||||
|
}
|
||||||
|
|
||||||
|
export function rectPolygon(rect: { x: number; y: number; w: number; h: number }): Quad {
|
||||||
|
return [
|
||||||
|
{ x: rect.x, y: rect.y },
|
||||||
|
{ x: rect.x + rect.w, y: rect.y },
|
||||||
|
{ x: rect.x + rect.w, y: rect.y + rect.h },
|
||||||
|
{ x: rect.x, y: rect.y + rect.h },
|
||||||
|
];
|
||||||
|
}
|
||||||
|
|
||||||
|
export function viewportPolygon(viewport: Size): Quad {
|
||||||
|
return rectPolygon({ x: 0, y: 0, w: viewport.width, h: viewport.height });
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Project a normalized unit-square point into an arbitrary content quad. */
|
||||||
|
export function projectUnitPoint(point: Point, quad: Quad): Point {
|
||||||
|
const [p0, p1, p2, p3] = quad;
|
||||||
|
const dx1 = p1.x - p2.x;
|
||||||
|
const dx2 = p3.x - p2.x;
|
||||||
|
const dx3 = p0.x - p1.x + p2.x - p3.x;
|
||||||
|
const dy1 = p1.y - p2.y;
|
||||||
|
const dy2 = p3.y - p2.y;
|
||||||
|
const dy3 = p0.y - p1.y + p2.y - p3.y;
|
||||||
|
|
||||||
|
let a: number;
|
||||||
|
let b: number;
|
||||||
|
let c: number;
|
||||||
|
let d: number;
|
||||||
|
let e: number;
|
||||||
|
let f: number;
|
||||||
|
let g = 0;
|
||||||
|
let h = 0;
|
||||||
|
|
||||||
|
if (Math.abs(dx3) < 1e-9 && Math.abs(dy3) < 1e-9) {
|
||||||
|
a = p1.x - p0.x;
|
||||||
|
b = p3.x - p0.x;
|
||||||
|
c = p0.x;
|
||||||
|
d = p1.y - p0.y;
|
||||||
|
e = p3.y - p0.y;
|
||||||
|
f = p0.y;
|
||||||
|
} else {
|
||||||
|
const denominator = dx1 * dy2 - dx2 * dy1;
|
||||||
|
if (Math.abs(denominator) < 1e-9) {
|
||||||
|
return {
|
||||||
|
x: p0.x + (p1.x - p0.x) * point.x + (p3.x - p0.x) * point.y,
|
||||||
|
y: p0.y + (p1.y - p0.y) * point.x + (p3.y - p0.y) * point.y,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
g = (dx3 * dy2 - dx2 * dy3) / denominator;
|
||||||
|
h = (dx1 * dy3 - dx3 * dy1) / denominator;
|
||||||
|
a = p1.x - p0.x + g * p1.x;
|
||||||
|
b = p3.x - p0.x + h * p3.x;
|
||||||
|
c = p0.x;
|
||||||
|
d = p1.y - p0.y + g * p1.y;
|
||||||
|
e = p3.y - p0.y + h * p3.y;
|
||||||
|
f = p0.y;
|
||||||
|
}
|
||||||
|
|
||||||
|
const scale = g * point.x + h * point.y + 1;
|
||||||
|
return {
|
||||||
|
x: (a * point.x + b * point.y + c) / scale,
|
||||||
|
y: (d * point.x + e * point.y + f) / scale,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function projectPolygon(polygon: Polygon, edge: ProjectiveEdge): Polygon {
|
||||||
|
const { width, height } = edge.sourceViewport;
|
||||||
|
if (width <= 0 || height <= 0) return [];
|
||||||
|
return polygon.map((point) =>
|
||||||
|
projectUnitPoint({ x: point.x / width, y: point.y / height }, edge.destinationQuad),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function polygonSignedArea(points: Polygon): number {
|
||||||
|
let area = 0;
|
||||||
|
for (let index = 0; index < points.length; index += 1) {
|
||||||
|
const current = points[index];
|
||||||
|
const next = points[(index + 1) % points.length];
|
||||||
|
area += current.x * next.y - next.x * current.y;
|
||||||
|
}
|
||||||
|
return area / 2;
|
||||||
|
}
|
||||||
|
|
||||||
|
function edgeIntersection(start: Point, end: Point, clipStart: Point, clipEnd: Point): Point {
|
||||||
|
const subjectX = end.x - start.x;
|
||||||
|
const subjectY = end.y - start.y;
|
||||||
|
const clipX = clipEnd.x - clipStart.x;
|
||||||
|
const clipY = clipEnd.y - clipStart.y;
|
||||||
|
const denominator = subjectX * clipY - subjectY * clipX;
|
||||||
|
if (Math.abs(denominator) < 1e-9) return end;
|
||||||
|
const offsetX = clipStart.x - start.x;
|
||||||
|
const offsetY = clipStart.y - start.y;
|
||||||
|
const scale = (offsetX * clipY - offsetY * clipX) / denominator;
|
||||||
|
return { x: start.x + scale * subjectX, y: start.y + scale * subjectY };
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Clip a polygon to a convex polygon, preserving every resulting vertex. */
|
||||||
|
export function clipPolygon(subject: Polygon, clip: Polygon): Polygon {
|
||||||
|
if (subject.length < 3 || clip.length < 3) return [];
|
||||||
|
const orientation = polygonSignedArea(clip) >= 0 ? 1 : -1;
|
||||||
|
let output = [...subject];
|
||||||
|
for (let edge = 0; edge < clip.length && output.length > 0; edge += 1) {
|
||||||
|
const clipStart = clip[edge];
|
||||||
|
const clipEnd = clip[(edge + 1) % clip.length];
|
||||||
|
const input = output;
|
||||||
|
output = [];
|
||||||
|
const inside = (point: Point) =>
|
||||||
|
orientation *
|
||||||
|
((clipEnd.x - clipStart.x) * (point.y - clipStart.y) -
|
||||||
|
(clipEnd.y - clipStart.y) * (point.x - clipStart.x)) >=
|
||||||
|
-1e-9;
|
||||||
|
let start = input[input.length - 1];
|
||||||
|
for (const end of input) {
|
||||||
|
const startInside = inside(start);
|
||||||
|
const endInside = inside(end);
|
||||||
|
if (endInside) {
|
||||||
|
if (!startInside) output.push(edgeIntersection(start, end, clipStart, clipEnd));
|
||||||
|
output.push(end);
|
||||||
|
} else if (startInside) {
|
||||||
|
output.push(edgeIntersection(start, end, clipStart, clipEnd));
|
||||||
|
}
|
||||||
|
start = end;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return output;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function polygonBounds(points: Polygon): ViewportRect | null {
|
||||||
|
if (points.length < 3) return null;
|
||||||
|
const xs = points.map((point) => point.x);
|
||||||
|
const ys = points.map((point) => point.y);
|
||||||
|
const x = Math.min(...xs);
|
||||||
|
const y = Math.min(...ys);
|
||||||
|
const right = Math.max(...xs);
|
||||||
|
const bottom = Math.max(...ys);
|
||||||
|
if (right <= x || bottom <= y) return null;
|
||||||
|
return { x, y, width: right - x, height: bottom - y };
|
||||||
|
}
|
||||||
|
|
||||||
|
export function regionBounds(region: Region): ViewportRect | null {
|
||||||
|
return polygonBounds(region.flat());
|
||||||
|
}
|
||||||
|
|
||||||
|
export function polygonArea(points: Polygon): number {
|
||||||
|
return Math.abs(polygonSignedArea(points));
|
||||||
|
}
|
||||||
|
|
||||||
|
export function polygonCentroid(points: Polygon): Point | null {
|
||||||
|
if (points.length < 3) return null;
|
||||||
|
let twiceArea = 0;
|
||||||
|
let x = 0;
|
||||||
|
let y = 0;
|
||||||
|
for (let index = 0; index < points.length; index += 1) {
|
||||||
|
const current = points[index];
|
||||||
|
const next = points[(index + 1) % points.length];
|
||||||
|
const cross = current.x * next.y - next.x * current.y;
|
||||||
|
twiceArea += cross;
|
||||||
|
x += (current.x + next.x) * cross;
|
||||||
|
y += (current.y + next.y) * cross;
|
||||||
|
}
|
||||||
|
if (Math.abs(twiceArea) < 1e-9) return null;
|
||||||
|
return { x: x / (3 * twiceArea), y: y / (3 * twiceArea) };
|
||||||
|
}
|
||||||
|
|
||||||
|
export function projectAndClipRegion(
|
||||||
|
region: Region,
|
||||||
|
edges: ProjectiveEdge[],
|
||||||
|
topViewport: Size,
|
||||||
|
): Region {
|
||||||
|
let projected = region;
|
||||||
|
for (const edge of edges) {
|
||||||
|
const sourceClip = viewportPolygon(edge.sourceViewport);
|
||||||
|
projected = projected
|
||||||
|
.map((polygon) => clipPolygon(polygon, sourceClip))
|
||||||
|
.filter((polygon) => polygon.length >= 3)
|
||||||
|
.map((polygon) =>
|
||||||
|
[edge.destinationQuad, ...(edge.destinationClips ?? [])].reduce(
|
||||||
|
(current, clip) => clipPolygon(current, clip),
|
||||||
|
projectPolygon(polygon, edge),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.filter((polygon) => polygon.length >= 3);
|
||||||
|
}
|
||||||
|
const topClip = viewportPolygon(topViewport);
|
||||||
|
return projected
|
||||||
|
.map((polygon) => clipPolygon(polygon, topClip))
|
||||||
|
.filter((polygon) => polygon.length >= 3);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function projectRegionToViewport(region: Region, projection: GeometryProjection): Region {
|
||||||
|
const clipped = region
|
||||||
|
.map((polygon) =>
|
||||||
|
projection.sourceClips.reduce((current, clip) => clipPolygon(current, clip), polygon),
|
||||||
|
)
|
||||||
|
.filter((polygon) => polygon.length >= 3);
|
||||||
|
return projectAndClipRegion(clipped, projection.edges, projection.topViewport);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function projectRectToViewport(
|
||||||
|
rect: { x: number; y: number; w: number; h: number } | null,
|
||||||
|
projection: GeometryProjection,
|
||||||
|
): ViewportRect | null {
|
||||||
|
if (!rect) return null;
|
||||||
|
return regionBounds(projectRegionToViewport([rectPolygon(rect)], projection));
|
||||||
|
}
|
||||||
|
|
||||||
|
export function childFrameProjection(
|
||||||
|
parent: GeometryProjection,
|
||||||
|
ownerRectInParent: { x: number; y: number; w: number; h: number },
|
||||||
|
): GeometryProjection {
|
||||||
|
const destinationQuad = rectPolygon(ownerRectInParent);
|
||||||
|
return {
|
||||||
|
sourceClips: [],
|
||||||
|
edges: [
|
||||||
|
{
|
||||||
|
sourceViewport: { width: ownerRectInParent.w, height: ownerRectInParent.h },
|
||||||
|
destinationQuad,
|
||||||
|
destinationClips: parent.sourceClips,
|
||||||
|
},
|
||||||
|
...parent.edges,
|
||||||
|
],
|
||||||
|
topViewport: parent.topViewport,
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -8,6 +8,7 @@ import {
|
|||||||
HELP_REQUEST,
|
HELP_REQUEST,
|
||||||
type HelpCancelMessage,
|
type HelpCancelMessage,
|
||||||
type HelpFinishMessage,
|
type HelpFinishMessage,
|
||||||
|
type HelpHighlightRect,
|
||||||
type HelpQueryResponse,
|
type HelpQueryResponse,
|
||||||
type HelpRequestMessage,
|
type HelpRequestMessage,
|
||||||
isHelpAckMessage,
|
isHelpAckMessage,
|
||||||
@@ -25,6 +26,7 @@ import type {
|
|||||||
ResolvedTarget,
|
ResolvedTarget,
|
||||||
RpcError,
|
RpcError,
|
||||||
} from "@/transport/types";
|
} from "@/transport/types";
|
||||||
|
import { resolveNodeGeometry } from "./frame-geometry";
|
||||||
import {
|
import {
|
||||||
type CdpRunner,
|
type CdpRunner,
|
||||||
type ChromeTabsApi,
|
type ChromeTabsApi,
|
||||||
@@ -37,7 +39,6 @@ import {
|
|||||||
import { lookupSnapshotRef } from "./snapshot-ref";
|
import { lookupSnapshotRef } from "./snapshot-ref";
|
||||||
|
|
||||||
const DEFAULT_HELP_TIMEOUT_MS = 300_000;
|
const DEFAULT_HELP_TIMEOUT_MS = 300_000;
|
||||||
const HELP_ATTR = "data-bsk-help";
|
|
||||||
const HELP_SEND_RETRIES = 3;
|
const HELP_SEND_RETRIES = 3;
|
||||||
const HELP_SEND_RETRY_DELAY_MS = 350;
|
const HELP_SEND_RETRY_DELAY_MS = 350;
|
||||||
const HELP_REARM_DEBOUNCE_MS = 150;
|
const HELP_REARM_DEBOUNCE_MS = 150;
|
||||||
@@ -74,6 +75,7 @@ interface ActiveHelpRequest {
|
|||||||
title?: string;
|
title?: string;
|
||||||
targets: HelpTarget[];
|
targets: HelpTarget[];
|
||||||
selectors: string[];
|
selectors: string[];
|
||||||
|
rects: HelpHighlightRect[];
|
||||||
timeoutMs: number;
|
timeoutMs: number;
|
||||||
notificationId: string;
|
notificationId: string;
|
||||||
resolvedTargets?: ResolvedTarget[];
|
resolvedTargets?: ResolvedTarget[];
|
||||||
@@ -134,6 +136,7 @@ function helpRequestMessage(help: ActiveHelpRequest, tabId: number): HelpRequest
|
|||||||
...(help.title ? { title: help.title } : {}),
|
...(help.title ? { title: help.title } : {}),
|
||||||
displayMode: isPrimaryTab ? "full" : "compact",
|
displayMode: isPrimaryTab ? "full" : "compact",
|
||||||
selectors: isPrimaryTab ? help.selectors : [],
|
selectors: isPrimaryTab ? help.selectors : [],
|
||||||
|
rects: isPrimaryTab ? help.rects : [],
|
||||||
timeoutMs: help.timeoutMs,
|
timeoutMs: help.timeoutMs,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
@@ -165,33 +168,6 @@ async function sendHelpRequestWithAck(
|
|||||||
throw lastError ?? new Error("failed to show help overlay");
|
throw lastError ?? new Error("failed to show help overlay");
|
||||||
}
|
}
|
||||||
|
|
||||||
async function tagRefTarget(
|
|
||||||
cdp: CdpRunner,
|
|
||||||
tabId: number,
|
|
||||||
backendNodeId: number,
|
|
||||||
index: number,
|
|
||||||
): Promise<string | null> {
|
|
||||||
let objectId: string | undefined;
|
|
||||||
try {
|
|
||||||
const resolved = await cdp.send<{ object?: { objectId?: string } }>(tabId, "DOM.resolveNode", {
|
|
||||||
backendNodeId,
|
|
||||||
});
|
|
||||||
objectId = resolved.object?.objectId;
|
|
||||||
if (!objectId) return null;
|
|
||||||
await cdp.send(tabId, "Runtime.callFunctionOn", {
|
|
||||||
objectId,
|
|
||||||
functionDeclaration: `function(){ this.setAttribute("${HELP_ATTR}", "${index}"); }`,
|
|
||||||
});
|
|
||||||
return `[${HELP_ATTR}="${index}"]`;
|
|
||||||
} catch {
|
|
||||||
return null;
|
|
||||||
} finally {
|
|
||||||
if (objectId) {
|
|
||||||
await cdp.send(tabId, "Runtime.releaseObject", { objectId }).catch(() => {});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async function selectorExists(
|
async function selectorExists(
|
||||||
cdp: CdpRunner,
|
cdp: CdpRunner,
|
||||||
tabId: number,
|
tabId: number,
|
||||||
@@ -209,36 +185,22 @@ async function selectorExists(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async function clearRefTags(cdp: CdpRunner | undefined, tabId: number): Promise<void> {
|
|
||||||
if (!cdp) return;
|
|
||||||
try {
|
|
||||||
const doc = await cdp.send<{ root?: { nodeId?: number } }>(tabId, "DOM.getDocument", {
|
|
||||||
depth: 0,
|
|
||||||
});
|
|
||||||
const rootId = doc.root?.nodeId;
|
|
||||||
if (rootId === undefined) return;
|
|
||||||
const { nodeIds } = await cdp.send<{ nodeIds: number[] }>(tabId, "DOM.querySelectorAll", {
|
|
||||||
nodeId: rootId,
|
|
||||||
selector: `[${HELP_ATTR}]`,
|
|
||||||
});
|
|
||||||
for (const nodeId of nodeIds ?? []) {
|
|
||||||
await cdp.send(tabId, "DOM.removeAttribute", { nodeId, name: HELP_ATTR }).catch(() => {});
|
|
||||||
}
|
|
||||||
} catch {
|
|
||||||
// Best-effort cleanup.
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async function resolveHelpTargets(
|
async function resolveHelpTargets(
|
||||||
ctx: ReturnType<SessionManager["get"]>,
|
ctx: ReturnType<SessionManager["get"]>,
|
||||||
tabId: number,
|
tabId: number,
|
||||||
targets: HelpTarget[],
|
targets: HelpTarget[],
|
||||||
deps: RequestHelpDeps,
|
deps: RequestHelpDeps,
|
||||||
): Promise<{ selectors: string[]; resolvedTargets?: ResolvedTarget[] }> {
|
options: { scrollIntoView: boolean },
|
||||||
|
): Promise<{
|
||||||
|
selectors: string[];
|
||||||
|
rects: HelpHighlightRect[];
|
||||||
|
resolvedTargets?: ResolvedTarget[];
|
||||||
|
}> {
|
||||||
const selectors: string[] = [];
|
const selectors: string[] = [];
|
||||||
|
const rects: HelpHighlightRect[] = [];
|
||||||
const resolved: ResolvedTarget[] = [];
|
const resolved: ResolvedTarget[] = [];
|
||||||
let rootNodeId: number | undefined;
|
let rootNodeId: number | undefined;
|
||||||
if (deps.cdp && targets.length > 0) {
|
if (deps.cdp && targets.some((target) => Boolean(target.selector))) {
|
||||||
try {
|
try {
|
||||||
const doc = await deps.cdp.send<{ root?: { nodeId?: number } }>(tabId, "DOM.getDocument", {
|
const doc = await deps.cdp.send<{ root?: { nodeId?: number } }>(tabId, "DOM.getDocument", {
|
||||||
depth: 0,
|
depth: 0,
|
||||||
@@ -262,28 +224,55 @@ async function resolveHelpTargets(
|
|||||||
resolved.push({ matched, selector: tgt.selector });
|
resolved.push({ matched, selector: tgt.selector });
|
||||||
} else if (tgt.ref) {
|
} else if (tgt.ref) {
|
||||||
const looked = ctx ? lookupSnapshotRef(ctx, tgt.ref, tabId) : null;
|
const looked = ctx ? lookupSnapshotRef(ctx, tgt.ref, tabId) : null;
|
||||||
const backendNodeId = looked?.backendNodeId ?? null;
|
let rect: HelpHighlightRect | null = null;
|
||||||
let sel: string | null = null;
|
if (looked && deps.cdp) {
|
||||||
if (backendNodeId !== null && deps.cdp) {
|
const target = {
|
||||||
sel = await tagRefTarget(deps.cdp, tabId, backendNodeId, i);
|
tabId,
|
||||||
|
...(looked.cdpSessionId ? { sessionId: looked.cdpSessionId } : {}),
|
||||||
|
};
|
||||||
|
const geometry = await resolveNodeGeometry(
|
||||||
|
deps.cdp,
|
||||||
|
tabId,
|
||||||
|
{
|
||||||
|
target,
|
||||||
|
backendNodeId: looked.backendNodeId,
|
||||||
|
...(looked.frameId ? { frameId: looked.frameId } : {}),
|
||||||
|
},
|
||||||
|
{ scrollIntoView: options.scrollIntoView },
|
||||||
|
);
|
||||||
|
if (!isRpcError(geometry)) {
|
||||||
|
rect = {
|
||||||
|
top: geometry.topBounds.y,
|
||||||
|
left: geometry.topBounds.x,
|
||||||
|
width: geometry.topBounds.width,
|
||||||
|
height: geometry.topBounds.height,
|
||||||
|
};
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if (sel) selectors.push(sel);
|
if (rect) rects.push(rect);
|
||||||
resolved.push({ matched: sel !== null, ref: tgt.ref });
|
resolved.push({
|
||||||
|
matched: rect !== null,
|
||||||
|
ref: tgt.ref,
|
||||||
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return { selectors, resolvedTargets: resolved.length > 0 ? resolved : undefined };
|
return { selectors, rects, resolvedTargets: resolved.length > 0 ? resolved : undefined };
|
||||||
}
|
}
|
||||||
|
|
||||||
async function refreshHelpTargets(help: ActiveHelpRequest): Promise<void> {
|
async function refreshHelpTargets(
|
||||||
await clearRefTags(help.deps.cdp, help.primaryTabId);
|
help: ActiveHelpRequest,
|
||||||
const { selectors, resolvedTargets } = await resolveHelpTargets(
|
options: { scrollIntoView: boolean } = { scrollIntoView: true },
|
||||||
|
): Promise<void> {
|
||||||
|
const { selectors, rects, resolvedTargets } = await resolveHelpTargets(
|
||||||
help.ctx,
|
help.ctx,
|
||||||
help.primaryTabId,
|
help.primaryTabId,
|
||||||
help.targets,
|
help.targets,
|
||||||
help.deps,
|
help.deps,
|
||||||
|
options,
|
||||||
);
|
);
|
||||||
help.selectors = selectors;
|
help.selectors = selectors;
|
||||||
|
help.rects = rects;
|
||||||
help.resolvedTargets = resolvedTargets;
|
help.resolvedTargets = resolvedTargets;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -294,12 +283,7 @@ async function cleanupHelp(help: ActiveHelpRequest): Promise<void> {
|
|||||||
const tabsToCancel = new Set([help.primaryTabId, ...help.overlayTabIds]);
|
const tabsToCancel = new Set([help.primaryTabId, ...help.overlayTabIds]);
|
||||||
await Promise.all(
|
await Promise.all(
|
||||||
[...tabsToCancel].map((tabId) =>
|
[...tabsToCancel].map((tabId) =>
|
||||||
Promise.all([
|
help.deps.sendToTab(tabId, { type: HELP_CANCEL, requestId: help.requestId }).catch(() => {}),
|
||||||
clearRefTags(help.deps.cdp, tabId),
|
|
||||||
help.deps
|
|
||||||
.sendToTab(tabId, { type: HELP_CANCEL, requestId: help.requestId })
|
|
||||||
.catch(() => {}),
|
|
||||||
]),
|
|
||||||
),
|
),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@@ -476,7 +460,9 @@ function attachHelpRuntimeListener(deps: RequestHelpDeps): () => void {
|
|||||||
sendResponse({ active: false });
|
sendResponse({ active: false });
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if (tabId === help.primaryTabId) await refreshHelpTargets(help);
|
if (tabId === help.primaryTabId) {
|
||||||
|
await refreshHelpTargets(help, { scrollIntoView: false });
|
||||||
|
}
|
||||||
help.overlayTabIds.add(tabId);
|
help.overlayTabIds.add(tabId);
|
||||||
sendResponse({
|
sendResponse({
|
||||||
active: true,
|
active: true,
|
||||||
@@ -486,6 +472,7 @@ function attachHelpRuntimeListener(deps: RequestHelpDeps): () => void {
|
|||||||
...(help.title ? { title: help.title } : {}),
|
...(help.title ? { title: help.title } : {}),
|
||||||
displayMode: tabId === help.primaryTabId ? "full" : "compact",
|
displayMode: tabId === help.primaryTabId ? "full" : "compact",
|
||||||
selectors: tabId === help.primaryTabId ? help.selectors : [],
|
selectors: tabId === help.primaryTabId ? help.selectors : [],
|
||||||
|
rects: tabId === help.primaryTabId ? help.rects : [],
|
||||||
timeoutMs: help.timeoutMs,
|
timeoutMs: help.timeoutMs,
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
@@ -686,7 +673,13 @@ export async function handleRequestHelp(
|
|||||||
if (deps.autoAttachLifecycle !== false) ensureHelpLifecycleListeners(deps);
|
if (deps.autoAttachLifecycle !== false) ensureHelpLifecycleListeners(deps);
|
||||||
const tabId = target.tabId;
|
const tabId = target.tabId;
|
||||||
const initialTargets = params.targets ?? [];
|
const initialTargets = params.targets ?? [];
|
||||||
const { selectors, resolvedTargets } = await resolveHelpTargets(ctx, tabId, initialTargets, deps);
|
const { selectors, rects, resolvedTargets } = await resolveHelpTargets(
|
||||||
|
ctx,
|
||||||
|
tabId,
|
||||||
|
initialTargets,
|
||||||
|
deps,
|
||||||
|
{ scrollIntoView: true },
|
||||||
|
);
|
||||||
|
|
||||||
await deps.windows.update(target.windowId, { focused: true }).catch(() => {});
|
await deps.windows.update(target.windowId, { focused: true }).catch(() => {});
|
||||||
await deps.activateTab(tabId).catch(() => {});
|
await deps.activateTab(tabId).catch(() => {});
|
||||||
@@ -729,6 +722,7 @@ export async function handleRequestHelp(
|
|||||||
...(params.title ? { title: params.title } : {}),
|
...(params.title ? { title: params.title } : {}),
|
||||||
targets: initialTargets,
|
targets: initialTargets,
|
||||||
selectors,
|
selectors,
|
||||||
|
rects,
|
||||||
timeoutMs,
|
timeoutMs,
|
||||||
notificationId,
|
notificationId,
|
||||||
resolvedTargets,
|
resolvedTargets,
|
||||||
|
|||||||
@@ -3,20 +3,23 @@
|
|||||||
//
|
//
|
||||||
// All interaction tools:
|
// All interaction tools:
|
||||||
// 1. Resolve target tab (sandbox: must be inside Agent Window).
|
// 1. Resolve target tab (sandbox: must be inside Agent Window).
|
||||||
// 2. Resolve target element by `ref` (RefStore.resolve with tabId
|
// 2. Resolve target element by `ref` (compound frame/session identity)
|
||||||
// binding) or `selector` (DOM.querySelector + describeNode).
|
// or `selector` (DOM.querySelector + describeNode).
|
||||||
// 3. Scroll the node into view, then dispatch the appropriate
|
// 3. Scroll the node and its frame owners into view, project its live
|
||||||
// `Input.*` CDP events.
|
// content quad into the top viewport, then dispatch `Input.*` events.
|
||||||
// 4. Honour `AbortSignal` so canceled calls don't issue follow-up CDP
|
// 4. Honour `AbortSignal` so canceled calls don't issue follow-up CDP
|
||||||
// commands.
|
// commands.
|
||||||
|
|
||||||
import { ChromiumCdp } from "@/browser-driver/chromium-cdp";
|
import { ChromiumCdp } from "@/browser-driver/chromium-cdp";
|
||||||
|
import type { CdpTarget } from "@/browser-driver/frame-graph";
|
||||||
import type { SessionContext, SessionManager } from "@/session-manager/manager";
|
import type { SessionContext, SessionManager } from "@/session-manager/manager";
|
||||||
import type {
|
import type {
|
||||||
ClickParams,
|
ClickParams,
|
||||||
ClickResult,
|
ClickResult,
|
||||||
FillParams,
|
FillParams,
|
||||||
FillResult,
|
FillResult,
|
||||||
|
HoverParams,
|
||||||
|
HoverResult,
|
||||||
KeyModifier,
|
KeyModifier,
|
||||||
MouseButton,
|
MouseButton,
|
||||||
PressParams,
|
PressParams,
|
||||||
@@ -26,17 +29,13 @@ import type {
|
|||||||
SelectResult,
|
SelectResult,
|
||||||
} from "@/transport/types";
|
} from "@/transport/types";
|
||||||
import { attachDialogs, markDialogCursor } from "./dialogs";
|
import { attachDialogs, markDialogCursor } from "./dialogs";
|
||||||
import {
|
import { backendNodeToObject } from "./element-geometry";
|
||||||
backendNodeToObject,
|
|
||||||
boxCentre,
|
|
||||||
nodeCentre,
|
|
||||||
quadCentre,
|
|
||||||
scrollNodeIntoView,
|
|
||||||
} from "./element-geometry";
|
|
||||||
import { rpcError } from "./errors";
|
import { rpcError } from "./errors";
|
||||||
|
import { resolveNodeGeometry, scrollElementAndFramesIntoView } from "./frame-geometry";
|
||||||
import {
|
import {
|
||||||
type CdpRunner,
|
type CdpRunner,
|
||||||
type ChromeTabsApi,
|
type ChromeTabsApi,
|
||||||
|
cdpRunnerForTarget,
|
||||||
chromeTabsApi,
|
chromeTabsApi,
|
||||||
enforceAgentWindow,
|
enforceAgentWindow,
|
||||||
isRpcError,
|
isRpcError,
|
||||||
@@ -53,9 +52,12 @@ export interface InteractionDeps {
|
|||||||
defaultTimeoutMs?: number;
|
defaultTimeoutMs?: number;
|
||||||
/** Temporarily disable overlay click blocker during CDP automation. */
|
/** Temporarily disable overlay click blocker during CDP automation. */
|
||||||
bypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
|
bypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
|
||||||
|
/** Keep hover hit-testing active for the caller's next observation/action. */
|
||||||
|
keepOverlayBypassAfterHover?: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
const DEFAULT_TIMEOUT_MS = 30_000;
|
const DEFAULT_TIMEOUT_MS = 30_000;
|
||||||
|
const DEFAULT_HOVER_SETTLE_MS = 200;
|
||||||
|
|
||||||
let defaultDeps: { cdp: ChromiumCdp; tabsApi: ChromeTabsApi } | null = null;
|
let defaultDeps: { cdp: ChromiumCdp; tabsApi: ChromeTabsApi } | null = null;
|
||||||
function getDefaultDeps(): { cdp: ChromiumCdp; tabsApi: ChromeTabsApi } {
|
function getDefaultDeps(): { cdp: ChromiumCdp; tabsApi: ChromeTabsApi } {
|
||||||
@@ -100,6 +102,27 @@ function throwIfAborted(signal: AbortSignal | undefined): RpcError | null {
|
|||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function isAbortLikeError(err: unknown): boolean {
|
||||||
|
return err instanceof DOMException && err.name === "AbortError";
|
||||||
|
}
|
||||||
|
|
||||||
|
async function wait(ms: number, signal?: AbortSignal): Promise<void> {
|
||||||
|
if (signal?.aborted) throw new DOMException("aborted", "AbortError");
|
||||||
|
await new Promise<void>((resolve, reject) => {
|
||||||
|
const cleanup = () => signal?.removeEventListener("abort", onAbort);
|
||||||
|
const timer = setTimeout(() => {
|
||||||
|
cleanup();
|
||||||
|
resolve();
|
||||||
|
}, ms);
|
||||||
|
const onAbort = () => {
|
||||||
|
clearTimeout(timer);
|
||||||
|
cleanup();
|
||||||
|
reject(new DOMException("aborted", "AbortError"));
|
||||||
|
};
|
||||||
|
signal?.addEventListener("abort", onAbort, { once: true });
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Resolve `{ref?, selector?}` into a `backendNodeId`. Returns an
|
* Resolve `{ref?, selector?}` into a `backendNodeId`. Returns an
|
||||||
* `RpcError` if the caller supplied neither (or both), or if neither
|
* `RpcError` if the caller supplied neither (or both), or if neither
|
||||||
@@ -111,7 +134,16 @@ async function resolveBackendNode(
|
|||||||
target: { tabId: number },
|
target: { tabId: number },
|
||||||
params: { ref?: string; selector?: string },
|
params: { ref?: string; selector?: string },
|
||||||
toolName: string,
|
toolName: string,
|
||||||
): Promise<{ backendNodeId: number; usedRef?: string; usedSelector?: string } | RpcError> {
|
): Promise<
|
||||||
|
| {
|
||||||
|
backendNodeId: number;
|
||||||
|
cdpTarget: CdpTarget;
|
||||||
|
frameId?: string;
|
||||||
|
usedRef?: string;
|
||||||
|
usedSelector?: string;
|
||||||
|
}
|
||||||
|
| RpcError
|
||||||
|
> {
|
||||||
const hasRef = typeof params.ref === "string" && params.ref.length > 0;
|
const hasRef = typeof params.ref === "string" && params.ref.length > 0;
|
||||||
const hasSelector = typeof params.selector === "string" && params.selector.length > 0;
|
const hasSelector = typeof params.selector === "string" && params.selector.length > 0;
|
||||||
if (hasRef && hasSelector) {
|
if (hasRef && hasSelector) {
|
||||||
@@ -129,7 +161,15 @@ async function resolveBackendNode(
|
|||||||
if (hasRef) {
|
if (hasRef) {
|
||||||
const resolved = resolveSnapshotRef(ctx, params.ref as string, target.tabId);
|
const resolved = resolveSnapshotRef(ctx, params.ref as string, target.tabId);
|
||||||
if (isRpcError(resolved)) return resolved;
|
if (isRpcError(resolved)) return resolved;
|
||||||
return { backendNodeId: resolved.backendNodeId, usedRef: resolved.refKey };
|
return {
|
||||||
|
backendNodeId: resolved.backendNodeId,
|
||||||
|
cdpTarget: {
|
||||||
|
tabId: target.tabId,
|
||||||
|
...(resolved.cdpSessionId ? { sessionId: resolved.cdpSessionId } : {}),
|
||||||
|
},
|
||||||
|
...(resolved.frameId ? { frameId: resolved.frameId } : {}),
|
||||||
|
usedRef: resolved.refKey,
|
||||||
|
};
|
||||||
}
|
}
|
||||||
// selector path
|
// selector path
|
||||||
try {
|
try {
|
||||||
@@ -166,7 +206,7 @@ async function resolveBackendNode(
|
|||||||
message: "DOM.describeNode returned no backendNodeId",
|
message: "DOM.describeNode returned no backendNodeId",
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
return { backendNodeId, usedSelector: params.selector };
|
return { backendNodeId, cdpTarget: { tabId: target.tabId }, usedSelector: params.selector };
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
return {
|
return {
|
||||||
code: "cdp_failed",
|
code: "cdp_failed",
|
||||||
@@ -202,19 +242,19 @@ export async function handleClick(
|
|||||||
return { code: "cancelled", message: "click aborted" };
|
return { code: "cancelled", message: "click aborted" };
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
const geometry = await resolveNodeGeometry(
|
||||||
const scrollErr = await scrollNodeIntoView(deps.cdp, target.tabId, node.backendNodeId);
|
deps.cdp,
|
||||||
if (scrollErr) return scrollErr;
|
target.tabId,
|
||||||
} catch (err) {
|
{
|
||||||
return {
|
target: node.cdpTarget,
|
||||||
code: "cdp_failed",
|
backendNodeId: node.backendNodeId,
|
||||||
message: err instanceof Error ? err.message : String(err),
|
...(node.frameId ? { frameId: node.frameId } : {}),
|
||||||
};
|
},
|
||||||
}
|
{ scrollIntoView: true },
|
||||||
|
);
|
||||||
const centre = await nodeCentre(deps.cdp, target.tabId, node.backendNodeId);
|
if (isRpcError(geometry)) return geometry;
|
||||||
if (isRpcError(centre)) return centre;
|
const centre = geometry.actionPoint;
|
||||||
|
|
||||||
if (throwIfAborted(deps.signal)) {
|
if (throwIfAborted(deps.signal)) {
|
||||||
return { code: "cancelled", message: "click aborted" };
|
return { code: "cancelled", message: "click aborted" };
|
||||||
@@ -286,7 +326,104 @@ export async function handleClick(
|
|||||||
message: err instanceof Error ? err.message : String(err),
|
message: err instanceof Error ? err.message : String(err),
|
||||||
};
|
};
|
||||||
} finally {
|
} finally {
|
||||||
if (automationBypassEnabled && deps.bypassOverlay) {
|
if (automationBypassEnabled && deps.bypassOverlay && !deps.keepOverlayBypassAfterHover) {
|
||||||
|
try {
|
||||||
|
await deps.bypassOverlay(target.tabId, false);
|
||||||
|
} catch (err) {
|
||||||
|
console.debug("[bsk interaction] overlay bypass disable failed", err);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return attachDialogs(deps.cdp, target.tabId, dialogCursor, {
|
||||||
|
tab_id: target.tabId,
|
||||||
|
used_ref: node.usedRef,
|
||||||
|
used_selector: node.usedSelector,
|
||||||
|
x: centre.x,
|
||||||
|
y: centre.y,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// tool.hover
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
export async function handleHover(
|
||||||
|
manager: SessionManager,
|
||||||
|
params: HoverParams,
|
||||||
|
deps: InteractionDeps = getDefaultDeps(),
|
||||||
|
): Promise<HoverResult | RpcError> {
|
||||||
|
const ctxOrErr = lookupSession(manager, params, "hover");
|
||||||
|
if (isRpcError(ctxOrErr)) return ctxOrErr;
|
||||||
|
const ctx = ctxOrErr;
|
||||||
|
const aborted = throwIfAborted(deps.signal);
|
||||||
|
if (aborted) return aborted;
|
||||||
|
const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi);
|
||||||
|
if (isRpcError(target)) return target;
|
||||||
|
const denied = enforceAgentWindow(ctx, target, "hover");
|
||||||
|
if (denied) return denied;
|
||||||
|
const dialogCursor = markDialogCursor(deps.cdp, target.tabId);
|
||||||
|
|
||||||
|
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "hover");
|
||||||
|
if (isRpcError(node)) return node;
|
||||||
|
|
||||||
|
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||||
|
const geometry = await resolveNodeGeometry(
|
||||||
|
deps.cdp,
|
||||||
|
target.tabId,
|
||||||
|
{
|
||||||
|
target: node.cdpTarget,
|
||||||
|
backendNodeId: node.backendNodeId,
|
||||||
|
...(node.frameId ? { frameId: node.frameId } : {}),
|
||||||
|
},
|
||||||
|
{ scrollIntoView: true },
|
||||||
|
);
|
||||||
|
if (isRpcError(geometry)) return geometry;
|
||||||
|
const centre = geometry.actionPoint;
|
||||||
|
|
||||||
|
if (throwIfAborted(deps.signal)) {
|
||||||
|
return { code: "cancelled", message: "hover aborted" };
|
||||||
|
}
|
||||||
|
|
||||||
|
const modifiers = modifiersBitfield(params.modifiers);
|
||||||
|
const overlayBlocking = await checkOverlayAtPoint(deps.cdp, target.tabId, centre.x, centre.y);
|
||||||
|
let automationBypassEnabled = false;
|
||||||
|
let hoverCompleted = false;
|
||||||
|
if (overlayBlocking && deps.bypassOverlay) {
|
||||||
|
try {
|
||||||
|
await deps.bypassOverlay(target.tabId, true);
|
||||||
|
automationBypassEnabled = true;
|
||||||
|
} catch (err) {
|
||||||
|
console.debug("[bsk interaction] overlay bypass enable failed", err);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
await deps.cdp.send(target.tabId, "Input.dispatchMouseEvent", {
|
||||||
|
type: "mouseMoved",
|
||||||
|
x: centre.x,
|
||||||
|
y: centre.y,
|
||||||
|
modifiers,
|
||||||
|
});
|
||||||
|
const settleMs = params.settle_ms ?? DEFAULT_HOVER_SETTLE_MS;
|
||||||
|
if (settleMs > 0) {
|
||||||
|
await wait(settleMs, deps.signal);
|
||||||
|
}
|
||||||
|
hoverCompleted = true;
|
||||||
|
} catch (err) {
|
||||||
|
if (isAbortLikeError(err)) {
|
||||||
|
return { code: "cancelled", message: "hover aborted" };
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
code: "cdp_failed",
|
||||||
|
message: err instanceof Error ? err.message : String(err),
|
||||||
|
};
|
||||||
|
} finally {
|
||||||
|
if (
|
||||||
|
automationBypassEnabled &&
|
||||||
|
deps.bypassOverlay &&
|
||||||
|
(!deps.keepOverlayBypassAfterHover || !hoverCompleted)
|
||||||
|
) {
|
||||||
try {
|
try {
|
||||||
await deps.bypassOverlay(target.tabId, false);
|
await deps.bypassOverlay(target.tabId, false);
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
@@ -462,10 +599,11 @@ export async function handleFill(
|
|||||||
|
|
||||||
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "fill");
|
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "fill");
|
||||||
if (isRpcError(node)) return node;
|
if (isRpcError(node)) return node;
|
||||||
|
const nodeCdp = cdpRunnerForTarget(deps.cdp, node.cdpTarget);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||||
const described = await deps.cdp.send<{ node?: DescribedNode }>(
|
const described = await nodeCdp.send<{ node?: DescribedNode }>(
|
||||||
target.tabId,
|
target.tabId,
|
||||||
"DOM.describeNode",
|
"DOM.describeNode",
|
||||||
{
|
{
|
||||||
@@ -479,12 +617,18 @@ export async function handleFill(
|
|||||||
`element ${described.node?.nodeName ?? "?"} not fillable (need input/textarea/contenteditable)`,
|
`element ${described.node?.nodeName ?? "?"} not fillable (need input/textarea/contenteditable)`,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
const scrollErr = await scrollNodeIntoView(deps.cdp, target.tabId, node.backendNodeId);
|
const scrollErr = await scrollElementAndFramesIntoView(
|
||||||
|
deps.cdp,
|
||||||
|
target.tabId,
|
||||||
|
node.cdpTarget,
|
||||||
|
node.backendNodeId,
|
||||||
|
node.frameId,
|
||||||
|
);
|
||||||
if (scrollErr) return scrollErr;
|
if (scrollErr) return scrollErr;
|
||||||
if (throwIfAborted(deps.signal)) {
|
if (throwIfAborted(deps.signal)) {
|
||||||
return { code: "cancelled", message: "fill aborted" };
|
return { code: "cancelled", message: "fill aborted" };
|
||||||
}
|
}
|
||||||
await deps.cdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
|
await nodeCdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
return {
|
return {
|
||||||
code: "cdp_failed",
|
code: "cdp_failed",
|
||||||
@@ -496,7 +640,7 @@ export async function handleFill(
|
|||||||
return { code: "cancelled", message: "fill aborted" };
|
return { code: "cancelled", message: "fill aborted" };
|
||||||
}
|
}
|
||||||
|
|
||||||
const objectIdOrErr = await backendNodeToObject(deps.cdp, target.tabId, node.backendNodeId);
|
const objectIdOrErr = await backendNodeToObject(nodeCdp, target.tabId, node.backendNodeId);
|
||||||
if (isRpcError(objectIdOrErr)) return objectIdOrErr;
|
if (isRpcError(objectIdOrErr)) return objectIdOrErr;
|
||||||
const objectId = objectIdOrErr;
|
const objectId = objectIdOrErr;
|
||||||
const clearBefore = params.clear_before ?? true;
|
const clearBefore = params.clear_before ?? true;
|
||||||
@@ -505,7 +649,7 @@ export async function handleFill(
|
|||||||
if (clearBefore) {
|
if (clearBefore) {
|
||||||
// Clear input/textarea value or wipe contenteditable innerText,
|
// Clear input/textarea value or wipe contenteditable innerText,
|
||||||
// then fire `input` so frameworks observe the empty state.
|
// then fire `input` so frameworks observe the empty state.
|
||||||
await deps.cdp.send(target.tabId, "Runtime.callFunctionOn", {
|
await nodeCdp.send(target.tabId, "Runtime.callFunctionOn", {
|
||||||
objectId,
|
objectId,
|
||||||
functionDeclaration: `function() {
|
functionDeclaration: `function() {
|
||||||
if (this.isContentEditable) { this.textContent = ''; }
|
if (this.isContentEditable) { this.textContent = ''; }
|
||||||
@@ -532,7 +676,7 @@ export async function handleFill(
|
|||||||
return { code: "cancelled", message: "fill aborted" };
|
return { code: "cancelled", message: "fill aborted" };
|
||||||
}
|
}
|
||||||
// Fire `input` + `change` so React / Vue controlled inputs commit.
|
// Fire `input` + `change` so React / Vue controlled inputs commit.
|
||||||
await deps.cdp.send(target.tabId, "Runtime.callFunctionOn", {
|
await nodeCdp.send(target.tabId, "Runtime.callFunctionOn", {
|
||||||
objectId,
|
objectId,
|
||||||
functionDeclaration: `function() {
|
functionDeclaration: `function() {
|
||||||
this.dispatchEvent(new Event('input', { bubbles: true }));
|
this.dispatchEvent(new Event('input', { bubbles: true }));
|
||||||
@@ -730,14 +874,21 @@ export async function handlePress(
|
|||||||
if (params.ref || params.selector) {
|
if (params.ref || params.selector) {
|
||||||
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "press");
|
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "press");
|
||||||
if (isRpcError(node)) return node;
|
if (isRpcError(node)) return node;
|
||||||
|
const nodeCdp = cdpRunnerForTarget(deps.cdp, node.cdpTarget);
|
||||||
try {
|
try {
|
||||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||||
const scrollErr = await scrollNodeIntoView(deps.cdp, target.tabId, node.backendNodeId);
|
const scrollErr = await scrollElementAndFramesIntoView(
|
||||||
|
deps.cdp,
|
||||||
|
target.tabId,
|
||||||
|
node.cdpTarget,
|
||||||
|
node.backendNodeId,
|
||||||
|
node.frameId,
|
||||||
|
);
|
||||||
if (scrollErr) return scrollErr;
|
if (scrollErr) return scrollErr;
|
||||||
if (throwIfAborted(deps.signal)) {
|
if (throwIfAborted(deps.signal)) {
|
||||||
return { code: "cancelled", message: "press aborted" };
|
return { code: "cancelled", message: "press aborted" };
|
||||||
}
|
}
|
||||||
await deps.cdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
|
await nodeCdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
return {
|
return {
|
||||||
code: "cdp_failed",
|
code: "cdp_failed",
|
||||||
@@ -842,10 +993,11 @@ export async function handleSelect(
|
|||||||
|
|
||||||
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "select");
|
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "select");
|
||||||
if (isRpcError(node)) return node;
|
if (isRpcError(node)) return node;
|
||||||
|
const nodeCdp = cdpRunnerForTarget(deps.cdp, node.cdpTarget);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||||
const described = await deps.cdp.send<{ node?: DescribedNode }>(
|
const described = await nodeCdp.send<{ node?: DescribedNode }>(
|
||||||
target.tabId,
|
target.tabId,
|
||||||
"DOM.describeNode",
|
"DOM.describeNode",
|
||||||
{ backendNodeId: node.backendNodeId },
|
{ backendNodeId: node.backendNodeId },
|
||||||
@@ -868,12 +1020,18 @@ export async function handleSelect(
|
|||||||
"single-select <select> requires exactly one value",
|
"single-select <select> requires exactly one value",
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
const scrollErr = await scrollNodeIntoView(deps.cdp, target.tabId, node.backendNodeId);
|
const scrollErr = await scrollElementAndFramesIntoView(
|
||||||
|
deps.cdp,
|
||||||
|
target.tabId,
|
||||||
|
node.cdpTarget,
|
||||||
|
node.backendNodeId,
|
||||||
|
node.frameId,
|
||||||
|
);
|
||||||
if (scrollErr) return scrollErr;
|
if (scrollErr) return scrollErr;
|
||||||
if (throwIfAborted(deps.signal)) {
|
if (throwIfAborted(deps.signal)) {
|
||||||
return { code: "cancelled", message: "select aborted" };
|
return { code: "cancelled", message: "select aborted" };
|
||||||
}
|
}
|
||||||
await deps.cdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
|
await nodeCdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
return {
|
return {
|
||||||
code: "cdp_failed",
|
code: "cdp_failed",
|
||||||
@@ -885,12 +1043,12 @@ export async function handleSelect(
|
|||||||
return { code: "cancelled", message: "select aborted" };
|
return { code: "cancelled", message: "select aborted" };
|
||||||
}
|
}
|
||||||
|
|
||||||
const objectIdOrErr = await backendNodeToObject(deps.cdp, target.tabId, node.backendNodeId);
|
const objectIdOrErr = await backendNodeToObject(nodeCdp, target.tabId, node.backendNodeId);
|
||||||
if (isRpcError(objectIdOrErr)) return objectIdOrErr;
|
if (isRpcError(objectIdOrErr)) return objectIdOrErr;
|
||||||
const objectId = objectIdOrErr;
|
const objectId = objectIdOrErr;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const evaluated = await deps.cdp.send<{
|
const evaluated = await nodeCdp.send<{
|
||||||
result?: { value?: SelectMutationResult | null };
|
result?: { value?: SelectMutationResult | null };
|
||||||
}>(target.tabId, "Runtime.callFunctionOn", {
|
}>(target.tabId, "Runtime.callFunctionOn", {
|
||||||
objectId,
|
objectId,
|
||||||
@@ -949,9 +1107,6 @@ export async function handleSelect(
|
|||||||
|
|
||||||
export const __testing__ = {
|
export const __testing__ = {
|
||||||
DEFAULT_TIMEOUT_MS,
|
DEFAULT_TIMEOUT_MS,
|
||||||
quadCentre,
|
|
||||||
boxCentre,
|
|
||||||
resolveBackendNode,
|
resolveBackendNode,
|
||||||
nodeCentre,
|
|
||||||
isFillable,
|
isFillable,
|
||||||
};
|
};
|
||||||
@@ -325,14 +325,14 @@ function startLifecycleWait(
|
|||||||
if (source.tabId !== expectedTabId) return;
|
if (source.tabId !== expectedTabId) return;
|
||||||
|
|
||||||
if (targetName === "commit" && method === "Page.frameNavigated") {
|
if (targetName === "commit" && method === "Page.frameNavigated") {
|
||||||
const p = params as { frame?: { id?: string; parentId?: string } };
|
const p = params as { frame?: { id?: string; parentId?: string; loaderId?: string } };
|
||||||
const expectedFrameId = currentFrameId();
|
const expectedFrameId = currentFrameId();
|
||||||
if (expectedFrameId.length > 0) {
|
if (expectedFrameId.length > 0) {
|
||||||
if (p.frame?.id !== expectedFrameId) return;
|
if (p.frame?.id !== expectedFrameId) return;
|
||||||
} else if (p.frame?.parentId) {
|
} else if (p.frame?.parentId) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if (!acceptLifecycleEvent("commit", p.frame?.id)) return;
|
if (!acceptLifecycleEvent("commit", p.frame?.id, p.frame?.loaderId)) return;
|
||||||
finish({ reached: "match", lastLifecycle: "commit" });
|
finish({ reached: "match", lastLifecycle: "commit" });
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -37,17 +37,22 @@ export async function handleNetwork(
|
|||||||
manager: SessionManager,
|
manager: SessionManager,
|
||||||
params: NetworkParams,
|
params: NetworkParams,
|
||||||
deps: NetworkDeps = defaultNetworkDeps(),
|
deps: NetworkDeps = defaultNetworkDeps(),
|
||||||
|
signal?: AbortSignal,
|
||||||
): Promise<NetworkResult | RpcError> {
|
): Promise<NetworkResult | RpcError> {
|
||||||
|
if (signal?.aborted) return { code: "cancelled", message: "network aborted" };
|
||||||
const ctxOrErr = lookupSession(manager, params, "network");
|
const ctxOrErr = lookupSession(manager, params, "network");
|
||||||
if (isRpcError(ctxOrErr)) return ctxOrErr;
|
if (isRpcError(ctxOrErr)) return ctxOrErr;
|
||||||
const bounds = parseBufferedReadBounds(params);
|
const bounds = parseBufferedReadBounds(params);
|
||||||
if (isRpcError(bounds)) return bounds;
|
if (isRpcError(bounds)) return bounds;
|
||||||
const target = await resolveTargetTab(manager, ctxOrErr, params.tab_id, deps.tabsApi);
|
const target = await resolveTargetTab(manager, ctxOrErr, params.tab_id, deps.tabsApi);
|
||||||
if (isRpcError(target)) return target;
|
if (isRpcError(target)) return target;
|
||||||
|
if (signal?.aborted) return { code: "cancelled", message: "network aborted" };
|
||||||
|
|
||||||
try {
|
try {
|
||||||
|
if (signal?.aborted) return { code: "cancelled", message: "network aborted" };
|
||||||
deps.cdp.trackSessionTab?.(ctxOrErr.sessionId, target.tabId);
|
deps.cdp.trackSessionTab?.(ctxOrErr.sessionId, target.tabId);
|
||||||
await deps.cdp.ensureNetworkCapture(target.tabId);
|
await deps.cdp.ensureNetworkCapture(target.tabId);
|
||||||
|
if (signal?.aborted) return { code: "cancelled", message: "network aborted" };
|
||||||
return deps.cdp.networkEntriesSince(
|
return deps.cdp.networkEntriesSince(
|
||||||
target.tabId,
|
target.tabId,
|
||||||
bounds.since,
|
bounds.since,
|
||||||
|
|||||||
@@ -5,17 +5,21 @@
|
|||||||
|
|
||||||
import {
|
import {
|
||||||
type ActiveScopeBlock,
|
type ActiveScopeBlock,
|
||||||
|
applyVomInteractionRecovery,
|
||||||
type CondSurface,
|
type CondSurface,
|
||||||
|
isVomReferenceNode,
|
||||||
renderVom,
|
renderVom,
|
||||||
type VomNode,
|
type VomNode,
|
||||||
|
type VomOptions,
|
||||||
type VomScene,
|
type VomScene,
|
||||||
} from "@browser-skill/vom";
|
} from "@browser-skill/vom";
|
||||||
import { ChromiumCdp } from "@/browser-driver/chromium-cdp";
|
import { ChromiumCdp } from "@/browser-driver/chromium-cdp";
|
||||||
|
import type { CdpTarget } from "@/browser-driver/frame-graph";
|
||||||
import {
|
import {
|
||||||
type CaptureSuppressSendToTab,
|
type CaptureSuppressSendToTab,
|
||||||
withOverlaysHiddenForCapture,
|
withOverlaysHiddenForCapture,
|
||||||
} from "@/lib/capture-suppress-bridge";
|
} from "@/lib/capture-suppress-bridge";
|
||||||
import type { SessionManager } from "@/session-manager/manager";
|
import type { SessionContext, SessionManager } from "@/session-manager/manager";
|
||||||
import type {
|
import type {
|
||||||
GetHtmlParams,
|
GetHtmlParams,
|
||||||
GetHtmlResult,
|
GetHtmlResult,
|
||||||
@@ -28,15 +32,17 @@ import type {
|
|||||||
SnapshotResult,
|
SnapshotResult,
|
||||||
} from "@/transport/types";
|
} from "@/transport/types";
|
||||||
import { attachDialogs, markDialogCursor } from "./dialogs";
|
import { attachDialogs, markDialogCursor } from "./dialogs";
|
||||||
import { nodeBoundingRect, scrollNodeIntoView } from "./element-geometry";
|
|
||||||
import { rpcError } from "./errors";
|
import { rpcError } from "./errors";
|
||||||
|
import { resolveNodeGeometry } from "./frame-geometry";
|
||||||
import {
|
import {
|
||||||
type ChromeTabsApi,
|
type ChromeTabsApi,
|
||||||
enforceToolTargetScope,
|
enforceToolTargetScope,
|
||||||
isRpcError,
|
isRpcError,
|
||||||
lookupSession,
|
lookupSession,
|
||||||
|
type ResolvedTargetTab,
|
||||||
resolveCdpAccessibleTargetTab,
|
resolveCdpAccessibleTargetTab,
|
||||||
type CdpRunner as SharedCdpRunner,
|
type CdpRunner as SharedCdpRunner,
|
||||||
|
sendToCdpTarget,
|
||||||
normaliseRef as sharedNormaliseRef,
|
normaliseRef as sharedNormaliseRef,
|
||||||
type ToolEffect,
|
type ToolEffect,
|
||||||
} from "./shared";
|
} from "./shared";
|
||||||
@@ -47,6 +53,11 @@ import {
|
|||||||
captureViewModel,
|
captureViewModel,
|
||||||
collectOverlayExcludedBackendIds,
|
collectOverlayExcludedBackendIds,
|
||||||
} from "./vom/capture";
|
} from "./vom/capture";
|
||||||
|
import { type CapturedFrameDocument, captureFrameData } from "./vom/frame-capture";
|
||||||
|
import {
|
||||||
|
type CaptureVomObservationResult,
|
||||||
|
projectRecordSafeObservation,
|
||||||
|
} from "./vom/record-safe-observation";
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Shared helpers (legacy aliases — observation.ts kept exporting these
|
// Shared helpers (legacy aliases — observation.ts kept exporting these
|
||||||
@@ -134,35 +145,66 @@ function defaultScreenshotDeps(): ScreenshotDeps {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function cancelled(tool: string): RpcError {
|
||||||
|
return { code: "cancelled", message: `${tool} aborted` };
|
||||||
|
}
|
||||||
|
|
||||||
|
function abortError(tool: string): Error {
|
||||||
|
const error = new Error(`${tool} aborted`);
|
||||||
|
error.name = "AbortError";
|
||||||
|
return error;
|
||||||
|
}
|
||||||
|
|
||||||
|
function isAbortError(error: unknown): boolean {
|
||||||
|
return (
|
||||||
|
typeof error === "object" &&
|
||||||
|
error !== null &&
|
||||||
|
(error as { name?: string }).name === "AbortError"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function throwIfAborted(signal: AbortSignal | undefined, tool: string): void {
|
||||||
|
if (signal?.aborted) throw abortError(tool);
|
||||||
|
}
|
||||||
|
|
||||||
async function captureElementScreenshot(
|
async function captureElementScreenshot(
|
||||||
cdp: SharedCdpRunner,
|
cdp: SharedCdpRunner,
|
||||||
tabId: number,
|
tabId: number,
|
||||||
|
target: CdpTarget,
|
||||||
backendNodeId: number,
|
backendNodeId: number,
|
||||||
|
frameId?: string,
|
||||||
|
signal?: AbortSignal,
|
||||||
): Promise<{ image_base64: string; width: number; height: number } | RpcError> {
|
): Promise<{ image_base64: string; width: number; height: number } | RpcError> {
|
||||||
const scrollErr = await scrollNodeIntoView(cdp, tabId, backendNodeId);
|
if (signal?.aborted) return cancelled("screenshot");
|
||||||
if (scrollErr) return scrollErr;
|
const geometry = await resolveNodeGeometry(
|
||||||
|
cdp,
|
||||||
const rectOrErr = await nodeBoundingRect(cdp, tabId, backendNodeId);
|
tabId,
|
||||||
if (isRpcError(rectOrErr)) return rectOrErr;
|
{ target, backendNodeId, ...(frameId ? { frameId } : {}) },
|
||||||
|
{ scrollIntoView: true },
|
||||||
|
);
|
||||||
|
if (isRpcError(geometry)) return geometry;
|
||||||
|
if (signal?.aborted) return cancelled("screenshot");
|
||||||
|
const rect = geometry.topBounds;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const shot = await cdp.send<{ data?: string }>(tabId, "Page.captureScreenshot", {
|
const shot = await cdp.send<{ data?: string }>(tabId, "Page.captureScreenshot", {
|
||||||
format: "png",
|
format: "png",
|
||||||
clip: {
|
clip: {
|
||||||
x: rectOrErr.x,
|
x: rect.x,
|
||||||
y: rectOrErr.y,
|
y: rect.y,
|
||||||
width: rectOrErr.width,
|
width: rect.width,
|
||||||
height: rectOrErr.height,
|
height: rect.height,
|
||||||
scale: 1,
|
scale: 1,
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
if (signal?.aborted) return cancelled("screenshot");
|
||||||
const image_base64 = shot.data ?? "";
|
const image_base64 = shot.data ?? "";
|
||||||
if (!image_base64) {
|
if (!image_base64) {
|
||||||
return { code: "cdp_failed", message: "Page.captureScreenshot returned no data" };
|
return { code: "cdp_failed", message: "Page.captureScreenshot returned no data" };
|
||||||
}
|
}
|
||||||
const dims = parsePngDimensions(image_base64) ?? {
|
const dims = parsePngDimensions(image_base64) ?? {
|
||||||
width: Math.round(rectOrErr.width),
|
width: Math.round(rect.width),
|
||||||
height: Math.round(rectOrErr.height),
|
height: Math.round(rect.height),
|
||||||
};
|
};
|
||||||
return { image_base64, width: dims.width, height: dims.height };
|
return { image_base64, width: dims.width, height: dims.height };
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
@@ -173,11 +215,71 @@ async function captureElementScreenshot(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Full-tab PNG capture. The primary path is `chrome.tabs.captureVisibleTab`;
|
||||||
|
* when it rejects, fall back to CDP `Page.captureScreenshot` with
|
||||||
|
* `fromSurface: true`. `captureVisibleTab` reads back the window surface,
|
||||||
|
* which fails outright on some Windows/Chrome combinations (Chromium's
|
||||||
|
* FAILURE_REASON_READBACK_FAILED — "Failed to capture tab: image readback
|
||||||
|
* failed"); the CDP path captures through the renderer's BeginFrame
|
||||||
|
* pipeline instead, which does not depend on that readback.
|
||||||
|
*
|
||||||
|
* When both paths fail the returned error carries
|
||||||
|
* `data.reason = "screenshot_capture_failed"` and both underlying messages
|
||||||
|
* so the CLI can point the user at the browser-side cause.
|
||||||
|
*/
|
||||||
|
async function captureFullTabPng(
|
||||||
|
deps: ScreenshotDeps,
|
||||||
|
ctx: SessionContext,
|
||||||
|
target: ResolvedTargetTab,
|
||||||
|
signal?: AbortSignal,
|
||||||
|
): Promise<string | RpcError> {
|
||||||
|
if (signal?.aborted) return cancelled("screenshot");
|
||||||
|
try {
|
||||||
|
const dataUrl = await deps.captureApi.captureVisibleTab(target.windowId, { format: "png" });
|
||||||
|
if (signal?.aborted) return cancelled("screenshot");
|
||||||
|
return stripDataUrlPrefix(dataUrl);
|
||||||
|
} catch (primaryErr) {
|
||||||
|
if (signal?.aborted) return cancelled("screenshot");
|
||||||
|
const cdp = deps.cdp;
|
||||||
|
if (!cdp) {
|
||||||
|
return {
|
||||||
|
code: "cdp_failed",
|
||||||
|
message: primaryErr instanceof Error ? primaryErr.message : String(primaryErr),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
let fallbackMsg: string;
|
||||||
|
try {
|
||||||
|
if (signal?.aborted) return cancelled("screenshot");
|
||||||
|
cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||||
|
await cdp.ensureAttachedToUrl?.(target.tabId, target.url);
|
||||||
|
if (signal?.aborted) return cancelled("screenshot");
|
||||||
|
const shot = await cdp.send<{ data?: string }>(target.tabId, "Page.captureScreenshot", {
|
||||||
|
format: "png",
|
||||||
|
fromSurface: true,
|
||||||
|
});
|
||||||
|
if (signal?.aborted) return cancelled("screenshot");
|
||||||
|
if (shot.data) return shot.data;
|
||||||
|
fallbackMsg = "Page.captureScreenshot returned no data";
|
||||||
|
} catch (fallbackErr) {
|
||||||
|
fallbackMsg = fallbackErr instanceof Error ? fallbackErr.message : String(fallbackErr);
|
||||||
|
}
|
||||||
|
const primaryMsg = primaryErr instanceof Error ? primaryErr.message : String(primaryErr);
|
||||||
|
return rpcError(
|
||||||
|
"cdp_failed",
|
||||||
|
"screenshot_capture_failed",
|
||||||
|
`captureVisibleTab failed: ${primaryMsg}; CDP Page.captureScreenshot fallback failed: ${fallbackMsg}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
export async function handleScreenshot(
|
export async function handleScreenshot(
|
||||||
manager: SessionManager,
|
manager: SessionManager,
|
||||||
params: ScreenshotParams,
|
params: ScreenshotParams,
|
||||||
deps: ScreenshotDeps = defaultScreenshotDeps(),
|
deps: ScreenshotDeps = defaultScreenshotDeps(),
|
||||||
|
signal?: AbortSignal,
|
||||||
): Promise<ScreenshotResult | RpcError> {
|
): Promise<ScreenshotResult | RpcError> {
|
||||||
|
if (signal?.aborted) return cancelled("screenshot");
|
||||||
const ctxOrErr = lookupSession(manager, params, "screenshot");
|
const ctxOrErr = lookupSession(manager, params, "screenshot");
|
||||||
if (isRpcError(ctxOrErr)) return ctxOrErr;
|
if (isRpcError(ctxOrErr)) return ctxOrErr;
|
||||||
const ctx = ctxOrErr;
|
const ctx = ctxOrErr;
|
||||||
@@ -189,6 +291,7 @@ export async function handleScreenshot(
|
|||||||
"screenshot",
|
"screenshot",
|
||||||
);
|
);
|
||||||
if (isRpcError(target)) return target;
|
if (isRpcError(target)) return target;
|
||||||
|
if (signal?.aborted) return cancelled("screenshot");
|
||||||
const dialogCursor = deps.cdp ? markDialogCursor(deps.cdp, target.tabId) : 0;
|
const dialogCursor = deps.cdp ? markDialogCursor(deps.cdp, target.tabId) : 0;
|
||||||
const withShotDialogs = <T extends object>(result: T) =>
|
const withShotDialogs = <T extends object>(result: T) =>
|
||||||
deps.cdp ? attachDialogs(deps.cdp, target.tabId, dialogCursor, result) : result;
|
deps.cdp ? attachDialogs(deps.cdp, target.tabId, dialogCursor, result) : result;
|
||||||
@@ -200,15 +303,30 @@ export async function handleScreenshot(
|
|||||||
}
|
}
|
||||||
const node = resolveSnapshotRef(ctx, ref, target.tabId);
|
const node = resolveSnapshotRef(ctx, ref, target.tabId);
|
||||||
if (isRpcError(node)) return node;
|
if (isRpcError(node)) return node;
|
||||||
|
if (signal?.aborted) return cancelled("screenshot");
|
||||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||||
await deps.cdp.ensureAttachedToUrl?.(target.tabId, target.url);
|
await deps.cdp.ensureAttachedToUrl?.(target.tabId, target.url);
|
||||||
|
if (signal?.aborted) return cancelled("screenshot");
|
||||||
const cdp = deps.cdp;
|
const cdp = deps.cdp;
|
||||||
|
const nodeTarget = {
|
||||||
|
tabId: target.tabId,
|
||||||
|
...(node.cdpSessionId ? { sessionId: node.cdpSessionId } : {}),
|
||||||
|
};
|
||||||
const captured = await withOverlaysHiddenForCapture(
|
const captured = await withOverlaysHiddenForCapture(
|
||||||
target.tabId,
|
target.tabId,
|
||||||
() => captureElementScreenshot(cdp, target.tabId, node.backendNodeId),
|
() =>
|
||||||
|
captureElementScreenshot(
|
||||||
|
cdp,
|
||||||
|
target.tabId,
|
||||||
|
nodeTarget,
|
||||||
|
node.backendNodeId,
|
||||||
|
node.frameId,
|
||||||
|
signal,
|
||||||
|
),
|
||||||
deps.sendToTab,
|
deps.sendToTab,
|
||||||
);
|
);
|
||||||
if (isRpcError(captured)) return captured;
|
if (isRpcError(captured)) return captured;
|
||||||
|
if (signal?.aborted) return cancelled("screenshot");
|
||||||
return withShotDialogs({
|
return withShotDialogs({
|
||||||
image_base64: captured.image_base64,
|
image_base64: captured.image_base64,
|
||||||
width: captured.width,
|
width: captured.width,
|
||||||
@@ -226,27 +344,22 @@ export async function handleScreenshot(
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
const captured = await withOverlaysHiddenForCapture(
|
||||||
const dataUrl = await withOverlaysHiddenForCapture(
|
target.tabId,
|
||||||
target.tabId,
|
() => captureFullTabPng(deps, ctx, target, signal),
|
||||||
() => deps.captureApi.captureVisibleTab(target.windowId, { format: "png" }),
|
deps.sendToTab,
|
||||||
deps.sendToTab,
|
);
|
||||||
);
|
if (isRpcError(captured)) return captured;
|
||||||
const image_base64 = stripDataUrlPrefix(dataUrl);
|
if (signal?.aborted) return cancelled("screenshot");
|
||||||
const dims = parsePngDimensions(image_base64) ?? { width: 0, height: 0 };
|
const image_base64 = captured;
|
||||||
return withShotDialogs({
|
const dims = parsePngDimensions(image_base64) ?? { width: 0, height: 0 };
|
||||||
image_base64,
|
return withShotDialogs({
|
||||||
width: dims.width,
|
image_base64,
|
||||||
height: dims.height,
|
width: dims.width,
|
||||||
format: "png",
|
height: dims.height,
|
||||||
tab_id: target.tabId,
|
format: "png",
|
||||||
});
|
tab_id: target.tabId,
|
||||||
} catch (err) {
|
});
|
||||||
return {
|
|
||||||
code: "cdp_failed",
|
|
||||||
message: err instanceof Error ? err.message : String(err),
|
|
||||||
};
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
@@ -263,6 +376,7 @@ export type CdpRunner = SharedCdpRunner;
|
|||||||
/** Subset of CDP `AXNode` we care about — see `Accessibility.AXNode`. */
|
/** Subset of CDP `AXNode` we care about — see `Accessibility.AXNode`. */
|
||||||
export interface CdpAxNode {
|
export interface CdpAxNode {
|
||||||
nodeId: string;
|
nodeId: string;
|
||||||
|
frameId?: string;
|
||||||
parentId?: string;
|
parentId?: string;
|
||||||
backendDOMNodeId?: number;
|
backendDOMNodeId?: number;
|
||||||
ignored?: boolean;
|
ignored?: boolean;
|
||||||
@@ -302,7 +416,15 @@ function isModalSignal(
|
|||||||
|
|
||||||
function isSensitive(capturedNode: CapturedNode | undefined): boolean {
|
function isSensitive(capturedNode: CapturedNode | undefined): boolean {
|
||||||
const attrs = capturedNode?.attrs ?? {};
|
const attrs = capturedNode?.attrs ?? {};
|
||||||
return (attrs.type ?? "").toLowerCase() === "password";
|
const type = (attrs.type ?? "").toLowerCase();
|
||||||
|
if (type === "password") return true;
|
||||||
|
const autocomplete = (attrs.autocomplete ?? "").toLowerCase();
|
||||||
|
return (
|
||||||
|
autocomplete.startsWith("cc-") ||
|
||||||
|
autocomplete === "one-time-code" ||
|
||||||
|
autocomplete === "current-password" ||
|
||||||
|
autocomplete === "new-password"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
interface AxNodeSignals {
|
interface AxNodeSignals {
|
||||||
@@ -352,7 +474,13 @@ const AX_TEXT_STOP_ROLES = new Set([
|
|||||||
"columnheader",
|
"columnheader",
|
||||||
"rowheader",
|
"rowheader",
|
||||||
]);
|
]);
|
||||||
const SENSITIVE_AX_INPUT_TYPES = new Set(["password", "credit-card"]);
|
const SENSITIVE_AX_INPUT_TYPES = new Set([
|
||||||
|
"password",
|
||||||
|
"credit-card",
|
||||||
|
"one-time-code",
|
||||||
|
"current-password",
|
||||||
|
"new-password",
|
||||||
|
]);
|
||||||
|
|
||||||
function axPropertyString(axNode: CdpAxNode, name: string): string | undefined {
|
function axPropertyString(axNode: CdpAxNode, name: string): string | undefined {
|
||||||
const prop = axNode.properties?.find((item) => item.name === name);
|
const prop = axNode.properties?.find((item) => item.name === name);
|
||||||
@@ -495,7 +623,7 @@ function iframeNameFor(node: CapturedNode): string | undefined {
|
|||||||
function isRenderableIframeControl(node: CapturedNode): boolean {
|
function isRenderableIframeControl(node: CapturedNode): boolean {
|
||||||
const tag = normalizeTag(node.tag);
|
const tag = normalizeTag(node.tag);
|
||||||
if (!IFRAME_RENDERABLE_TAGS.has(tag)) return false;
|
if (!IFRAME_RENDERABLE_TAGS.has(tag)) return false;
|
||||||
if (!node.rect) return false;
|
if (!node.localRect && !node.rect) return false;
|
||||||
if (node.pointerEvents === "none") return false;
|
if (node.pointerEvents === "none") return false;
|
||||||
if ((node.attrs.type ?? "").toLowerCase() === "hidden") return false;
|
if ((node.attrs.type ?? "").toLowerCase() === "hidden") return false;
|
||||||
|
|
||||||
@@ -667,7 +795,12 @@ function applyCapturedSignals(
|
|||||||
signals: VomNodeDomSignals,
|
signals: VomNodeDomSignals,
|
||||||
): VomNode {
|
): VomNode {
|
||||||
if (!capturedNode) return node;
|
if (!capturedNode) return node;
|
||||||
const attrs = capturedNode.attrs;
|
const attrs =
|
||||||
|
node.sensitive && Object.prototype.hasOwnProperty.call(capturedNode.attrs, "value")
|
||||||
|
? Object.fromEntries(
|
||||||
|
Object.entries(capturedNode.attrs).filter(([name]) => name.toLowerCase() !== "value"),
|
||||||
|
)
|
||||||
|
: capturedNode.attrs;
|
||||||
const text = cleanAttr(capturedNode.textContent);
|
const text = cleanAttr(capturedNode.textContent);
|
||||||
const nearbyText = nearbyTextFor(capturedNode, signals.childrenByParentId);
|
const nearbyText = nearbyTextFor(capturedNode, signals.childrenByParentId);
|
||||||
const placeholder = cleanAttr(capturedNode.formPlaceholder) ?? cleanAttr(attrs.placeholder);
|
const placeholder = cleanAttr(capturedNode.formPlaceholder) ?? cleanAttr(attrs.placeholder);
|
||||||
@@ -734,8 +867,7 @@ function vomNodeFromCaptured(
|
|||||||
signals: VomNodeDomSignals,
|
signals: VomNodeDomSignals,
|
||||||
): VomNode {
|
): VomNode {
|
||||||
const sensitive = isSensitive(capturedNode);
|
const sensitive = isSensitive(capturedNode);
|
||||||
const value =
|
const value = sensitive ? undefined : capturedNode.formValue;
|
||||||
capturedNode.formValue ?? (sensitive ? (capturedNode.attrs.value ?? "") : undefined);
|
|
||||||
const tag = normalizeTag(capturedNode.tag);
|
const tag = normalizeTag(capturedNode.tag);
|
||||||
const node = applyCapturedSignals(
|
const node = applyCapturedSignals(
|
||||||
{
|
{
|
||||||
@@ -771,8 +903,17 @@ function axVomNode(
|
|||||||
): VomNode {
|
): VomNode {
|
||||||
const role = axString(axNode.role);
|
const role = axString(axNode.role);
|
||||||
const capturedValue = capturedNode?.formValue;
|
const capturedValue = capturedNode?.formValue;
|
||||||
const value = capturedValue ?? axValue(axNode.value);
|
const sourceValue = capturedValue ?? axValue(axNode.value);
|
||||||
const signalsForNode = axSignals.get(axNode.nodeId);
|
const signalsForNode = axSignals.get(axNode.nodeId);
|
||||||
|
const sensitive = isSensitive(capturedNode) || signalsForNode?.sensitive === true;
|
||||||
|
const value = sensitive ? undefined : sourceValue;
|
||||||
|
const inputState =
|
||||||
|
inputStateFor(capturedNode, sourceValue) ??
|
||||||
|
(sensitive && ["textbox", "searchbox"].includes(role?.toLowerCase() ?? "")
|
||||||
|
? sourceValue === undefined || sourceValue === ""
|
||||||
|
? "empty"
|
||||||
|
: "filled"
|
||||||
|
: undefined);
|
||||||
let name = FORM_CONTROL_TAGS.has(normalizeTag(capturedNode?.tag))
|
let name = FORM_CONTROL_TAGS.has(normalizeTag(capturedNode?.tag))
|
||||||
? formControlName(capturedNode, axString(axNode.name), axSiblingLabel(axNode, axById), signals)
|
? formControlName(capturedNode, axString(axNode.name), axSiblingLabel(axNode, axById), signals)
|
||||||
: (axString(axNode.name) ?? signalsForNode?.aggregatedText);
|
: (axString(axNode.name) ?? signalsForNode?.aggregatedText);
|
||||||
@@ -788,8 +929,8 @@ function axVomNode(
|
|||||||
position: capturedNode?.position || "static",
|
position: capturedNode?.position || "static",
|
||||||
pointerEvents: capturedNode?.pointerEvents || "auto",
|
pointerEvents: capturedNode?.pointerEvents || "auto",
|
||||||
modal: isModalSignal(axNode, capturedNode),
|
modal: isModalSignal(axNode, capturedNode),
|
||||||
sensitive: isSensitive(capturedNode) || signalsForNode?.sensitive === true,
|
sensitive,
|
||||||
inputState: inputStateFor(capturedNode, value),
|
inputState,
|
||||||
};
|
};
|
||||||
if (role) node.role = role;
|
if (role) node.role = role;
|
||||||
if (name) node.name = name;
|
if (name) node.name = name;
|
||||||
@@ -927,39 +1068,28 @@ function buildActiveScopeBlocks(nodes: VomNode[], signals: VomNodeDomSignals): A
|
|||||||
return blocks;
|
return blocks;
|
||||||
}
|
}
|
||||||
|
|
||||||
function labelForSurfaceMatch(node: VomNode): string {
|
|
||||||
return (
|
|
||||||
cleanAttr(node.name) ??
|
|
||||||
cleanAttr(node.text) ??
|
|
||||||
cleanAttr(node.attrs?.["aria-label"]) ??
|
|
||||||
cleanAttr(node.attrs?.title) ??
|
|
||||||
""
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
function buildConditionalSurfaces(nodes: VomNode[], captured: CapturedViewModel): CondSurface[] {
|
function buildConditionalSurfaces(nodes: VomNode[], captured: CapturedViewModel): CondSurface[] {
|
||||||
const probes = captured.surfaceProbes ?? [];
|
const probes = captured.surfaceProbes ?? [];
|
||||||
if (probes.length === 0) return [];
|
if (probes.length === 0) return [];
|
||||||
|
|
||||||
const surfaces: CondSurface[] = [];
|
const surfaces: CondSurface[] = [];
|
||||||
const nodesWithLabels = nodes
|
const recoveredNodes = applyVomInteractionRecovery(nodes);
|
||||||
.map((node) => ({ node, key: normalizeProbeKey(labelForSurfaceMatch(node)) }))
|
const signals = capturedOnlySignals(captured.nodes);
|
||||||
.filter((entry) => entry.key.length > 0);
|
|
||||||
const used = new Set<number>();
|
const used = new Set<number>();
|
||||||
for (const probe of probes) {
|
for (const probe of probes) {
|
||||||
const probeKey = normalizeProbeKey(probe.triggerLabel);
|
if (probe.subItems.length === 0 || used.has(probe.triggerBackendNodeId)) continue;
|
||||||
if (!probeKey || probe.subItems.length === 0) continue;
|
const match = findSurfaceTriggerNode(
|
||||||
const match =
|
probe.triggerBackendNodeId,
|
||||||
nodesWithLabels.find((entry) => entry.key === probeKey && !used.has(entry.node.id)) ??
|
recoveredNodes,
|
||||||
nodesWithLabels.find(
|
signals,
|
||||||
(entry) =>
|
probe.triggerPoint,
|
||||||
!used.has(entry.node.id) &&
|
);
|
||||||
(probeKey.startsWith(entry.key) || entry.key.startsWith(probeKey)),
|
|
||||||
);
|
|
||||||
if (!match) continue;
|
if (!match) continue;
|
||||||
used.add(match.node.id);
|
if (used.has(match.id)) continue;
|
||||||
|
used.add(probe.triggerBackendNodeId);
|
||||||
|
used.add(match.id);
|
||||||
surfaces.push({
|
surfaces.push({
|
||||||
triggerId: match.node.id,
|
triggerId: match.id,
|
||||||
triggerAction: probe.triggerAction,
|
triggerAction: probe.triggerAction,
|
||||||
subItems: probe.subItems,
|
subItems: probe.subItems,
|
||||||
});
|
});
|
||||||
@@ -967,6 +1097,77 @@ function buildConditionalSurfaces(nodes: VomNode[], captured: CapturedViewModel)
|
|||||||
return surfaces;
|
return surfaces;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function findSurfaceTriggerNode(
|
||||||
|
triggerBackendNodeId: number,
|
||||||
|
nodes: VomNode[],
|
||||||
|
signals: VomNodeDomSignals,
|
||||||
|
triggerPoint?: { x: number; y: number },
|
||||||
|
): VomNode | undefined {
|
||||||
|
const renderedById = new Map(nodes.map((node) => [node.id, node]));
|
||||||
|
const exact = renderedById.get(triggerBackendNodeId);
|
||||||
|
if (exact && isSurfaceAttachableNode(exact)) return exact;
|
||||||
|
|
||||||
|
const domDescendant = nodes.find(
|
||||||
|
(node) =>
|
||||||
|
isSurfaceAttachableNode(node) &&
|
||||||
|
(node.domParentId === triggerBackendNodeId ||
|
||||||
|
node.domAncestorIds?.includes(triggerBackendNodeId) === true),
|
||||||
|
);
|
||||||
|
if (domDescendant) return domDescendant;
|
||||||
|
|
||||||
|
const queue = [...(signals.childrenByParentId.get(triggerBackendNodeId) ?? [])];
|
||||||
|
while (queue.length > 0) {
|
||||||
|
const child = queue.shift() as CapturedNode;
|
||||||
|
const rendered = renderedById.get(child.backendNodeId);
|
||||||
|
if (rendered && isSurfaceAttachableNode(rendered)) return rendered;
|
||||||
|
queue.push(...(signals.childrenByParentId.get(child.backendNodeId) ?? []));
|
||||||
|
}
|
||||||
|
|
||||||
|
let current = signals.capturedByBackendId.get(triggerBackendNodeId);
|
||||||
|
while (current?.parentBackendNodeId !== null && current?.parentBackendNodeId !== undefined) {
|
||||||
|
const parent = renderedById.get(current.parentBackendNodeId);
|
||||||
|
if (parent && isSurfaceAttachableNode(parent)) return parent;
|
||||||
|
current = signals.capturedByBackendId.get(current.parentBackendNodeId);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (triggerPoint) {
|
||||||
|
return findSurfaceNodeByPoint(triggerPoint, nodes, signals);
|
||||||
|
}
|
||||||
|
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
function isSurfaceAttachableNode(node: VomNode): boolean {
|
||||||
|
return isVomReferenceNode(node);
|
||||||
|
}
|
||||||
|
|
||||||
|
function findSurfaceNodeByPoint(
|
||||||
|
point: { x: number; y: number },
|
||||||
|
nodes: VomNode[],
|
||||||
|
signals: VomNodeDomSignals,
|
||||||
|
): VomNode | undefined {
|
||||||
|
let best: { node: VomNode; score: number } | undefined;
|
||||||
|
for (const node of nodes) {
|
||||||
|
if (!isSurfaceAttachableNode(node)) continue;
|
||||||
|
const rect = node.rect ?? signals.capturedByBackendId.get(node.id)?.rect;
|
||||||
|
if (!rect) continue;
|
||||||
|
const contains =
|
||||||
|
point.x >= rect.x &&
|
||||||
|
point.x <= rect.x + rect.w &&
|
||||||
|
point.y >= rect.y &&
|
||||||
|
point.y <= rect.y + rect.h;
|
||||||
|
const centerX = rect.x + rect.w / 2;
|
||||||
|
const centerY = rect.y + rect.h / 2;
|
||||||
|
const distance = Math.hypot(point.x - centerX, point.y - centerY);
|
||||||
|
if (!contains && distance > 40) continue;
|
||||||
|
const score = contains ? distance : distance + 1_000;
|
||||||
|
if (!best || score < best.score) {
|
||||||
|
best = { node, score };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return best?.node;
|
||||||
|
}
|
||||||
|
|
||||||
function axNodeInOverlaySubtree(
|
function axNodeInOverlaySubtree(
|
||||||
axNode: CdpAxNode,
|
axNode: CdpAxNode,
|
||||||
axById: Map<string, CdpAxNode>,
|
axById: Map<string, CdpAxNode>,
|
||||||
@@ -992,6 +1193,8 @@ export interface BuildVomSceneOptions {
|
|||||||
pageUrl?: string;
|
pageUrl?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export type VomFrameDocument = CapturedFrameDocument<CdpAxNode>;
|
||||||
|
|
||||||
function externalHrefHost(
|
function externalHrefHost(
|
||||||
href: string | undefined,
|
href: string | undefined,
|
||||||
pageUrl: string | undefined,
|
pageUrl: string | undefined,
|
||||||
@@ -1085,12 +1288,127 @@ export function buildVomScene(
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function buildFrameVomScene(
|
||||||
|
documents: VomFrameDocument[],
|
||||||
|
captured: CapturedViewModel,
|
||||||
|
options: BuildVomSceneOptions = {},
|
||||||
|
): VomScene {
|
||||||
|
const documentByFrameId = new Map(documents.map((document) => [document.frameId, document]));
|
||||||
|
const rootFrameId =
|
||||||
|
captured.rootFrameId ?? documents.find((document) => !document.parentFrameId)?.frameId;
|
||||||
|
const orderedFrameIds = [
|
||||||
|
...(rootFrameId ? [rootFrameId] : []),
|
||||||
|
...documents.map((document) => document.frameId),
|
||||||
|
].filter((frameId, index, all) => all.indexOf(frameId) === index);
|
||||||
|
|
||||||
|
const localScenes = new Map<string, VomScene>();
|
||||||
|
for (const frameId of orderedFrameIds) {
|
||||||
|
const document = documentByFrameId.get(frameId);
|
||||||
|
if (!document) continue;
|
||||||
|
localScenes.set(
|
||||||
|
frameId,
|
||||||
|
buildVomScene(
|
||||||
|
document.axNodes,
|
||||||
|
{
|
||||||
|
viewport: captured.viewport,
|
||||||
|
nodes: document.domNodes,
|
||||||
|
iframeNodes: new Map(),
|
||||||
|
surfaceProbes: frameId === rootFrameId ? captured.surfaceProbes : [],
|
||||||
|
excludedBackendNodeIds:
|
||||||
|
frameId === rootFrameId ? captured.excludedBackendNodeIds : new Set(),
|
||||||
|
},
|
||||||
|
{ pageUrl: document.url ?? options.pageUrl },
|
||||||
|
),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const sceneIdByFrameAndBackend = new Map<string, number>();
|
||||||
|
let nextSceneId = 1;
|
||||||
|
for (const [frameId, scene] of localScenes) {
|
||||||
|
for (const node of scene.nodes) {
|
||||||
|
sceneIdByFrameAndBackend.set(`${frameId}:${node.id}`, nextSceneId);
|
||||||
|
nextSceneId += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const nodes: VomNode[] = [];
|
||||||
|
const surfaces: CondSurface[] = [];
|
||||||
|
const activeScopeBlocks: ActiveScopeBlock[] = [];
|
||||||
|
|
||||||
|
for (const [frameId, scene] of localScenes) {
|
||||||
|
const document = documentByFrameId.get(frameId);
|
||||||
|
const ownerBackendNodeId = document?.ownerBackendNodeId;
|
||||||
|
const parentFrameId = document?.parentFrameId;
|
||||||
|
const ownerSceneId =
|
||||||
|
ownerBackendNodeId !== undefined && parentFrameId
|
||||||
|
? sceneIdByFrameAndBackend.get(`${parentFrameId}:${ownerBackendNodeId}`)
|
||||||
|
: undefined;
|
||||||
|
if (frameId !== rootFrameId && ownerSceneId === undefined) continue;
|
||||||
|
|
||||||
|
const suppressedRoots = new Set(
|
||||||
|
frameId === rootFrameId
|
||||||
|
? []
|
||||||
|
: scene.nodes
|
||||||
|
.filter(
|
||||||
|
(node) =>
|
||||||
|
node.parentId === null &&
|
||||||
|
["rootwebarea", "webarea"].includes(node.role?.toLowerCase() ?? ""),
|
||||||
|
)
|
||||||
|
.map((node) => node.id),
|
||||||
|
);
|
||||||
|
|
||||||
|
for (const node of scene.nodes) {
|
||||||
|
if (suppressedRoots.has(node.id)) continue;
|
||||||
|
const sceneNodeId = sceneIdByFrameAndBackend.get(`${frameId}:${node.id}`) as number;
|
||||||
|
let parentId =
|
||||||
|
node.parentId !== null
|
||||||
|
? sceneIdByFrameAndBackend.get(`${frameId}:${node.parentId}`)
|
||||||
|
: ownerSceneId;
|
||||||
|
if (node.parentId !== null && suppressedRoots.has(node.parentId)) parentId = ownerSceneId;
|
||||||
|
const remapId = (id: number) => sceneIdByFrameAndBackend.get(`${frameId}:${id}`);
|
||||||
|
nodes.push({
|
||||||
|
...node,
|
||||||
|
id: sceneNodeId,
|
||||||
|
backendNodeId: node.id,
|
||||||
|
frameId,
|
||||||
|
contextScopeId: document?.contextScopeId ?? frameId,
|
||||||
|
parentId: parentId ?? null,
|
||||||
|
...(node.domParentId !== undefined
|
||||||
|
? { domParentId: node.domParentId === null ? null : (remapId(node.domParentId) ?? null) }
|
||||||
|
: {}),
|
||||||
|
...(node.domAncestorIds
|
||||||
|
? { domAncestorIds: node.domAncestorIds.flatMap((id) => remapId(id) ?? []) }
|
||||||
|
: {}),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const surface of scene.surfaces ?? []) {
|
||||||
|
const triggerId = sceneIdByFrameAndBackend.get(`${frameId}:${surface.triggerId}`);
|
||||||
|
if (triggerId !== undefined) surfaces.push({ ...surface, triggerId });
|
||||||
|
}
|
||||||
|
for (const block of scene.activeScopeBlocks ?? []) {
|
||||||
|
const triggerId = sceneIdByFrameAndBackend.get(`${frameId}:${block.triggerId}`);
|
||||||
|
if (triggerId !== undefined) activeScopeBlocks.push({ ...block, triggerId });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
viewport: captured.viewport,
|
||||||
|
nodes,
|
||||||
|
...(rootFrameId ? { rootFrameId } : {}),
|
||||||
|
...(surfaces.length > 0 ? { surfaces } : {}),
|
||||||
|
...(activeScopeBlocks.length > 0 ? { activeScopeBlocks } : {}),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
export interface SnapshotDeps {
|
export interface SnapshotDeps {
|
||||||
cdp: CdpRunner;
|
cdp: CdpRunner;
|
||||||
tabsApi: {
|
tabsApi: {
|
||||||
get(tabId: number): Promise<chrome.tabs.Tab>;
|
get(tabId: number): Promise<chrome.tabs.Tab>;
|
||||||
query(q: chrome.tabs.QueryInfo): Promise<chrome.tabs.Tab[]>;
|
query(q: chrome.tabs.QueryInfo): Promise<chrome.tabs.Tab[]>;
|
||||||
};
|
};
|
||||||
|
conditionalSurfaceProbe?: boolean;
|
||||||
|
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
|
||||||
}
|
}
|
||||||
|
|
||||||
let defaultDeps: SnapshotDeps | null = null;
|
let defaultDeps: SnapshotDeps | null = null;
|
||||||
@@ -1142,7 +1460,9 @@ export async function handleGetHtml(
|
|||||||
manager: SessionManager,
|
manager: SessionManager,
|
||||||
params: GetHtmlParams,
|
params: GetHtmlParams,
|
||||||
deps: SnapshotDeps = getDefaultDeps(),
|
deps: SnapshotDeps = getDefaultDeps(),
|
||||||
|
signal?: AbortSignal,
|
||||||
): Promise<GetHtmlResult | RpcError> {
|
): Promise<GetHtmlResult | RpcError> {
|
||||||
|
if (signal?.aborted) return cancelled("get_html");
|
||||||
const ctxOrErr = lookupSession(manager, params, "get_html");
|
const ctxOrErr = lookupSession(manager, params, "get_html");
|
||||||
if (isRpcError(ctxOrErr)) return ctxOrErr;
|
if (isRpcError(ctxOrErr)) return ctxOrErr;
|
||||||
const ctx = ctxOrErr;
|
const ctx = ctxOrErr;
|
||||||
@@ -1154,21 +1474,31 @@ export async function handleGetHtml(
|
|||||||
"get_html",
|
"get_html",
|
||||||
);
|
);
|
||||||
if (isRpcError(target)) return target;
|
if (isRpcError(target)) return target;
|
||||||
|
if (signal?.aborted) return cancelled("get_html");
|
||||||
const dialogCursor = markDialogCursor(deps.cdp, target.tabId);
|
const dialogCursor = markDialogCursor(deps.cdp, target.tabId);
|
||||||
|
|
||||||
const maxBytes =
|
const maxBytes =
|
||||||
params.max_bytes && params.max_bytes > 0 ? params.max_bytes : DEFAULT_GET_HTML_MAX_BYTES;
|
params.max_bytes && params.max_bytes > 0 ? params.max_bytes : DEFAULT_GET_HTML_MAX_BYTES;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
|
throwIfAborted(signal, "get_html");
|
||||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||||
await deps.cdp.ensureAttachedToUrl?.(target.tabId, target.url);
|
await deps.cdp.ensureAttachedToUrl?.(target.tabId, target.url);
|
||||||
|
throwIfAborted(signal, "get_html");
|
||||||
let html: string;
|
let html: string;
|
||||||
if (params.ref) {
|
if (params.ref) {
|
||||||
const resolved = resolveSnapshotRef(ctx, params.ref, target.tabId);
|
const resolved = resolveSnapshotRef(ctx, params.ref, target.tabId);
|
||||||
if (isRpcError(resolved)) return resolved;
|
if (isRpcError(resolved)) return resolved;
|
||||||
const resp = await deps.cdp.send<{ outerHTML?: string }>(target.tabId, "DOM.getOuterHTML", {
|
const resp = await sendToCdpTarget<{ outerHTML?: string }>(
|
||||||
backendNodeId: resolved.backendNodeId,
|
deps.cdp,
|
||||||
});
|
{
|
||||||
|
tabId: target.tabId,
|
||||||
|
...(resolved.cdpSessionId ? { sessionId: resolved.cdpSessionId } : {}),
|
||||||
|
},
|
||||||
|
"DOM.getOuterHTML",
|
||||||
|
{ backendNodeId: resolved.backendNodeId },
|
||||||
|
);
|
||||||
|
throwIfAborted(signal, "get_html");
|
||||||
html = resp.outerHTML ?? "";
|
html = resp.outerHTML ?? "";
|
||||||
} else {
|
} else {
|
||||||
const doc = await deps.cdp.send<{ root?: { nodeId?: number } }>(
|
const doc = await deps.cdp.send<{ root?: { nodeId?: number } }>(
|
||||||
@@ -1176,6 +1506,7 @@ export async function handleGetHtml(
|
|||||||
"DOM.getDocument",
|
"DOM.getDocument",
|
||||||
{ depth: 0 },
|
{ depth: 0 },
|
||||||
);
|
);
|
||||||
|
throwIfAborted(signal, "get_html");
|
||||||
const nodeId = doc.root?.nodeId;
|
const nodeId = doc.root?.nodeId;
|
||||||
if (typeof nodeId !== "number") {
|
if (typeof nodeId !== "number") {
|
||||||
return {
|
return {
|
||||||
@@ -1186,6 +1517,7 @@ export async function handleGetHtml(
|
|||||||
const resp = await deps.cdp.send<{ outerHTML?: string }>(target.tabId, "DOM.getOuterHTML", {
|
const resp = await deps.cdp.send<{ outerHTML?: string }>(target.tabId, "DOM.getOuterHTML", {
|
||||||
nodeId,
|
nodeId,
|
||||||
});
|
});
|
||||||
|
throwIfAborted(signal, "get_html");
|
||||||
html = resp.outerHTML ?? "";
|
html = resp.outerHTML ?? "";
|
||||||
}
|
}
|
||||||
const originalBytes = utf8ByteLength(html);
|
const originalBytes = utf8ByteLength(html);
|
||||||
@@ -1197,6 +1529,7 @@ export async function handleGetHtml(
|
|||||||
tab_id: target.tabId,
|
tab_id: target.tabId,
|
||||||
});
|
});
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
|
if (isAbortError(err)) return cancelled("get_html");
|
||||||
return {
|
return {
|
||||||
code: "cdp_failed",
|
code: "cdp_failed",
|
||||||
message: err instanceof Error ? err.message : String(err),
|
message: err instanceof Error ? err.message : String(err),
|
||||||
@@ -1216,34 +1549,81 @@ interface LayoutMetricsViewportReply {
|
|||||||
async function fallbackCapturedViewModel(
|
async function fallbackCapturedViewModel(
|
||||||
cdp: CdpRunner,
|
cdp: CdpRunner,
|
||||||
tabId: number,
|
tabId: number,
|
||||||
|
signal?: AbortSignal,
|
||||||
): Promise<CapturedViewModel> {
|
): Promise<CapturedViewModel> {
|
||||||
|
throwIfAborted(signal, "observation");
|
||||||
let viewport = { width: 0, height: 0 };
|
let viewport = { width: 0, height: 0 };
|
||||||
try {
|
try {
|
||||||
const metrics = await cdp.send<LayoutMetricsViewportReply>(tabId, "Page.getLayoutMetrics", {});
|
const metrics = await cdp.send<LayoutMetricsViewportReply>(tabId, "Page.getLayoutMetrics", {});
|
||||||
const vpSrc = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
|
throwIfAborted(signal, "observation");
|
||||||
|
const source = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
|
||||||
viewport = {
|
viewport = {
|
||||||
width: vpSrc.clientWidth ?? 0,
|
width: source.clientWidth ?? 0,
|
||||||
height: vpSrc.clientHeight ?? 0,
|
height: source.clientHeight ?? 0,
|
||||||
};
|
};
|
||||||
} catch {
|
} catch (error) {
|
||||||
// viewport stays zero-sized
|
if (isAbortError(error)) throw error;
|
||||||
}
|
}
|
||||||
const excludedBackendNodeIds = await collectOverlayExcludedBackendIds(cdp, tabId);
|
const excludedBackendNodeIds = await collectOverlayExcludedBackendIds(cdp, tabId, signal);
|
||||||
|
throwIfAborted(signal, "observation");
|
||||||
return { ...emptyCapturedViewModel(viewport), excludedBackendNodeIds };
|
return { ...emptyCapturedViewModel(viewport), excludedBackendNodeIds };
|
||||||
}
|
}
|
||||||
|
|
||||||
async function captureForVom(
|
async function captureForVom(
|
||||||
cdp: CdpRunner,
|
cdp: CdpRunner,
|
||||||
tabId: number,
|
tabId: number,
|
||||||
conditionalSurfaceProbe: boolean,
|
options: CaptureVomObservationOptions,
|
||||||
): Promise<CapturedViewModel> {
|
): Promise<CapturedViewModel> {
|
||||||
try {
|
try {
|
||||||
return await captureViewModel(cdp, tabId, { conditionalSurfaceProbe });
|
return await captureViewModel(cdp, tabId, {
|
||||||
} catch {
|
conditionalSurfaceProbe: options.conditionalSurfaceProbe ?? false,
|
||||||
return fallbackCapturedViewModel(cdp, tabId);
|
hoverProbeBypassOverlay: options.hoverProbeBypassOverlay,
|
||||||
|
signal: options.signal,
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
if (isAbortError(error)) throw error;
|
||||||
|
return fallbackCapturedViewModel(cdp, tabId, options.signal);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface CaptureVomObservationOptions extends VomOptions {
|
||||||
|
conditionalSurfaceProbe?: boolean;
|
||||||
|
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
|
||||||
|
signal?: AbortSignal;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function captureVomObservation(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
tabId: number,
|
||||||
|
url: string | undefined,
|
||||||
|
options: CaptureVomObservationOptions = {},
|
||||||
|
): Promise<CaptureVomObservationResult> {
|
||||||
|
throwIfAborted(options.signal, "observation");
|
||||||
|
await cdp.ensureAttachedToUrl?.(tabId, url);
|
||||||
|
throwIfAborted(options.signal, "observation");
|
||||||
|
const captured = await captureForVom(cdp, tabId, options);
|
||||||
|
throwIfAborted(options.signal, "observation");
|
||||||
|
const documents = await captureFrameData<CdpAxNode>(cdp, tabId, captured, options.signal);
|
||||||
|
throwIfAborted(options.signal, "observation");
|
||||||
|
const scene =
|
||||||
|
captured.rootFrameId && captured.frameNodes?.size
|
||||||
|
? buildFrameVomScene(documents, captured, { pageUrl: url })
|
||||||
|
: buildVomScene(documents[0]?.axNodes ?? [], captured, { pageUrl: url });
|
||||||
|
const rendered = renderVom(scene, {
|
||||||
|
maxDepth: options.maxDepth,
|
||||||
|
maxTokens: options.maxTokens,
|
||||||
|
redactValues: options.redactValues,
|
||||||
|
activeRegionPolicy: options.activeRegionPolicy,
|
||||||
|
});
|
||||||
|
throwIfAborted(options.signal, "observation");
|
||||||
|
return projectRecordSafeObservation({
|
||||||
|
rootFrameId: captured.rootFrameId ?? documents[0]?.frameId ?? "root",
|
||||||
|
frameDocuments: documents,
|
||||||
|
rendered,
|
||||||
|
surfaceProbes: captured.surfaceProbes,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
async function handleVomObservation(
|
async function handleVomObservation(
|
||||||
manager: SessionManager,
|
manager: SessionManager,
|
||||||
params: SnapshotParams | ObserveParams,
|
params: SnapshotParams | ObserveParams,
|
||||||
@@ -1251,7 +1631,9 @@ async function handleVomObservation(
|
|||||||
effect: ToolEffect,
|
effect: ToolEffect,
|
||||||
conditionalSurfaceProbe: boolean,
|
conditionalSurfaceProbe: boolean,
|
||||||
deps: SnapshotDeps = getDefaultDeps(),
|
deps: SnapshotDeps = getDefaultDeps(),
|
||||||
|
signal?: AbortSignal,
|
||||||
): Promise<SnapshotResult | ObserveResult | RpcError> {
|
): Promise<SnapshotResult | ObserveResult | RpcError> {
|
||||||
|
if (signal?.aborted) return cancelled(toolName);
|
||||||
const ctxOrErr = lookupSession(manager, params, toolName);
|
const ctxOrErr = lookupSession(manager, params, toolName);
|
||||||
if (isRpcError(ctxOrErr)) return ctxOrErr;
|
if (isRpcError(ctxOrErr)) return ctxOrErr;
|
||||||
const ctx = ctxOrErr;
|
const ctx = ctxOrErr;
|
||||||
@@ -1263,39 +1645,63 @@ async function handleVomObservation(
|
|||||||
toolName,
|
toolName,
|
||||||
);
|
);
|
||||||
if (isRpcError(target)) return target;
|
if (isRpcError(target)) return target;
|
||||||
|
if (signal?.aborted) return cancelled(toolName);
|
||||||
const denied = enforceToolTargetScope(ctx, target, effect, toolName);
|
const denied = enforceToolTargetScope(ctx, target, effect, toolName);
|
||||||
if (denied) return denied;
|
if (denied) return denied;
|
||||||
const dialogCursor = markDialogCursor(deps.cdp, target.tabId);
|
const dialogCursor = markDialogCursor(deps.cdp, target.tabId);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
|
throwIfAborted(signal, toolName);
|
||||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||||
await deps.cdp.ensureAttachedToUrl?.(target.tabId, target.url);
|
const effectiveConditionalSurfaceProbe =
|
||||||
await deps.cdp.send<unknown>(target.tabId, "Accessibility.enable", {});
|
deps.conditionalSurfaceProbe ?? conditionalSurfaceProbe;
|
||||||
const result = await deps.cdp.send<{ nodes: CdpAxNode[] }>(
|
const observation = await captureVomObservation(deps.cdp, target.tabId, target.url, {
|
||||||
target.tabId,
|
|
||||||
"Accessibility.getFullAXTree",
|
|
||||||
{},
|
|
||||||
);
|
|
||||||
const axNodes = result.nodes ?? [];
|
|
||||||
const captured = await captureForVom(deps.cdp, target.tabId, conditionalSurfaceProbe);
|
|
||||||
const scene = buildVomScene(axNodes, captured, { pageUrl: target.url });
|
|
||||||
const rendered = renderVom(scene, {
|
|
||||||
maxDepth: params.max_depth,
|
maxDepth: params.max_depth,
|
||||||
maxTokens: params.max_tokens,
|
maxTokens: params.max_tokens,
|
||||||
activeRegionPolicy: true,
|
activeRegionPolicy: true,
|
||||||
|
conditionalSurfaceProbe: effectiveConditionalSurfaceProbe,
|
||||||
|
hoverProbeBypassOverlay: deps.hoverProbeBypassOverlay,
|
||||||
|
signal,
|
||||||
});
|
});
|
||||||
|
throwIfAborted(signal, toolName);
|
||||||
|
const targetByFrameId = new Map(
|
||||||
|
observation.frames.map((frame) => [frame.frameId, frame.target]),
|
||||||
|
);
|
||||||
ctx.refStore.replace(
|
ctx.refStore.replace(
|
||||||
rendered.refs.map(
|
observation.refs.map((ref) => {
|
||||||
(r) => [r.ref, { backendNodeId: r.backendNodeId, tabId: target.tabId }] as const,
|
const refTarget = ref.frameId ? targetByFrameId.get(ref.frameId) : undefined;
|
||||||
),
|
return [
|
||||||
|
ref.ref,
|
||||||
|
{
|
||||||
|
backendNodeId: ref.backendNodeId,
|
||||||
|
tabId: target.tabId,
|
||||||
|
...(ref.frameId ? { frameId: ref.frameId } : {}),
|
||||||
|
...(refTarget?.sessionId ? { cdpSessionId: refTarget.sessionId } : {}),
|
||||||
|
},
|
||||||
|
] as const;
|
||||||
|
}),
|
||||||
);
|
);
|
||||||
return attachDialogs(deps.cdp, target.tabId, dialogCursor, {
|
return attachDialogs(deps.cdp, target.tabId, dialogCursor, {
|
||||||
text: rendered.text,
|
text: observation.text,
|
||||||
ref_count: rendered.refs.length,
|
ref_count: observation.refs.length,
|
||||||
tab_id: target.tabId,
|
tab_id: target.tabId,
|
||||||
truncated: rendered.truncated,
|
truncated: observation.truncated,
|
||||||
|
...(toolName === "observe" && (params as ObserveParams).debug_surfaces
|
||||||
|
? {
|
||||||
|
debug: {
|
||||||
|
surface_probes: (observation.surfaceProbes ?? []).map((probe) => ({
|
||||||
|
trigger_backend_node_id: probe.triggerBackendNodeId,
|
||||||
|
...(probe.triggerPoint ? { trigger_point: probe.triggerPoint } : {}),
|
||||||
|
trigger_action: probe.triggerAction,
|
||||||
|
sub_items: probe.subItems,
|
||||||
|
...(probe.confidence ? { confidence: probe.confidence } : {}),
|
||||||
|
})),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
: {}),
|
||||||
});
|
});
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
|
if (isAbortError(err)) return cancelled(toolName);
|
||||||
return {
|
return {
|
||||||
code: "cdp_failed",
|
code: "cdp_failed",
|
||||||
message: err instanceof Error ? err.message : String(err),
|
message: err instanceof Error ? err.message : String(err),
|
||||||
@@ -1307,14 +1713,16 @@ export async function handleSnapshot(
|
|||||||
manager: SessionManager,
|
manager: SessionManager,
|
||||||
params: SnapshotParams,
|
params: SnapshotParams,
|
||||||
deps: SnapshotDeps = getDefaultDeps(),
|
deps: SnapshotDeps = getDefaultDeps(),
|
||||||
|
signal?: AbortSignal,
|
||||||
): Promise<SnapshotResult | RpcError> {
|
): Promise<SnapshotResult | RpcError> {
|
||||||
return handleVomObservation(manager, params, "snapshot", "passive_read", false, deps);
|
return handleVomObservation(manager, params, "snapshot", "passive_read", false, deps, signal);
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function handleObserve(
|
export async function handleObserve(
|
||||||
manager: SessionManager,
|
manager: SessionManager,
|
||||||
params: ObserveParams,
|
params: ObserveParams,
|
||||||
deps: SnapshotDeps = getDefaultDeps(),
|
deps: SnapshotDeps = getDefaultDeps(),
|
||||||
|
signal?: AbortSignal,
|
||||||
): Promise<ObserveResult | RpcError> {
|
): Promise<ObserveResult | RpcError> {
|
||||||
return handleVomObservation(manager, params, "observe", "transient_input", true, deps);
|
return handleVomObservation(manager, params, "observe", "transient_input", true, deps, signal);
|
||||||
}
|
}
|
||||||
@@ -17,11 +17,11 @@ import {
|
|||||||
type RecordStartMessage,
|
type RecordStartMessage,
|
||||||
type RecordStopMessage,
|
type RecordStopMessage,
|
||||||
} from "@/lib/record-bridge";
|
} from "@/lib/record-bridge";
|
||||||
import { appendRecordedPayload, observeRecordedNavigation } from "@/lib/recording-step-buffer";
|
import { appendRecordedPayload, observeRecordedNavigation } from "@/lib/recording/step-buffer";
|
||||||
import { reduceTraceSteps, resolveTraceStartUrl } from "@/lib/trace-reducer";
|
import { buildTraceV2 } from "@/lib/recording/trace-reducer-v2";
|
||||||
|
import type { RecordingDraftStep } from "@/lib/recording/types";
|
||||||
import type { SessionManager } from "@/session-manager/manager";
|
import type { SessionManager } from "@/session-manager/manager";
|
||||||
import type {
|
import type {
|
||||||
DraftTraceStep,
|
|
||||||
RecordAwaitParams,
|
RecordAwaitParams,
|
||||||
RecordAwaitResult,
|
RecordAwaitResult,
|
||||||
RecordStartParams,
|
RecordStartParams,
|
||||||
@@ -29,7 +29,7 @@ import type {
|
|||||||
RecordStopParams,
|
RecordStopParams,
|
||||||
RecordStopResult,
|
RecordStopResult,
|
||||||
RpcError,
|
RpcError,
|
||||||
Trace,
|
TraceV2,
|
||||||
} from "@/transport/types";
|
} from "@/transport/types";
|
||||||
import { handleNavigate } from "./navigation";
|
import { handleNavigate } from "./navigation";
|
||||||
import {
|
import {
|
||||||
@@ -47,11 +47,11 @@ interface ActiveRecording {
|
|||||||
agentWindowId: number;
|
agentWindowId: number;
|
||||||
startUrl?: string;
|
startUrl?: string;
|
||||||
purpose?: string;
|
purpose?: string;
|
||||||
steps: DraftTraceStep[];
|
steps: RecordingDraftStep[];
|
||||||
startedAt: string;
|
startedAt: string;
|
||||||
startedAtMs: number;
|
startedAtMs: number;
|
||||||
finishPromise: Promise<Trace>;
|
finishPromise: Promise<TraceV2>;
|
||||||
resolveFinish: (trace: Trace) => void;
|
resolveFinish: (trace: TraceV2) => void;
|
||||||
rejectFinish: (err: Error) => void;
|
rejectFinish: (err: Error) => void;
|
||||||
settled: boolean;
|
settled: boolean;
|
||||||
finishing: boolean;
|
finishing: boolean;
|
||||||
@@ -154,17 +154,13 @@ async function sendRecordStartWithAck(
|
|||||||
throw lastError ?? new Error("failed to start recording in content script");
|
throw lastError ?? new Error("failed to start recording in content script");
|
||||||
}
|
}
|
||||||
|
|
||||||
function buildTrace(recording: ActiveRecording): Trace {
|
function buildTrace(recording: ActiveRecording): TraceV2 {
|
||||||
const { pages, steps } = reduceTraceSteps(recording.steps, recording.startUrl);
|
return buildTraceV2({
|
||||||
const startUrl = resolveTraceStartUrl(recording.steps, recording.startUrl, pages);
|
steps: recording.steps,
|
||||||
return {
|
startedAt: recording.startedAt,
|
||||||
|
...(recording.startUrl ? { startUrl: recording.startUrl } : {}),
|
||||||
...(recording.purpose ? { purpose: recording.purpose } : {}),
|
...(recording.purpose ? { purpose: recording.purpose } : {}),
|
||||||
recorded_at: new Date().toISOString(),
|
});
|
||||||
started_at: recording.startedAt,
|
|
||||||
entry: { start_url: startUrl },
|
|
||||||
pages,
|
|
||||||
steps,
|
|
||||||
};
|
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface RecordDeps {
|
export interface RecordDeps {
|
||||||
@@ -532,7 +528,7 @@ async function finishRecordingByRequest(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async function finishRecording(sessionId: string, deps: RecordDeps): Promise<Trace | null> {
|
async function finishRecording(sessionId: string, deps: RecordDeps): Promise<TraceV2 | null> {
|
||||||
const recording = recordings.get(sessionId);
|
const recording = recordings.get(sessionId);
|
||||||
if (!recording || recording.settled || recording.finishing) return null;
|
if (!recording || recording.settled || recording.finishing) return null;
|
||||||
recording.finishing = true;
|
recording.finishing = true;
|
||||||
@@ -574,9 +570,9 @@ export async function handleRecordStart(
|
|||||||
// on the destination page can RECORD_QUERY → rearm → show RecordOverlay
|
// on the destination page can RECORD_QUERY → rearm → show RecordOverlay
|
||||||
// instead of flashing ControlOverlay ("Agent 正在控制").
|
// instead of flashing ControlOverlay ("Agent 正在控制").
|
||||||
const requestId = makeRequestId(target.tabId);
|
const requestId = makeRequestId(target.tabId);
|
||||||
let resolveFinish!: (trace: Trace) => void;
|
let resolveFinish!: (trace: TraceV2) => void;
|
||||||
let rejectFinish!: (err: Error) => void;
|
let rejectFinish!: (err: Error) => void;
|
||||||
const finishPromise = new Promise<Trace>((resolve, reject) => {
|
const finishPromise = new Promise<TraceV2>((resolve, reject) => {
|
||||||
resolveFinish = resolve;
|
resolveFinish = resolve;
|
||||||
rejectFinish = reject;
|
rejectFinish = reject;
|
||||||
});
|
});
|
||||||
@@ -785,9 +781,9 @@ export async function handleRecordAwait(
|
|||||||
return { code: "cancelled", message: "record_await aborted" };
|
return { code: "cancelled", message: "record_await aborted" };
|
||||||
}
|
}
|
||||||
|
|
||||||
const outcome = await new Promise<{ trace: Trace } | { error: RpcError }>((resolve) => {
|
const outcome = await new Promise<{ trace: TraceV2 } | { error: RpcError }>((resolve) => {
|
||||||
let settled = false;
|
let settled = false;
|
||||||
const finish = (result: { trace: Trace } | { error: RpcError }) => {
|
const finish = (result: { trace: TraceV2 } | { error: RpcError }) => {
|
||||||
if (settled) return;
|
if (settled) return;
|
||||||
settled = true;
|
settled = true;
|
||||||
if (timer) clearTimeout(timer);
|
if (timer) clearTimeout(timer);
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
import type { SessionManager } from "@/session-manager/manager";
|
import { type SessionManager, SessionStartCleanupError } from "@/session-manager/manager";
|
||||||
import type { RpcError } from "@/transport/types";
|
import type { RpcError } from "@/transport/types";
|
||||||
|
import { rpcError } from "./errors";
|
||||||
import { clearRecordingForSession } from "./record";
|
import { clearRecordingForSession } from "./record";
|
||||||
import { isRpcError } from "./shared";
|
import { isRpcError } from "./shared";
|
||||||
import { returnBorrowedTab, type TabManagementDeps } from "./tabs";
|
import { returnBorrowedTab, type TabManagementDeps } from "./tabs";
|
||||||
@@ -50,12 +51,18 @@ export interface SessionStartParams {
|
|||||||
width?: number;
|
width?: number;
|
||||||
/** Optional Agent Window outer height in CSS pixels (100..=7680). */
|
/** Optional Agent Window outer height in CSS pixels (100..=7680). */
|
||||||
height?: number;
|
height?: number;
|
||||||
|
/** Defaults to true so existing clients preserve visible Agent Windows. */
|
||||||
|
focused?: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface SessionStartResult {
|
export interface SessionStartResult {
|
||||||
agent_window_id?: number;
|
agent_window_id?: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface SessionStartDeps {
|
||||||
|
signal?: AbortSignal;
|
||||||
|
}
|
||||||
|
|
||||||
export interface SessionStopParams {
|
export interface SessionStopParams {
|
||||||
session_id: string;
|
session_id: string;
|
||||||
}
|
}
|
||||||
@@ -69,6 +76,7 @@ export interface SessionStopResult {
|
|||||||
}
|
}
|
||||||
|
|
||||||
export interface SessionStopDeps {
|
export interface SessionStopDeps {
|
||||||
|
signal?: AbortSignal;
|
||||||
cdp?: {
|
cdp?: {
|
||||||
detachSession(sessionId: string): Promise<void>;
|
detachSession(sessionId: string): Promise<void>;
|
||||||
};
|
};
|
||||||
@@ -89,6 +97,7 @@ export interface SessionStopDeps {
|
|||||||
export async function handleSessionStart(
|
export async function handleSessionStart(
|
||||||
manager: SessionManager,
|
manager: SessionManager,
|
||||||
params: SessionStartParams,
|
params: SessionStartParams,
|
||||||
|
deps: SessionStartDeps = {},
|
||||||
): Promise<SessionStartResult | RpcError> {
|
): Promise<SessionStartResult | RpcError> {
|
||||||
if (!params?.session_id) {
|
if (!params?.session_id) {
|
||||||
return {
|
return {
|
||||||
@@ -99,9 +108,26 @@ export async function handleSessionStart(
|
|||||||
const sizeOrErr = validateWindowSize(params.width, params.height);
|
const sizeOrErr = validateWindowSize(params.width, params.height);
|
||||||
if (isRpcError(sizeOrErr)) return sizeOrErr;
|
if (isRpcError(sizeOrErr)) return sizeOrErr;
|
||||||
try {
|
try {
|
||||||
const ctx = await manager.start(params.session_id, sizeOrErr);
|
const ctx = await manager.start(params.session_id, {
|
||||||
|
size: sizeOrErr,
|
||||||
|
focused: params.focused,
|
||||||
|
signal: deps.signal,
|
||||||
|
});
|
||||||
return { agent_window_id: ctx.agentWindowId };
|
return { agent_window_id: ctx.agentWindowId };
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
|
if (err instanceof SessionStartCleanupError) {
|
||||||
|
return rpcError("protocol_error", "cleanup_failed", err.message, {
|
||||||
|
resource_type: "agent_window",
|
||||||
|
resource_id: err.windowId,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if (
|
||||||
|
typeof err === "object" &&
|
||||||
|
err !== null &&
|
||||||
|
(err as { name?: string }).name === "AbortError"
|
||||||
|
) {
|
||||||
|
return { code: "cancelled", message: "session_start aborted" };
|
||||||
|
}
|
||||||
// chrome.windows.create / SessionManager failures are not CDP
|
// chrome.windows.create / SessionManager failures are not CDP
|
||||||
// failures (§4.5 reserves cdp_failed for raw CDP errors). Surface
|
// failures (§4.5 reserves cdp_failed for raw CDP errors). Surface
|
||||||
// them as protocol_error so the CLI maps to the right exit code
|
// them as protocol_error so the CLI maps to the right exit code
|
||||||
@@ -152,6 +178,9 @@ export async function handleSessionStop(
|
|||||||
message: `session ${params.session_id} unknown`,
|
message: `session ${params.session_id} unknown`,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
if (deps.signal?.aborted) {
|
||||||
|
return { code: "cancelled", message: "session_stop aborted before teardown" };
|
||||||
|
}
|
||||||
|
|
||||||
// Step 1: auto-return borrowed tabs. Iterate over a snapshot of the
|
// Step 1: auto-return borrowed tabs. Iterate over a snapshot of the
|
||||||
// ids so deletions during iteration do not break the Map iterator.
|
// ids so deletions during iteration do not break the Map iterator.
|
||||||
@@ -162,6 +191,7 @@ export async function handleSessionStop(
|
|||||||
try {
|
try {
|
||||||
const tabManagement = {
|
const tabManagement = {
|
||||||
...(deps.tabManagement ?? {}),
|
...(deps.tabManagement ?? {}),
|
||||||
|
signal: deps.signal,
|
||||||
isAgentWindowId:
|
isAgentWindowId:
|
||||||
deps.tabManagement?.isAgentWindowId ??
|
deps.tabManagement?.isAgentWindowId ??
|
||||||
((windowId: number) => manager.findByWindowId(windowId) !== null),
|
((windowId: number) => manager.findByWindowId(windowId) !== null),
|
||||||
@@ -189,6 +219,17 @@ export async function handleSessionStop(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (deps.signal?.aborted) {
|
||||||
|
const nonCancelFailures = returnFailures?.filter((failure) => failure.code !== "cancelled");
|
||||||
|
if (nonCancelFailures && nonCancelFailures.length > 0) {
|
||||||
|
return {
|
||||||
|
...(returnedTabIds.length > 0 ? { returned_tab_ids: returnedTabIds } : {}),
|
||||||
|
return_failures: nonCancelFailures,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
return { code: "cancelled", message: "session_stop aborted during tab return" };
|
||||||
|
}
|
||||||
|
|
||||||
const result: SessionStopResult = {};
|
const result: SessionStopResult = {};
|
||||||
if (returnedTabIds.length > 0) result.returned_tab_ids = returnedTabIds;
|
if (returnedTabIds.length > 0) result.returned_tab_ids = returnedTabIds;
|
||||||
if (returnFailures && returnFailures.length > 0) {
|
if (returnFailures && returnFailures.length > 0) {
|
||||||
@@ -208,6 +249,10 @@ export async function handleSessionStop(
|
|||||||
// Step 3: detach CDP sessions this session opened (no-op if none).
|
// Step 3: detach CDP sessions this session opened (no-op if none).
|
||||||
await deps.cdp?.detachSession(params.session_id);
|
await deps.cdp?.detachSession(params.session_id);
|
||||||
|
|
||||||
|
if (deps.signal?.aborted) {
|
||||||
|
return { code: "cancelled", message: "session_stop aborted before window close" };
|
||||||
|
}
|
||||||
|
|
||||||
// Step 4: close the Agent Window and drop the context.
|
// Step 4: close the Agent Window and drop the context.
|
||||||
await manager.stop(params.session_id);
|
await manager.stop(params.session_id);
|
||||||
|
|
||||||
|
|||||||
@@ -6,7 +6,8 @@
|
|||||||
// exactly the same sandbox + visibility rules as the M6 observation
|
// exactly the same sandbox + visibility rules as the M6 observation
|
||||||
// handlers (review parity).
|
// handlers (review parity).
|
||||||
|
|
||||||
import type { DialogCursor } from "@/browser-driver/chromium-cdp";
|
import type { CdpDebuggee, DialogCursor } from "@/browser-driver/chromium-cdp";
|
||||||
|
import type { CdpFrameGraph, CdpTarget } from "@/browser-driver/frame-graph";
|
||||||
import type { SessionContext, SessionManager } from "@/session-manager/manager";
|
import type { SessionContext, SessionManager } from "@/session-manager/manager";
|
||||||
import { normaliseRef } from "@/session-manager/ref-store";
|
import { normaliseRef } from "@/session-manager/ref-store";
|
||||||
import type { ConsoleResult, JavaScriptDialogInfo, RpcError } from "@/transport/types";
|
import type { ConsoleResult, JavaScriptDialogInfo, RpcError } from "@/transport/types";
|
||||||
@@ -51,9 +52,11 @@ export type { DialogCursor };
|
|||||||
*/
|
*/
|
||||||
export interface CdpRunner {
|
export interface CdpRunner {
|
||||||
send<T = unknown>(tabId: number, method: string, params?: object): Promise<T>;
|
send<T = unknown>(tabId: number, method: string, params?: object): Promise<T>;
|
||||||
|
sendToTarget?<T = unknown>(target: CdpTarget, method: string, params?: object): Promise<T>;
|
||||||
|
getFrameGraph?(tabId: number): Promise<CdpFrameGraph>;
|
||||||
ensureAttachedToUrl?(tabId: number, expectedUrl: string | undefined): Promise<void>;
|
ensureAttachedToUrl?(tabId: number, expectedUrl: string | undefined): Promise<void>;
|
||||||
trackSessionTab?(sessionId: string, tabId: number): void;
|
trackSessionTab?(sessionId: string, tabId: number): void;
|
||||||
onEvent?(handler: (source: chrome.debugger.Debuggee, method: string, params: unknown) => void): {
|
onEvent?(handler: (source: CdpDebuggee, method: string, params: unknown) => void): {
|
||||||
dispose(): void;
|
dispose(): void;
|
||||||
};
|
};
|
||||||
dialogCursor?(tabId: number): DialogCursor;
|
dialogCursor?(tabId: number): DialogCursor;
|
||||||
@@ -68,6 +71,25 @@ export interface CdpRunner {
|
|||||||
): ConsoleResult;
|
): ConsoleResult;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function sendToCdpTarget<T = unknown>(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
target: CdpTarget,
|
||||||
|
method: string,
|
||||||
|
params?: object,
|
||||||
|
): Promise<T> {
|
||||||
|
if (target.sessionId && cdp.sendToTarget) {
|
||||||
|
return cdp.sendToTarget<T>(target, method, params);
|
||||||
|
}
|
||||||
|
return cdp.send<T>(target.tabId, method, params);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function cdpRunnerForTarget(cdp: CdpRunner, target: CdpTarget): CdpRunner {
|
||||||
|
return {
|
||||||
|
send: <T = unknown>(_tabId: number, method: string, params?: object) =>
|
||||||
|
sendToCdpTarget<T>(cdp, target, method, params),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
export interface BufferedReadBounds {
|
export interface BufferedReadBounds {
|
||||||
since: number | undefined;
|
since: number | undefined;
|
||||||
limit: number;
|
limit: number;
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
// Snapshot ref resolution — normalise `@eN` / `eN`, look up
|
// Snapshot ref resolution — normalise `@eN` / `eN`, look up
|
||||||
// `backendNodeId` via the session RefStore, and build stable
|
// compound CDP node identity via the session RefStore, and build stable
|
||||||
// `ref_not_found` errors for hard-failure tool paths.
|
// `ref_not_found` errors for hard-failure tool paths.
|
||||||
|
|
||||||
import type { SessionContext } from "@/session-manager/manager";
|
import type { SessionContext } from "@/session-manager/manager";
|
||||||
@@ -10,6 +10,13 @@ import { rpcError } from "./errors";
|
|||||||
export interface SnapshotRefLookup {
|
export interface SnapshotRefLookup {
|
||||||
backendNodeId: number;
|
backendNodeId: number;
|
||||||
refKey: string;
|
refKey: string;
|
||||||
|
frameId?: string;
|
||||||
|
cdpSessionId?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
function refEntryForTab(ctx: SessionContext, refKey: string, tabId: number) {
|
||||||
|
const entry = ctx.refStore.resolveEntry(refKey);
|
||||||
|
return entry?.tabId === tabId ? entry : null;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -23,9 +30,14 @@ export function lookupSnapshotRef(
|
|||||||
tabId: number,
|
tabId: number,
|
||||||
): SnapshotRefLookup | null {
|
): SnapshotRefLookup | null {
|
||||||
const refKey = normaliseRef(ref);
|
const refKey = normaliseRef(ref);
|
||||||
const backendNodeId = ctx.refStore.resolve(refKey, { tabId });
|
const entry = refEntryForTab(ctx, refKey, tabId);
|
||||||
if (backendNodeId === null) return null;
|
if (!entry) return null;
|
||||||
return { backendNodeId, refKey };
|
return {
|
||||||
|
backendNodeId: entry.backendNodeId,
|
||||||
|
refKey,
|
||||||
|
...(entry.frameId ? { frameId: entry.frameId } : {}),
|
||||||
|
...(entry.cdpSessionId ? { cdpSessionId: entry.cdpSessionId } : {}),
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -37,7 +49,7 @@ export function resolveSnapshotRef(
|
|||||||
ctx: SessionContext,
|
ctx: SessionContext,
|
||||||
ref: string,
|
ref: string,
|
||||||
tabId: number,
|
tabId: number,
|
||||||
): { backendNodeId: number; refKey: string } | RpcError {
|
): SnapshotRefLookup | RpcError {
|
||||||
const looked = lookupSnapshotRef(ctx, ref, tabId);
|
const looked = lookupSnapshotRef(ctx, ref, tabId);
|
||||||
if (looked === null) {
|
if (looked === null) {
|
||||||
return rpcError(
|
return rpcError(
|
||||||
|
|||||||
@@ -157,6 +157,7 @@ export interface ChromeWindowsApi {
|
|||||||
get(windowId: number): Promise<chrome.windows.Window>;
|
get(windowId: number): Promise<chrome.windows.Window>;
|
||||||
getLastFocused(filters?: chrome.windows.QueryOptions): Promise<chrome.windows.Window>;
|
getLastFocused(filters?: chrome.windows.QueryOptions): Promise<chrome.windows.Window>;
|
||||||
create(props: chrome.windows.CreateData): Promise<chrome.windows.Window | undefined>;
|
create(props: chrome.windows.CreateData): Promise<chrome.windows.Window | undefined>;
|
||||||
|
remove(windowId: number): Promise<void>;
|
||||||
}
|
}
|
||||||
|
|
||||||
export const chromeWindowsApi: ChromeWindowsApi = {
|
export const chromeWindowsApi: ChromeWindowsApi = {
|
||||||
@@ -164,6 +165,7 @@ export const chromeWindowsApi: ChromeWindowsApi = {
|
|||||||
getLastFocused: (filters) =>
|
getLastFocused: (filters) =>
|
||||||
filters ? chrome.windows.getLastFocused(filters) : chrome.windows.getLastFocused(),
|
filters ? chrome.windows.getLastFocused(filters) : chrome.windows.getLastFocused(),
|
||||||
create: (p) => chrome.windows.create(p),
|
create: (p) => chrome.windows.create(p),
|
||||||
|
remove: (id) => chrome.windows.remove(id),
|
||||||
};
|
};
|
||||||
|
|
||||||
export interface AgentOverlayResetApi {
|
export interface AgentOverlayResetApi {
|
||||||
@@ -215,7 +217,9 @@ export async function handleTabList(
|
|||||||
manager: SessionManager,
|
manager: SessionManager,
|
||||||
params: TabListParams,
|
params: TabListParams,
|
||||||
api: ChromeTabsApi = chromeTabsApi,
|
api: ChromeTabsApi = chromeTabsApi,
|
||||||
|
signal?: AbortSignal,
|
||||||
): Promise<TabListResult | RpcError> {
|
): Promise<TabListResult | RpcError> {
|
||||||
|
if (signal?.aborted) return { code: "cancelled", message: "tab_list aborted" };
|
||||||
if (!params || typeof params.session_id !== "string" || params.session_id.length === 0) {
|
if (!params || typeof params.session_id !== "string" || params.session_id.length === 0) {
|
||||||
return {
|
return {
|
||||||
code: "invalid_params",
|
code: "invalid_params",
|
||||||
@@ -249,6 +253,7 @@ export async function handleTabList(
|
|||||||
const myAgentWindowId = ctx.agentWindowId;
|
const myAgentWindowId = ctx.agentWindowId;
|
||||||
|
|
||||||
const allTabs = await api.query({});
|
const allTabs = await api.query({});
|
||||||
|
if (signal?.aborted) return { code: "cancelled", message: "tab_list aborted" };
|
||||||
const tabs: TabInfo[] = [];
|
const tabs: TabInfo[] = [];
|
||||||
for (const t of allTabs) {
|
for (const t of allTabs) {
|
||||||
if (typeof t.id !== "number") continue;
|
if (typeof t.id !== "number") continue;
|
||||||
@@ -382,7 +387,12 @@ async function createTabAndCleanup(
|
|||||||
try {
|
try {
|
||||||
await getTabsApi(deps).remove(tab.id);
|
await getTabsApi(deps).remove(tab.id);
|
||||||
} catch (cleanupErr) {
|
} catch (cleanupErr) {
|
||||||
console.debug("[bsk tab_create] cleanup after abort failed", cleanupErr);
|
return rpcError(
|
||||||
|
"protocol_error",
|
||||||
|
"cleanup_failed",
|
||||||
|
`tab_create aborted but cleanup of tab ${tab.id} failed: ${describeError(cleanupErr)}`,
|
||||||
|
{ resource_type: "tab", resource_id: tab.id },
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return { code: "cancelled", message: "tab_create aborted" };
|
return { code: "cancelled", message: "tab_create aborted" };
|
||||||
@@ -693,7 +703,17 @@ async function moveTabForBorrow(
|
|||||||
try {
|
try {
|
||||||
await tabsApi.move(tabId, { windowId: originalWindowId, index: originalIndex });
|
await tabsApi.move(tabId, { windowId: originalWindowId, index: originalIndex });
|
||||||
} catch (rollbackErr) {
|
} catch (rollbackErr) {
|
||||||
console.debug("[bsk tab_borrow] rollback move failed", rollbackErr);
|
return rpcError(
|
||||||
|
"protocol_error",
|
||||||
|
"cleanup_failed",
|
||||||
|
`tab_borrow aborted but rollback of tab ${tabId} to window ${originalWindowId} failed: ${describeError(rollbackErr)}`,
|
||||||
|
{
|
||||||
|
resource_type: "tab",
|
||||||
|
resource_id: tabId,
|
||||||
|
original_window_id: originalWindowId,
|
||||||
|
original_index: originalIndex,
|
||||||
|
},
|
||||||
|
);
|
||||||
}
|
}
|
||||||
return { code: "cancelled", message: "tab_borrow aborted" };
|
return { code: "cancelled", message: "tab_borrow aborted" };
|
||||||
}
|
}
|
||||||
@@ -825,6 +845,12 @@ export interface ReturnOutcome {
|
|||||||
fallback: boolean;
|
fallback: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
interface FallbackWindowTarget {
|
||||||
|
windowId: number;
|
||||||
|
index: number;
|
||||||
|
created: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
function describeError(err: unknown): string {
|
function describeError(err: unknown): string {
|
||||||
return err instanceof Error ? err.message : String(err);
|
return err instanceof Error ? err.message : String(err);
|
||||||
}
|
}
|
||||||
@@ -833,10 +859,13 @@ async function chooseFallbackWindow(
|
|||||||
ctx: SessionContext,
|
ctx: SessionContext,
|
||||||
windowsApi: ChromeWindowsApi,
|
windowsApi: ChromeWindowsApi,
|
||||||
isAgentWindowId: (windowId: number) => boolean,
|
isAgentWindowId: (windowId: number) => boolean,
|
||||||
): Promise<{ windowId: number; index: number } | RpcError> {
|
signal?: AbortSignal,
|
||||||
|
): Promise<FallbackWindowTarget | RpcError> {
|
||||||
let lastFocusedError: unknown;
|
let lastFocusedError: unknown;
|
||||||
try {
|
try {
|
||||||
const last = await windowsApi.getLastFocused({ windowTypes: ["normal"] });
|
const last = await windowsApi.getLastFocused({ windowTypes: ["normal"] });
|
||||||
|
const cancelled = aborted(signal, "tab_return");
|
||||||
|
if (cancelled) return cancelled;
|
||||||
const lastId = typeof last?.id === "number" ? last.id : null;
|
const lastId = typeof last?.id === "number" ? last.id : null;
|
||||||
// Never relocate a user tab into *any* session's Agent Window: this
|
// Never relocate a user tab into *any* session's Agent Window: this
|
||||||
// session's is excluded explicitly, and other sessions' are excluded
|
// session's is excluded explicitly, and other sessions' are excluded
|
||||||
@@ -846,12 +875,15 @@ async function chooseFallbackWindow(
|
|||||||
// session write to it and, worse, see it destroyed when that session
|
// session write to it and, worse, see it destroyed when that session
|
||||||
// stops and closes its window.
|
// stops and closes its window.
|
||||||
if (lastId !== null && lastId !== ctx.agentWindowId && !isAgentWindowId(lastId)) {
|
if (lastId !== null && lastId !== ctx.agentWindowId && !isAgentWindowId(lastId)) {
|
||||||
return { windowId: lastId, index: -1 };
|
return { windowId: lastId, index: -1, created: false };
|
||||||
}
|
}
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
lastFocusedError = err;
|
lastFocusedError = err;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const cancelled = aborted(signal, "tab_return");
|
||||||
|
if (cancelled) return cancelled;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const created = await windowsApi.create({
|
const created = await windowsApi.create({
|
||||||
url: NEW_TAB_DEFAULT_URL,
|
url: NEW_TAB_DEFAULT_URL,
|
||||||
@@ -863,7 +895,20 @@ async function chooseFallbackWindow(
|
|||||||
message: "tab_return: failed to create fallback window",
|
message: "tab_return: failed to create fallback window",
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
return { windowId: created.id, index: 0 };
|
if (aborted(signal, "tab_return")) {
|
||||||
|
try {
|
||||||
|
await windowsApi.remove(created.id);
|
||||||
|
} catch (cleanupErr) {
|
||||||
|
return rpcError(
|
||||||
|
"protocol_error",
|
||||||
|
"cleanup_failed",
|
||||||
|
`tab_return aborted but cleanup of fallback window ${created.id} failed: ${describeError(cleanupErr)}`,
|
||||||
|
{ resource_type: "window", resource_id: created.id },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return { code: "cancelled", message: "tab_return aborted" };
|
||||||
|
}
|
||||||
|
return { windowId: created.id, index: 0, created: true };
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
const suffix = lastFocusedError
|
const suffix = lastFocusedError
|
||||||
? ` (after getLastFocused failed: ${describeError(lastFocusedError)})`
|
? ` (after getLastFocused failed: ${describeError(lastFocusedError)})`
|
||||||
@@ -875,6 +920,25 @@ async function chooseFallbackWindow(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function cleanupUnusedFallbackWindow(
|
||||||
|
windowsApi: ChromeWindowsApi,
|
||||||
|
target: FallbackWindowTarget,
|
||||||
|
reason: string,
|
||||||
|
): Promise<RpcError | null> {
|
||||||
|
if (!target.created) return null;
|
||||||
|
try {
|
||||||
|
await windowsApi.remove(target.windowId);
|
||||||
|
return null;
|
||||||
|
} catch (cleanupErr) {
|
||||||
|
return rpcError(
|
||||||
|
"protocol_error",
|
||||||
|
"cleanup_failed",
|
||||||
|
`${reason}; cleanup of fallback window ${target.windowId} failed: ${describeError(cleanupErr)}`,
|
||||||
|
{ resource_type: "window", resource_id: target.windowId },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
function resetAgentOverlaysInReturnedTab(
|
function resetAgentOverlaysInReturnedTab(
|
||||||
ctx: SessionContext,
|
ctx: SessionContext,
|
||||||
tabId: number,
|
tabId: number,
|
||||||
@@ -911,10 +975,13 @@ export async function returnBorrowedTab(
|
|||||||
const tabsApi = getTabsApi(deps);
|
const tabsApi = getTabsApi(deps);
|
||||||
const windowsApi = getWindowsApi(deps);
|
const windowsApi = getWindowsApi(deps);
|
||||||
const isAgentWindowId = getIsAgentWindowId(deps);
|
const isAgentWindowId = getIsAgentWindowId(deps);
|
||||||
|
const alreadyCancelled = aborted(deps.signal, "tab_return");
|
||||||
|
if (alreadyCancelled) return alreadyCancelled;
|
||||||
|
|
||||||
let targetWindowId = entry.originalWindowId;
|
let targetWindowId = entry.originalWindowId;
|
||||||
let targetIndex = entry.originalIndex;
|
let targetIndex = entry.originalIndex;
|
||||||
let fallback = false;
|
let fallback = false;
|
||||||
|
let fallbackTarget: FallbackWindowTarget | null = null;
|
||||||
|
|
||||||
// Check the original window is still around.
|
// Check the original window is still around.
|
||||||
let originalAlive = true;
|
let originalAlive = true;
|
||||||
@@ -924,14 +991,30 @@ export async function returnBorrowedTab(
|
|||||||
console.debug("[bsk tab_return] original window gone, falling back", err);
|
console.debug("[bsk tab_return] original window gone, falling back", err);
|
||||||
originalAlive = false;
|
originalAlive = false;
|
||||||
}
|
}
|
||||||
|
const cancelledAfterLookup = aborted(deps.signal, "tab_return");
|
||||||
|
if (cancelledAfterLookup) return cancelledAfterLookup;
|
||||||
if (!originalAlive) {
|
if (!originalAlive) {
|
||||||
fallback = true;
|
fallback = true;
|
||||||
const target = await chooseFallbackWindow(ctx, windowsApi, isAgentWindowId);
|
const target = await chooseFallbackWindow(ctx, windowsApi, isAgentWindowId, deps.signal);
|
||||||
if ("code" in target) return target;
|
if ("code" in target) return target;
|
||||||
|
fallbackTarget = target;
|
||||||
targetWindowId = target.windowId;
|
targetWindowId = target.windowId;
|
||||||
targetIndex = target.index;
|
targetIndex = target.index;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const cancelledBeforeMove = aborted(deps.signal, "tab_return");
|
||||||
|
if (cancelledBeforeMove) {
|
||||||
|
if (fallbackTarget) {
|
||||||
|
const cleanupError = await cleanupUnusedFallbackWindow(
|
||||||
|
windowsApi,
|
||||||
|
fallbackTarget,
|
||||||
|
"tab_return aborted before moving the borrowed tab",
|
||||||
|
);
|
||||||
|
if (cleanupError) return cleanupError;
|
||||||
|
}
|
||||||
|
return cancelledBeforeMove;
|
||||||
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const moved = await tabsApi.move(tabId, {
|
const moved = await tabsApi.move(tabId, {
|
||||||
windowId: targetWindowId,
|
windowId: targetWindowId,
|
||||||
@@ -947,14 +1030,33 @@ export async function returnBorrowedTab(
|
|||||||
fallback,
|
fallback,
|
||||||
};
|
};
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
|
if (fallbackTarget) {
|
||||||
|
const cleanupError = await cleanupUnusedFallbackWindow(
|
||||||
|
windowsApi,
|
||||||
|
fallbackTarget,
|
||||||
|
`tab_return could not move tab ${tabId}`,
|
||||||
|
);
|
||||||
|
if (cleanupError) return cleanupError;
|
||||||
|
}
|
||||||
|
const cancelledAfterMoveFailure = aborted(deps.signal, "tab_return");
|
||||||
|
if (cancelledAfterMoveFailure) return cancelledAfterMoveFailure;
|
||||||
if (!fallback) {
|
if (!fallback) {
|
||||||
const target = await chooseFallbackWindow(ctx, windowsApi, isAgentWindowId);
|
const target = await chooseFallbackWindow(ctx, windowsApi, isAgentWindowId, deps.signal);
|
||||||
if ("code" in target) {
|
if ("code" in target) {
|
||||||
return {
|
return {
|
||||||
code: "cdp_failed",
|
code: "cdp_failed",
|
||||||
message: `tab_return: chrome.tabs.move failed: ${describeError(err)}; fallback failed: ${target.message}`,
|
message: `tab_return: chrome.tabs.move failed: ${describeError(err)}; fallback failed: ${target.message}`,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
const cancelledBeforeFallbackMove = aborted(deps.signal, "tab_return");
|
||||||
|
if (cancelledBeforeFallbackMove) {
|
||||||
|
const cleanupError = await cleanupUnusedFallbackWindow(
|
||||||
|
windowsApi,
|
||||||
|
target,
|
||||||
|
"tab_return aborted before fallback move",
|
||||||
|
);
|
||||||
|
return cleanupError ?? cancelledBeforeFallbackMove;
|
||||||
|
}
|
||||||
try {
|
try {
|
||||||
const moved = await tabsApi.move(tabId, {
|
const moved = await tabsApi.move(tabId, {
|
||||||
windowId: target.windowId,
|
windowId: target.windowId,
|
||||||
@@ -970,6 +1072,12 @@ export async function returnBorrowedTab(
|
|||||||
fallback: true,
|
fallback: true,
|
||||||
};
|
};
|
||||||
} catch (fallbackErr) {
|
} catch (fallbackErr) {
|
||||||
|
const cleanupError = await cleanupUnusedFallbackWindow(
|
||||||
|
windowsApi,
|
||||||
|
target,
|
||||||
|
`tab_return fallback move for tab ${tabId} failed`,
|
||||||
|
);
|
||||||
|
if (cleanupError) return cleanupError;
|
||||||
return {
|
return {
|
||||||
code: "cdp_failed",
|
code: "cdp_failed",
|
||||||
message: `tab_return: chrome.tabs.move failed: ${describeError(err)}; fallback move failed: ${describeError(fallbackErr)}`,
|
message: `tab_return: chrome.tabs.move failed: ${describeError(err)}; fallback move failed: ${describeError(fallbackErr)}`,
|
||||||
|
|||||||
@@ -18,6 +18,8 @@ function fakeSnapshotReply() {
|
|||||||
"auto",
|
"auto",
|
||||||
"type",
|
"type",
|
||||||
"password",
|
"password",
|
||||||
|
"value",
|
||||||
|
"hunter2",
|
||||||
];
|
];
|
||||||
const i = (s: string) => S.indexOf(s);
|
const i = (s: string) => S.indexOf(s);
|
||||||
return {
|
return {
|
||||||
@@ -29,7 +31,8 @@ function fakeSnapshotReply() {
|
|||||||
nodeType: [1, 1, 1, 1],
|
nodeType: [1, 1, 1, 1],
|
||||||
nodeName: [i("html"), i("body"), i("div"), i("input")],
|
nodeName: [i("html"), i("body"), i("div"), i("input")],
|
||||||
backendNodeId: [10, 11, 12, 13],
|
backendNodeId: [10, 11, 12, 13],
|
||||||
attributes: [[], [], [], [i("type"), i("password")]],
|
attributes: [[], [], [], [i("type"), i("password"), i("value"), i("hunter2")]],
|
||||||
|
inputValue: { index: [3], value: [i("hunter2")] },
|
||||||
},
|
},
|
||||||
layout: {
|
layout: {
|
||||||
nodeIndex: [1, 2, 3],
|
nodeIndex: [1, 2, 3],
|
||||||
@@ -51,6 +54,156 @@ function fakeSnapshotReply() {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function hoverTriggerSnapshotReply() {
|
||||||
|
const S = [
|
||||||
|
"html",
|
||||||
|
"body",
|
||||||
|
"button",
|
||||||
|
"class",
|
||||||
|
"user-avatar",
|
||||||
|
"position",
|
||||||
|
"static",
|
||||||
|
"pointer-events",
|
||||||
|
"auto",
|
||||||
|
"cursor",
|
||||||
|
"pointer",
|
||||||
|
];
|
||||||
|
const i = (s: string) => S.indexOf(s);
|
||||||
|
return {
|
||||||
|
strings: S,
|
||||||
|
documents: [
|
||||||
|
{
|
||||||
|
nodes: {
|
||||||
|
parentIndex: [-1, 0, 1],
|
||||||
|
nodeType: [1, 1, 1],
|
||||||
|
nodeName: [i("html"), i("body"), i("button")],
|
||||||
|
backendNodeId: [10, 11, 12],
|
||||||
|
attributes: [[], [], [i("class"), i("user-avatar")]],
|
||||||
|
},
|
||||||
|
layout: {
|
||||||
|
nodeIndex: [1, 2],
|
||||||
|
styles: [
|
||||||
|
[i("static"), i("auto"), i("pointer")],
|
||||||
|
[i("static"), i("auto"), i("pointer")],
|
||||||
|
],
|
||||||
|
bounds: [
|
||||||
|
[0, 0, 1000, 800],
|
||||||
|
[940, 16, 32, 32],
|
||||||
|
],
|
||||||
|
paintOrders: [0, 1],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function twoHoverTriggerSnapshotReply() {
|
||||||
|
const S = [
|
||||||
|
"html",
|
||||||
|
"body",
|
||||||
|
"button",
|
||||||
|
"class",
|
||||||
|
"user-avatar",
|
||||||
|
"secondary-dropdown-trigger",
|
||||||
|
"position",
|
||||||
|
"static",
|
||||||
|
"pointer-events",
|
||||||
|
"auto",
|
||||||
|
"cursor",
|
||||||
|
"pointer",
|
||||||
|
];
|
||||||
|
const i = (s: string) => S.indexOf(s);
|
||||||
|
return {
|
||||||
|
strings: S,
|
||||||
|
documents: [
|
||||||
|
{
|
||||||
|
nodes: {
|
||||||
|
parentIndex: [-1, 0, 1, 1],
|
||||||
|
nodeType: [1, 1, 1, 1],
|
||||||
|
nodeName: [i("html"), i("body"), i("button"), i("button")],
|
||||||
|
backendNodeId: [10, 11, 12, 13],
|
||||||
|
attributes: [
|
||||||
|
[],
|
||||||
|
[],
|
||||||
|
[i("class"), i("user-avatar")],
|
||||||
|
[i("class"), i("secondary-dropdown-trigger")],
|
||||||
|
],
|
||||||
|
},
|
||||||
|
layout: {
|
||||||
|
nodeIndex: [1, 2, 3],
|
||||||
|
styles: [
|
||||||
|
[i("static"), i("auto"), i("pointer")],
|
||||||
|
[i("static"), i("auto"), i("pointer")],
|
||||||
|
[i("static"), i("auto"), i("pointer")],
|
||||||
|
],
|
||||||
|
bounds: [
|
||||||
|
[0, 0, 1000, 800],
|
||||||
|
[940, 16, 32, 32],
|
||||||
|
[100, 120, 32, 32],
|
||||||
|
],
|
||||||
|
paintOrders: [0, 1, 2],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function nestedHoverTriggerSnapshotReply() {
|
||||||
|
const S = [
|
||||||
|
"html",
|
||||||
|
"body",
|
||||||
|
"div",
|
||||||
|
"img",
|
||||||
|
"class",
|
||||||
|
"tg-avatar",
|
||||||
|
"tg-avatar__inner",
|
||||||
|
"tg-avatar__image",
|
||||||
|
"position",
|
||||||
|
"static",
|
||||||
|
"pointer-events",
|
||||||
|
"auto",
|
||||||
|
"cursor",
|
||||||
|
"pointer",
|
||||||
|
];
|
||||||
|
const i = (s: string) => S.indexOf(s);
|
||||||
|
return {
|
||||||
|
strings: S,
|
||||||
|
documents: [
|
||||||
|
{
|
||||||
|
nodes: {
|
||||||
|
parentIndex: [-1, 0, 1, 2, 3],
|
||||||
|
nodeType: [1, 1, 1, 1, 1],
|
||||||
|
nodeName: [i("html"), i("body"), i("div"), i("div"), i("img")],
|
||||||
|
backendNodeId: [10, 11, 12, 13, 14],
|
||||||
|
attributes: [
|
||||||
|
[],
|
||||||
|
[],
|
||||||
|
[i("class"), i("tg-avatar")],
|
||||||
|
[i("class"), i("tg-avatar__inner")],
|
||||||
|
[i("class"), i("tg-avatar__image")],
|
||||||
|
],
|
||||||
|
},
|
||||||
|
layout: {
|
||||||
|
nodeIndex: [1, 2, 3, 4],
|
||||||
|
styles: [
|
||||||
|
[i("static"), i("auto"), i("pointer")],
|
||||||
|
[i("static"), i("auto"), i("pointer")],
|
||||||
|
[i("static"), i("auto"), i("pointer")],
|
||||||
|
[i("static"), i("auto"), i("pointer")],
|
||||||
|
],
|
||||||
|
bounds: [
|
||||||
|
[0, 0, 1000, 800],
|
||||||
|
[940, 16, 32, 32],
|
||||||
|
[942, 18, 28, 28],
|
||||||
|
[944, 20, 24, 24],
|
||||||
|
],
|
||||||
|
paintOrders: [0, 1, 2, 3],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
function makeCdp(snapshot: unknown) {
|
function makeCdp(snapshot: unknown) {
|
||||||
return {
|
return {
|
||||||
send: vi.fn(async (_tab: number, method: string) => {
|
send: vi.fn(async (_tab: number, method: string) => {
|
||||||
@@ -67,7 +220,8 @@ function makeCdp(snapshot: unknown) {
|
|||||||
{
|
{
|
||||||
state: "filled",
|
state: "filled",
|
||||||
sensitive: true,
|
sensitive: true,
|
||||||
defaultValue: "",
|
defaultValue: "hunter2",
|
||||||
|
value: "hunter2",
|
||||||
placeholder: "secret",
|
placeholder: "secret",
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
@@ -112,10 +266,11 @@ describe("captureViewModel", () => {
|
|||||||
|
|
||||||
expect(input).toMatchObject({
|
expect(input).toMatchObject({
|
||||||
formState: "filled",
|
formState: "filled",
|
||||||
formDefaultValue: "",
|
|
||||||
formPlaceholder: "secret",
|
formPlaceholder: "secret",
|
||||||
|
attrs: { type: "password" },
|
||||||
});
|
});
|
||||||
expect(input?.formValue).toBeUndefined();
|
expect(input?.formValue).toBeUndefined();
|
||||||
|
expect(input?.formDefaultValue).toBeUndefined();
|
||||||
expect(cdp.send).toHaveBeenCalledWith(
|
expect(cdp.send).toHaveBeenCalledWith(
|
||||||
4,
|
4,
|
||||||
"Runtime.evaluate",
|
"Runtime.evaluate",
|
||||||
@@ -126,11 +281,11 @@ describe("captureViewModel", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it("runs hover surface probes when explicitly enabled", async () => {
|
it("runs hover surface probes when explicitly enabled", async () => {
|
||||||
let runtimeCalls = 0;
|
let hoverStateCalls = 0;
|
||||||
const cdp = {
|
const cdp = {
|
||||||
send: vi.fn(async (_tabId: number, method: string, params?: object) => {
|
send: vi.fn(async (_tabId: number, method: string, params?: object) => {
|
||||||
if (method === "DOMSnapshot.enable") return {};
|
if (method === "DOMSnapshot.enable") return {};
|
||||||
if (method === "DOMSnapshot.captureSnapshot") return fakeSnapshotReply();
|
if (method === "DOMSnapshot.captureSnapshot") return hoverTriggerSnapshotReply();
|
||||||
if (method === "Page.getLayoutMetrics") {
|
if (method === "Page.getLayoutMetrics") {
|
||||||
return {
|
return {
|
||||||
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
|
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
|
||||||
@@ -141,22 +296,23 @@ describe("captureViewModel", () => {
|
|||||||
if (expression.includes("input,textarea,select")) {
|
if (expression.includes("input,textarea,select")) {
|
||||||
return { result: { value: { controls: [], childFrames: [] } } };
|
return { result: { value: { controls: [], childFrames: [] } } };
|
||||||
}
|
}
|
||||||
runtimeCalls += 1;
|
if (expression.includes("document.styleSheets")) {
|
||||||
return runtimeCalls === 1
|
return { result: { value: [] } };
|
||||||
? {
|
}
|
||||||
result: {
|
if (expression.includes("querySelectorAll(selectors)")) {
|
||||||
value: [
|
hoverStateCalls += 1;
|
||||||
{
|
return hoverStateCalls === 1
|
||||||
triggerSel: ".menu",
|
? { result: { value: [] } }
|
||||||
affectedSub: " .items",
|
: {
|
||||||
label: "Products",
|
result: {
|
||||||
x: 50,
|
value: [
|
||||||
y: 20,
|
{ text: "My profile", role: "", tag: "a", x: 900, y: 60 },
|
||||||
},
|
{ text: "Sign out", role: "", tag: "a", x: 900, y: 90 },
|
||||||
],
|
],
|
||||||
},
|
},
|
||||||
}
|
};
|
||||||
: { result: { value: ["Shoes", "Bags"] } };
|
}
|
||||||
|
throw new Error(`unexpected Runtime.evaluate: ${expression.slice(0, 80)}`);
|
||||||
}
|
}
|
||||||
if (method === "Input.dispatchMouseEvent") return {};
|
if (method === "Input.dispatchMouseEvent") return {};
|
||||||
throw new Error(`unexpected ${method}`);
|
throw new Error(`unexpected ${method}`);
|
||||||
@@ -166,12 +322,18 @@ describe("captureViewModel", () => {
|
|||||||
const result = await captureViewModel(cdp, 4, { conditionalSurfaceProbe: true });
|
const result = await captureViewModel(cdp, 4, { conditionalSurfaceProbe: true });
|
||||||
|
|
||||||
expect(result.surfaceProbes).toEqual([
|
expect(result.surfaceProbes).toEqual([
|
||||||
{ triggerLabel: "Products", triggerAction: "hover", subItems: ["Shoes", "Bags"] },
|
{
|
||||||
|
triggerBackendNodeId: 12,
|
||||||
|
triggerPoint: { x: 956, y: 32 },
|
||||||
|
triggerAction: "hover",
|
||||||
|
subItems: ["My profile", "Sign out"],
|
||||||
|
confidence: "high",
|
||||||
|
},
|
||||||
]);
|
]);
|
||||||
expect(cdp.send).toHaveBeenCalledWith(
|
expect(cdp.send).toHaveBeenCalledWith(
|
||||||
4,
|
4,
|
||||||
"Input.dispatchMouseEvent",
|
"Input.dispatchMouseEvent",
|
||||||
expect.objectContaining({ type: "mouseMoved", x: 50, y: 20 }),
|
expect.objectContaining({ type: "mouseMoved", x: 956, y: 32 }),
|
||||||
);
|
);
|
||||||
expect(cdp.send).toHaveBeenCalledWith(
|
expect(cdp.send).toHaveBeenCalledWith(
|
||||||
4,
|
4,
|
||||||
@@ -180,6 +342,91 @@ describe("captureViewModel", () => {
|
|||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("uses a fresh baseline for each hover candidate", async () => {
|
||||||
|
let hoverStateCalls = 0;
|
||||||
|
const cdp = {
|
||||||
|
send: vi.fn(async (_tabId: number, method: string, params?: object) => {
|
||||||
|
if (method === "DOMSnapshot.enable") return {};
|
||||||
|
if (method === "DOMSnapshot.captureSnapshot") return twoHoverTriggerSnapshotReply();
|
||||||
|
if (method === "Page.getLayoutMetrics") {
|
||||||
|
return {
|
||||||
|
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if (method === "Runtime.evaluate") {
|
||||||
|
const expression = (params as { expression?: string } | undefined)?.expression ?? "";
|
||||||
|
if (expression.includes("input,textarea,select")) {
|
||||||
|
return { result: { value: { controls: [], childFrames: [] } } };
|
||||||
|
}
|
||||||
|
if (expression.includes("document.styleSheets")) {
|
||||||
|
return { result: { value: [] } };
|
||||||
|
}
|
||||||
|
if (expression.includes("querySelectorAll(selectors)")) {
|
||||||
|
hoverStateCalls += 1;
|
||||||
|
const visibleMenu = [
|
||||||
|
{ text: "My profile", role: "", tag: "a", x: 900, y: 60 },
|
||||||
|
{ text: "Sign out", role: "", tag: "a", x: 900, y: 90 },
|
||||||
|
];
|
||||||
|
return hoverStateCalls === 1
|
||||||
|
? { result: { value: [] } }
|
||||||
|
: { result: { value: visibleMenu } };
|
||||||
|
}
|
||||||
|
throw new Error(`unexpected Runtime.evaluate: ${expression.slice(0, 80)}`);
|
||||||
|
}
|
||||||
|
if (method === "Input.dispatchMouseEvent") return {};
|
||||||
|
throw new Error(`unexpected ${method}`);
|
||||||
|
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||||
|
};
|
||||||
|
|
||||||
|
const result = await captureViewModel(cdp, 4, { conditionalSurfaceProbe: true });
|
||||||
|
|
||||||
|
expect(result.surfaceProbes).toEqual([
|
||||||
|
expect.objectContaining({
|
||||||
|
triggerBackendNodeId: 12,
|
||||||
|
subItems: ["My profile", "Sign out"],
|
||||||
|
}),
|
||||||
|
]);
|
||||||
|
expect(hoverStateCalls).toBeGreaterThanOrEqual(4);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("deduplicates nested hover candidates for one visual trigger", async () => {
|
||||||
|
let hoverMoves = 0;
|
||||||
|
const cdp = {
|
||||||
|
send: vi.fn(async (_tabId: number, method: string, params?: object) => {
|
||||||
|
if (method === "DOMSnapshot.enable") return {};
|
||||||
|
if (method === "DOMSnapshot.captureSnapshot") return nestedHoverTriggerSnapshotReply();
|
||||||
|
if (method === "Page.getLayoutMetrics") {
|
||||||
|
return {
|
||||||
|
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if (method === "Runtime.evaluate") {
|
||||||
|
const expression = (params as { expression?: string } | undefined)?.expression ?? "";
|
||||||
|
if (expression.includes("input,textarea,select")) {
|
||||||
|
return { result: { value: { controls: [], childFrames: [] } } };
|
||||||
|
}
|
||||||
|
if (expression.includes("document.styleSheets")) {
|
||||||
|
return { result: { value: [] } };
|
||||||
|
}
|
||||||
|
if (expression.includes("querySelectorAll(selectors)")) {
|
||||||
|
return { result: { value: [] } };
|
||||||
|
}
|
||||||
|
throw new Error(`unexpected Runtime.evaluate: ${expression.slice(0, 80)}`);
|
||||||
|
}
|
||||||
|
if (method === "Input.dispatchMouseEvent") {
|
||||||
|
const point = params as { x?: number; y?: number };
|
||||||
|
if (point.x !== -10 && point.x !== 0) hoverMoves += 1;
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
throw new Error(`unexpected ${method}`);
|
||||||
|
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||||
|
};
|
||||||
|
|
||||||
|
await captureViewModel(cdp, 4, { conditionalSurfaceProbe: true });
|
||||||
|
|
||||||
|
expect(hoverMoves).toBe(1);
|
||||||
|
});
|
||||||
|
|
||||||
it("excludes the agent's own overlay shadow host and its inlined shadow subtree", async () => {
|
it("excludes the agent's own overlay shadow host and its inlined shadow subtree", async () => {
|
||||||
// DOMSnapshot inlines an open shadow root's content as descendants of the
|
// DOMSnapshot inlines an open shadow root's content as descendants of the
|
||||||
// host. The agent's WXT overlay host carries a fixed full-viewport
|
// host. The agent's WXT overlay host carries a fixed full-viewport
|
||||||
@@ -450,9 +697,7 @@ describe("captureViewModel", () => {
|
|||||||
};
|
};
|
||||||
const { nodes } = await captureViewModel(cdp, 4);
|
const { nodes } = await captureViewModel(cdp, 4);
|
||||||
const div = nodes.find((n) => n.backendNodeId === 12);
|
const div = nodes.find((n) => n.backendNodeId === 12);
|
||||||
// local rect preserves viewport-relative scroll math.
|
|
||||||
expect(div?.localRect?.y).toBe(-200);
|
expect(div?.localRect?.y).toBe(-200);
|
||||||
// exported rect is clipped to the top-level viewport before VOM consumes it.
|
|
||||||
expect(div?.rect).toMatchObject({ y: 0, h: 600 });
|
expect(div?.rect).toMatchObject({ y: 0, h: 600 });
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -597,6 +842,7 @@ describe("captureViewModel", () => {
|
|||||||
strings: S,
|
strings: S,
|
||||||
documents: [
|
documents: [
|
||||||
{
|
{
|
||||||
|
frameId: "main-frame",
|
||||||
scrollOffsetX: 0,
|
scrollOffsetX: 0,
|
||||||
scrollOffsetY: 0,
|
scrollOffsetY: 0,
|
||||||
nodes: {
|
nodes: {
|
||||||
@@ -624,6 +870,7 @@ describe("captureViewModel", () => {
|
|||||||
},
|
},
|
||||||
// documents[1]: the iframe's sub-document with a login input
|
// documents[1]: the iframe's sub-document with a login input
|
||||||
{
|
{
|
||||||
|
frameId: "child-frame",
|
||||||
scrollOffsetX: 0,
|
scrollOffsetX: 0,
|
||||||
scrollOffsetY: 0,
|
scrollOffsetY: 0,
|
||||||
nodes: {
|
nodes: {
|
||||||
@@ -653,7 +900,14 @@ describe("captureViewModel", () => {
|
|||||||
throw new Error(method);
|
throw new Error(method);
|
||||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||||
};
|
};
|
||||||
const { nodes, iframeNodes } = await captureViewModel(cdp, 4);
|
const {
|
||||||
|
nodes,
|
||||||
|
iframeNodes,
|
||||||
|
frameNodes,
|
||||||
|
frameOwnerBackendNodeIds,
|
||||||
|
frameParentIds,
|
||||||
|
rootFrameId,
|
||||||
|
} = await captureViewModel(cdp, 4);
|
||||||
// Main frame has 4 nodes; iframe is included
|
// Main frame has 4 nodes; iframe is included
|
||||||
expect(nodes.find((n) => n.backendNodeId === 13)?.tag).toBe("iframe");
|
expect(nodes.find((n) => n.backendNodeId === 13)?.tag).toBe("iframe");
|
||||||
// iframeNodes keyed by the <iframe> element's backendNodeId=13
|
// iframeNodes keyed by the <iframe> element's backendNodeId=13
|
||||||
@@ -666,9 +920,13 @@ describe("captureViewModel", () => {
|
|||||||
expect(input?.ownerFrameBackendNodeId).toBe(13);
|
expect(input?.ownerFrameBackendNodeId).toBe(13);
|
||||||
expect(input?.localRect).toEqual({ x: 0, y: 0, w: 200, h: 40 });
|
expect(input?.localRect).toEqual({ x: 0, y: 0, w: 200, h: 40 });
|
||||||
expect(input?.rect).toEqual({ x: 100, y: 300, w: 200, h: 40 });
|
expect(input?.rect).toEqual({ x: 100, y: 300, w: 200, h: 40 });
|
||||||
|
expect(rootFrameId).toBe("main-frame");
|
||||||
|
expect(frameNodes?.get("child-frame")).toBe(subNodes);
|
||||||
|
expect(frameOwnerBackendNodeIds?.get("child-frame")).toBe(13);
|
||||||
|
expect(frameParentIds?.get("child-frame")).toBe("main-frame");
|
||||||
});
|
});
|
||||||
|
|
||||||
it("clips iframe sub-document rects to the iframe viewport in top coordinates", async () => {
|
it("captures iframe documents without requiring owner-frame geometry", async () => {
|
||||||
const S = [
|
const S = [
|
||||||
"html",
|
"html",
|
||||||
"body",
|
"body",
|
||||||
@@ -696,16 +954,10 @@ describe("captureViewModel", () => {
|
|||||||
contentDocumentIndex: { index: [2], value: [1] },
|
contentDocumentIndex: { index: [2], value: [1] },
|
||||||
},
|
},
|
||||||
layout: {
|
layout: {
|
||||||
nodeIndex: [1, 2],
|
nodeIndex: [1],
|
||||||
styles: [
|
styles: [[i("static"), i("auto")]],
|
||||||
[i("static"), i("auto")],
|
bounds: [[0, 0, 1000, 800]],
|
||||||
[i("static"), i("auto")],
|
paintOrders: [0],
|
||||||
],
|
|
||||||
bounds: [
|
|
||||||
[0, 0, 1000, 800],
|
|
||||||
[950, 760, 200, 100],
|
|
||||||
],
|
|
||||||
paintOrders: [0, 1],
|
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -743,7 +995,7 @@ describe("captureViewModel", () => {
|
|||||||
const input = iframeNodes.get(13)?.find((n) => n.backendNodeId === 101);
|
const input = iframeNodes.get(13)?.find((n) => n.backendNodeId === 101);
|
||||||
|
|
||||||
expect(input?.localRect).toEqual({ x: 20, y: 20, w: 120, h: 60 });
|
expect(input?.localRect).toEqual({ x: 20, y: 20, w: 120, h: 60 });
|
||||||
expect(input?.rect).toEqual({ x: 970, y: 780, w: 30, h: 20 });
|
expect(input?.rect).toBeNull();
|
||||||
});
|
});
|
||||||
|
|
||||||
it("recursively normalizes nested iframe sub-documents", async () => {
|
it("recursively normalizes nested iframe sub-documents", async () => {
|
||||||
@@ -838,8 +1090,10 @@ describe("captureViewModel", () => {
|
|||||||
const nestedIframe = iframeNodes.get(13)?.find((n) => n.backendNodeId === 23);
|
const nestedIframe = iframeNodes.get(13)?.find((n) => n.backendNodeId === 23);
|
||||||
const input = iframeNodes.get(23)?.find((n) => n.backendNodeId === 31);
|
const input = iframeNodes.get(23)?.find((n) => n.backendNodeId === 31);
|
||||||
|
|
||||||
|
expect(nestedIframe?.localRect).toEqual({ x: 5, y: 6, w: 100, h: 80 });
|
||||||
expect(nestedIframe?.rect).toEqual({ x: 15, y: 26, w: 100, h: 80 });
|
expect(nestedIframe?.rect).toEqual({ x: 15, y: 26, w: 100, h: 80 });
|
||||||
expect(input?.ownerFrameBackendNodeId).toBe(23);
|
expect(input?.ownerFrameBackendNodeId).toBe(23);
|
||||||
|
expect(input?.localRect).toEqual({ x: 1, y: 2, w: 40, h: 20 });
|
||||||
expect(input?.rect).toEqual({ x: 16, y: 28, w: 40, h: 20 });
|
expect(input?.rect).toEqual({ x: 16, y: 28, w: 40, h: 20 });
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,135 @@
|
|||||||
|
import { describe, expect, it, vi } from "vitest";
|
||||||
|
import type { CdpRunner } from "../../shared";
|
||||||
|
import type { CapturedNode, CapturedViewModel } from "../capture";
|
||||||
|
import { captureFrameData } from "../frame-capture";
|
||||||
|
|
||||||
|
function ownerNode(backendNodeId: number, x: number): CapturedNode {
|
||||||
|
return {
|
||||||
|
backendNodeId,
|
||||||
|
parentBackendNodeId: 1,
|
||||||
|
frameId: "main",
|
||||||
|
ownerFrameBackendNodeId: null,
|
||||||
|
tag: "iframe",
|
||||||
|
attrs: {},
|
||||||
|
rect: { x, y: 100, w: 300, h: 200 },
|
||||||
|
localRect: { x, y: 100, w: 300, h: 200 },
|
||||||
|
paintOrder: 1,
|
||||||
|
position: "static",
|
||||||
|
pointerEvents: "auto",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function childSnapshot(frameId: string, backendNodeId: number) {
|
||||||
|
const strings = [frameId, "body", "button", "static", "auto", "pointer"];
|
||||||
|
return {
|
||||||
|
strings,
|
||||||
|
documents: [
|
||||||
|
{
|
||||||
|
frameId,
|
||||||
|
nodes: {
|
||||||
|
parentIndex: [-1, 0],
|
||||||
|
nodeName: [1, 2],
|
||||||
|
backendNodeId: [backendNodeId - 1, backendNodeId],
|
||||||
|
attributes: [[], []],
|
||||||
|
},
|
||||||
|
layout: {
|
||||||
|
nodeIndex: [0, 1],
|
||||||
|
styles: [
|
||||||
|
[3, 4, 4],
|
||||||
|
[3, 4, 5],
|
||||||
|
],
|
||||||
|
bounds: [
|
||||||
|
[0, 0, 300, 200],
|
||||||
|
[10, 20, 100, 40],
|
||||||
|
],
|
||||||
|
paintOrders: [0, 1],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("captureFrameData", () => {
|
||||||
|
it("captures and positions multiple OOPIF documents missing from the root snapshot", async () => {
|
||||||
|
const leftOwner = ownerNode(10, 50);
|
||||||
|
const rightOwner = ownerNode(20, 500);
|
||||||
|
const captured: CapturedViewModel = {
|
||||||
|
nodes: [leftOwner, rightOwner],
|
||||||
|
viewport: { width: 1000, height: 800 },
|
||||||
|
iframeNodes: new Map(),
|
||||||
|
frameNodes: new Map([["main", [leftOwner, rightOwner]]]),
|
||||||
|
frameOwnerBackendNodeIds: new Map(),
|
||||||
|
rootFrameId: "main",
|
||||||
|
excludedBackendNodeIds: new Set(),
|
||||||
|
};
|
||||||
|
const sendToTarget = vi.fn(async (target, method) => {
|
||||||
|
if (method === "Page.getLayoutMetrics") {
|
||||||
|
return { cssLayoutViewport: { clientWidth: 300, clientHeight: 200, pageX: 0, pageY: 0 } };
|
||||||
|
}
|
||||||
|
if (method === "DOMSnapshot.enable" || method === "Accessibility.enable") return {};
|
||||||
|
if (method === "DOMSnapshot.captureSnapshot") {
|
||||||
|
return target.sessionId === "left-session"
|
||||||
|
? childSnapshot("left", 101)
|
||||||
|
: childSnapshot("right", 201);
|
||||||
|
}
|
||||||
|
if (method === "Accessibility.getFullAXTree") return { nodes: [] };
|
||||||
|
throw new Error(`unexpected ${method}`);
|
||||||
|
});
|
||||||
|
const cdp: CdpRunner = {
|
||||||
|
send: vi.fn(async (_tabId, method, params) => {
|
||||||
|
if (method === "Accessibility.enable") return {};
|
||||||
|
if (method === "Accessibility.getFullAXTree") return { nodes: [] };
|
||||||
|
if (method === "DOM.getBoxModel") {
|
||||||
|
const backendNodeId = (params as { backendNodeId?: number })?.backendNodeId;
|
||||||
|
const x = backendNodeId === 10 ? 50 : 500;
|
||||||
|
return { model: { content: [x, 100, x + 300, 100, x + 300, 300, x, 300] } };
|
||||||
|
}
|
||||||
|
if (method === "Page.getLayoutMetrics") {
|
||||||
|
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } };
|
||||||
|
}
|
||||||
|
throw new Error(`unexpected root ${method}`);
|
||||||
|
}) as CdpRunner["send"],
|
||||||
|
sendToTarget: sendToTarget as unknown as NonNullable<CdpRunner["sendToTarget"]>,
|
||||||
|
getFrameGraph: vi.fn(async () => ({
|
||||||
|
rootFrameId: "main",
|
||||||
|
frames: [
|
||||||
|
{ frameId: "main", target: { tabId: 4 } },
|
||||||
|
{
|
||||||
|
frameId: "left",
|
||||||
|
parentFrameId: "main",
|
||||||
|
ownerBackendNodeId: 10,
|
||||||
|
target: { tabId: 4, sessionId: "left-session" },
|
||||||
|
},
|
||||||
|
{
|
||||||
|
frameId: "right",
|
||||||
|
parentFrameId: "main",
|
||||||
|
ownerBackendNodeId: 20,
|
||||||
|
target: { tabId: 4, sessionId: "right-session" },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
})),
|
||||||
|
};
|
||||||
|
|
||||||
|
const trees = await captureFrameData(cdp, 4, captured);
|
||||||
|
|
||||||
|
expect(trees.map((tree) => tree.frameId)).toEqual(["main", "left", "right"]);
|
||||||
|
expect(
|
||||||
|
captured.frameNodes?.get("left")?.find((node) => node.backendNodeId === 101)?.rect,
|
||||||
|
).toEqual({ x: 60, y: 120, w: 100, h: 40 });
|
||||||
|
expect(
|
||||||
|
captured.frameNodes?.get("right")?.find((node) => node.backendNodeId === 201)?.rect,
|
||||||
|
).toEqual({ x: 510, y: 120, w: 100, h: 40 });
|
||||||
|
expect(captured.frameOwnerBackendNodeIds).toEqual(
|
||||||
|
new Map([
|
||||||
|
["left", 10],
|
||||||
|
["right", 20],
|
||||||
|
]),
|
||||||
|
);
|
||||||
|
expect(captured.frameParentIds).toEqual(
|
||||||
|
new Map([
|
||||||
|
["left", "main"],
|
||||||
|
["right", "main"],
|
||||||
|
]),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,174 @@
|
|||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import type { CdpFrameGraph } from "@/browser-driver/frame-graph";
|
||||||
|
import type { CapturedNode, CapturedViewModel } from "../capture";
|
||||||
|
import { buildFrameDocuments, type FrameAxBatch, type FrameOwnedAxNode } from "../frame-document";
|
||||||
|
|
||||||
|
function domNode(frameId: string, backendNodeId: number): CapturedNode {
|
||||||
|
return {
|
||||||
|
backendNodeId,
|
||||||
|
parentBackendNodeId: null,
|
||||||
|
frameId,
|
||||||
|
tag: "div",
|
||||||
|
attrs: {},
|
||||||
|
rect: { x: 0, y: 0, w: 20, h: 20 },
|
||||||
|
paintOrder: 0,
|
||||||
|
position: "static",
|
||||||
|
pointerEvents: "auto",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function captured(frameNodes: Map<string, CapturedNode[]>): CapturedViewModel {
|
||||||
|
return {
|
||||||
|
nodes: frameNodes.get("main") ?? [],
|
||||||
|
viewport: { width: 800, height: 600 },
|
||||||
|
iframeNodes: new Map(),
|
||||||
|
frameNodes,
|
||||||
|
rootFrameId: "main",
|
||||||
|
excludedBackendNodeIds: new Set(),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("buildFrameDocuments", () => {
|
||||||
|
it("partitions mixed AX results by node ownership and cuts cross-frame parent edges", () => {
|
||||||
|
const graph: CdpFrameGraph = {
|
||||||
|
rootFrameId: "main",
|
||||||
|
frames: [
|
||||||
|
{ frameId: "main", target: { tabId: 4 } },
|
||||||
|
{ frameId: "child", parentFrameId: "main", target: { tabId: 4 } },
|
||||||
|
],
|
||||||
|
};
|
||||||
|
const batches: FrameAxBatch<FrameOwnedAxNode>[] = [
|
||||||
|
{
|
||||||
|
frame: graph.frames[0],
|
||||||
|
nodes: [
|
||||||
|
{ nodeId: "root", frameId: "main", backendDOMNodeId: 1, childIds: ["h"] },
|
||||||
|
{
|
||||||
|
nodeId: "h",
|
||||||
|
frameId: "child",
|
||||||
|
parentId: "root",
|
||||||
|
backendDOMNodeId: 100,
|
||||||
|
childIds: ["virtual"],
|
||||||
|
},
|
||||||
|
{ nodeId: "virtual", parentId: "h" },
|
||||||
|
],
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
const documents = buildFrameDocuments(
|
||||||
|
graph,
|
||||||
|
batches,
|
||||||
|
captured(
|
||||||
|
new Map([
|
||||||
|
["main", [domNode("main", 1)]],
|
||||||
|
["child", [domNode("child", 100)]],
|
||||||
|
]),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(documents.find((document) => document.frameId === "main")?.axNodes).toHaveLength(1);
|
||||||
|
const childNodes = documents.find((document) => document.frameId === "child")?.axNodes ?? [];
|
||||||
|
expect(childNodes.map((node) => node.nodeId)).toEqual(["h", "virtual"]);
|
||||||
|
expect(childNodes.find((node) => node.nodeId === "h")?.parentId).toBeUndefined();
|
||||||
|
expect(childNodes.find((node) => node.nodeId === "virtual")?.frameId).toBe("child");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("deduplicates overlapping same-target AX results using the strongest ownership", () => {
|
||||||
|
const graph: CdpFrameGraph = {
|
||||||
|
rootFrameId: "main",
|
||||||
|
frames: [
|
||||||
|
{ frameId: "main", target: { tabId: 4 } },
|
||||||
|
{ frameId: "child", parentFrameId: "main", target: { tabId: 4 } },
|
||||||
|
],
|
||||||
|
};
|
||||||
|
const childNode = { nodeId: "shared", backendDOMNodeId: 100 };
|
||||||
|
const batches: FrameAxBatch<FrameOwnedAxNode>[] = [
|
||||||
|
{
|
||||||
|
frame: graph.frames[0],
|
||||||
|
nodes: [{ ...childNode, frameId: "child" }],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
frame: graph.frames[1],
|
||||||
|
nodes: [childNode],
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
const documents = buildFrameDocuments(
|
||||||
|
graph,
|
||||||
|
batches,
|
||||||
|
captured(
|
||||||
|
new Map([
|
||||||
|
["main", [domNode("main", 1)]],
|
||||||
|
["child", [domNode("child", 100)]],
|
||||||
|
]),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(documents.find((document) => document.frameId === "main")?.axNodes).toEqual([]);
|
||||||
|
expect(documents.find((document) => document.frameId === "child")?.axNodes).toHaveLength(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("uses target-scoped backend ownership when OOPIF backend ids collide", () => {
|
||||||
|
const graph: CdpFrameGraph = {
|
||||||
|
rootFrameId: "main",
|
||||||
|
frames: [
|
||||||
|
{ frameId: "main", target: { tabId: 4 } },
|
||||||
|
{
|
||||||
|
frameId: "left",
|
||||||
|
parentFrameId: "main",
|
||||||
|
target: { tabId: 4, sessionId: "left-session" },
|
||||||
|
},
|
||||||
|
{
|
||||||
|
frameId: "right",
|
||||||
|
parentFrameId: "main",
|
||||||
|
target: { tabId: 4, sessionId: "right-session" },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
};
|
||||||
|
const batches: FrameAxBatch<FrameOwnedAxNode>[] = graph.frames.slice(1).map((frame) => ({
|
||||||
|
frame,
|
||||||
|
nodes: [{ nodeId: `${frame.frameId}-node`, backendDOMNodeId: 7 }],
|
||||||
|
}));
|
||||||
|
|
||||||
|
const documents = buildFrameDocuments(
|
||||||
|
graph,
|
||||||
|
batches,
|
||||||
|
captured(
|
||||||
|
new Map([
|
||||||
|
["main", []],
|
||||||
|
["left", [domNode("left", 7)]],
|
||||||
|
["right", [domNode("right", 7)]],
|
||||||
|
]),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(documents.find((document) => document.frameId === "left")?.axNodes[0]?.nodeId).toBe(
|
||||||
|
"left-node",
|
||||||
|
);
|
||||||
|
expect(documents.find((document) => document.frameId === "right")?.axNodes[0]?.nodeId).toBe(
|
||||||
|
"right-node",
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("uses captured frame parent identity instead of guessing from backend ids", () => {
|
||||||
|
const frameNodes = new Map([
|
||||||
|
["main", [domNode("main", 1)]],
|
||||||
|
["left", [domNode("left", 7)]],
|
||||||
|
["right", [domNode("right", 7)]],
|
||||||
|
["child", [domNode("child", 9)]],
|
||||||
|
]);
|
||||||
|
const capture = captured(frameNodes);
|
||||||
|
capture.frameOwnerBackendNodeIds = new Map([["child", 7]]);
|
||||||
|
capture.frameParentIds = new Map([["child", "left"]]);
|
||||||
|
const frames = ["main", "left", "right", "child"].map((frameId) => ({
|
||||||
|
frame: { frameId, target: { tabId: 4 } },
|
||||||
|
nodes: [],
|
||||||
|
}));
|
||||||
|
|
||||||
|
const documents = buildFrameDocuments(null, frames, capture);
|
||||||
|
|
||||||
|
expect(documents.find((document) => document.frameId === "child")).toMatchObject({
|
||||||
|
parentFrameId: "left",
|
||||||
|
ownerBackendNodeId: 7,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -5,7 +5,9 @@
|
|||||||
// `styles` columns are [position, pointer-events, cursor] in that order.
|
// `styles` columns are [position, pointer-events, cursor] in that order.
|
||||||
|
|
||||||
import type { Rect, Viewport } from "@browser-skill/vom";
|
import type { Rect, Viewport } from "@browser-skill/vom";
|
||||||
|
import { evaluateHoverTrigger } from "@/lib/hover-trigger-policy";
|
||||||
import { isOverlayHostNode, OVERLAY_HOST_SELECTOR } from "../../lib/overlay-bridge";
|
import { isOverlayHostNode, OVERLAY_HOST_SELECTOR } from "../../lib/overlay-bridge";
|
||||||
|
import { childFrameProjection, type GeometryProjection, projectRectToViewport } from "../geometry";
|
||||||
import type { CdpRunner } from "../shared";
|
import type { CdpRunner } from "../shared";
|
||||||
|
|
||||||
const REQUESTED_STYLES = ["position", "pointer-events", "cursor"] as const;
|
const REQUESTED_STYLES = ["position", "pointer-events", "cursor"] as const;
|
||||||
@@ -16,13 +18,14 @@ const STYLE_COL = Object.fromEntries(
|
|||||||
export interface CapturedNode {
|
export interface CapturedNode {
|
||||||
backendNodeId: number;
|
backendNodeId: number;
|
||||||
parentBackendNodeId: number | null;
|
parentBackendNodeId: number | null;
|
||||||
|
frameId?: string;
|
||||||
/** Owning iframe backend node id; `null` for the top-level document. */
|
/** Owning iframe backend node id; `null` for the top-level document. */
|
||||||
ownerFrameBackendNodeId?: number | null;
|
ownerFrameBackendNodeId?: number | null;
|
||||||
tag: string;
|
tag: string;
|
||||||
attrs: Record<string, string>;
|
attrs: Record<string, string>;
|
||||||
/** Top-level viewport-relative CSS px, clipped to the owning frame viewport. */
|
/** Top-level viewport-relative CSS px, clipped to the owning frame viewport. */
|
||||||
rect: Rect | null;
|
rect: Rect | null;
|
||||||
/** Frame-local viewport-relative CSS px before top-level translation/clipping. */
|
/** Frame-local viewport-relative CSS px before top-level projection. */
|
||||||
localRect?: Rect | null;
|
localRect?: Rect | null;
|
||||||
paintOrder: number;
|
paintOrder: number;
|
||||||
position: string;
|
position: string;
|
||||||
@@ -45,15 +48,22 @@ export interface CapturedNode {
|
|||||||
export type CapturedIframeNodes = Map<number, CapturedNode[]>;
|
export type CapturedIframeNodes = Map<number, CapturedNode[]>;
|
||||||
|
|
||||||
export interface CapturedSurfaceProbe {
|
export interface CapturedSurfaceProbe {
|
||||||
triggerLabel: string;
|
triggerBackendNodeId: number;
|
||||||
|
triggerPoint?: { x: number; y: number };
|
||||||
triggerAction: "hover" | "focus" | string;
|
triggerAction: "hover" | "focus" | string;
|
||||||
subItems: string[];
|
subItems: string[];
|
||||||
|
confidence?: "high" | "medium" | "low";
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface CapturedViewModel {
|
export interface CapturedViewModel {
|
||||||
nodes: CapturedNode[];
|
nodes: CapturedNode[];
|
||||||
viewport: Viewport;
|
viewport: Viewport;
|
||||||
iframeNodes: CapturedIframeNodes;
|
iframeNodes: CapturedIframeNodes;
|
||||||
|
frameNodes?: Map<string, CapturedNode[]>;
|
||||||
|
frameOwnerBackendNodeIds?: Map<string, number>;
|
||||||
|
/** Explicit DOMSnapshot frame ancestry; never inferred from backend node ids. */
|
||||||
|
frameParentIds?: Map<string, string>;
|
||||||
|
rootFrameId?: string;
|
||||||
surfaceProbes?: CapturedSurfaceProbe[];
|
surfaceProbes?: CapturedSurfaceProbe[];
|
||||||
/** Backend node ids belonging to the agent overlay host + its shadow subtree. */
|
/** Backend node ids belonging to the agent overlay host + its shadow subtree. */
|
||||||
excludedBackendNodeIds: Set<number>;
|
excludedBackendNodeIds: Set<number>;
|
||||||
@@ -61,6 +71,26 @@ export interface CapturedViewModel {
|
|||||||
|
|
||||||
export interface CaptureViewModelOptions {
|
export interface CaptureViewModelOptions {
|
||||||
conditionalSurfaceProbe?: boolean;
|
conditionalSurfaceProbe?: boolean;
|
||||||
|
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
|
||||||
|
signal?: AbortSignal;
|
||||||
|
}
|
||||||
|
|
||||||
|
function captureAbortError(): Error {
|
||||||
|
const error = new Error("observation aborted");
|
||||||
|
error.name = "AbortError";
|
||||||
|
return error;
|
||||||
|
}
|
||||||
|
|
||||||
|
function isAbortError(error: unknown): boolean {
|
||||||
|
return (
|
||||||
|
typeof error === "object" &&
|
||||||
|
error !== null &&
|
||||||
|
(error as { name?: string }).name === "AbortError"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function throwIfAborted(signal: AbortSignal | undefined): void {
|
||||||
|
if (signal?.aborted) throw captureAbortError();
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Sparse array format Chrome uses for infrequently-set per-node fields. */
|
/** Sparse array format Chrome uses for infrequently-set per-node fields. */
|
||||||
@@ -69,7 +99,12 @@ interface SparseArray {
|
|||||||
value: number[];
|
value: number[];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
interface RareBooleanData {
|
||||||
|
index: number[];
|
||||||
|
}
|
||||||
|
|
||||||
interface SnapshotDocument {
|
interface SnapshotDocument {
|
||||||
|
frameId?: string | number;
|
||||||
scrollOffsetX?: number;
|
scrollOffsetX?: number;
|
||||||
scrollOffsetY?: number;
|
scrollOffsetY?: number;
|
||||||
nodes?: {
|
nodes?: {
|
||||||
@@ -84,6 +119,10 @@ interface SnapshotDocument {
|
|||||||
nodeValue?: number[];
|
nodeValue?: number[];
|
||||||
/** Maps node array index → index into `documents[]` for frame content. */
|
/** Maps node array index → index into `documents[]` for frame content. */
|
||||||
contentDocumentIndex?: SparseArray;
|
contentDocumentIndex?: SparseArray;
|
||||||
|
inputValue?: SparseArray;
|
||||||
|
textValue?: SparseArray;
|
||||||
|
inputChecked?: RareBooleanData;
|
||||||
|
optionSelected?: RareBooleanData;
|
||||||
};
|
};
|
||||||
layout?: {
|
layout?: {
|
||||||
nodeIndex?: number[];
|
nodeIndex?: number[];
|
||||||
@@ -93,6 +132,12 @@ interface SnapshotDocument {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function snapshotFrameId(document: SnapshotDocument, strings: string[]): string | undefined {
|
||||||
|
if (typeof document.frameId === "string") return document.frameId || undefined;
|
||||||
|
if (typeof document.frameId === "number") return str(strings, document.frameId) || undefined;
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
|
||||||
interface SnapshotReply {
|
interface SnapshotReply {
|
||||||
strings?: string[];
|
strings?: string[];
|
||||||
documents?: SnapshotDocument[];
|
documents?: SnapshotDocument[];
|
||||||
@@ -112,6 +157,25 @@ function isFormControlTag(tag: string): boolean {
|
|||||||
return tag === "input" || tag === "textarea" || tag === "select";
|
return tag === "input" || tag === "textarea" || tag === "select";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function isSensitiveFormControl(node: Pick<CapturedNode, "tag" | "attrs">): boolean {
|
||||||
|
return node.tag === "input" && (node.attrs.type ?? "").toLowerCase() === "password";
|
||||||
|
}
|
||||||
|
|
||||||
|
function snapshotFormState(
|
||||||
|
value: string | undefined,
|
||||||
|
defaultValue: string,
|
||||||
|
hasDefaultValue: boolean,
|
||||||
|
sensitive: boolean,
|
||||||
|
): CapturedNode["formState"] {
|
||||||
|
if (sensitive) {
|
||||||
|
if (value === undefined && !hasDefaultValue) return undefined;
|
||||||
|
return (value ?? defaultValue) === "" ? "empty" : "filled";
|
||||||
|
}
|
||||||
|
if (value === undefined) return undefined;
|
||||||
|
if (value === "") return "empty";
|
||||||
|
return value === defaultValue ? "default" : "filled";
|
||||||
|
}
|
||||||
|
|
||||||
const MAX_FORM_ENRICH_CONTROLS = 250;
|
const MAX_FORM_ENRICH_CONTROLS = 250;
|
||||||
|
|
||||||
interface CapturedFormState {
|
interface CapturedFormState {
|
||||||
@@ -143,9 +207,8 @@ function formStateBatchExpression(maxControls: number): string {
|
|||||||
return {
|
return {
|
||||||
state,
|
state,
|
||||||
sensitive,
|
sensitive,
|
||||||
defaultValue,
|
|
||||||
placeholder,
|
placeholder,
|
||||||
...(sensitive ? {} : { value: rawValue }),
|
...(sensitive ? {} : { value: rawValue, defaultValue }),
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
const collect = (doc) => {
|
const collect = (doc) => {
|
||||||
@@ -187,10 +250,14 @@ function applyFormStates(nodes: CapturedNode[], states: CapturedFormState[]): vo
|
|||||||
index += 1;
|
index += 1;
|
||||||
if (!state) continue;
|
if (!state) continue;
|
||||||
node.formState = state.state;
|
node.formState = state.state;
|
||||||
node.formDefaultValue = state.defaultValue ?? "";
|
|
||||||
node.formPlaceholder = state.placeholder ?? "";
|
node.formPlaceholder = state.placeholder ?? "";
|
||||||
if (!state.sensitive && state.value !== undefined) {
|
if (state.sensitive || isSensitiveFormControl(node)) {
|
||||||
node.formValue = state.value;
|
delete node.formValue;
|
||||||
|
delete node.formDefaultValue;
|
||||||
|
delete node.attrs.value;
|
||||||
|
} else {
|
||||||
|
node.formDefaultValue = state.defaultValue ?? "";
|
||||||
|
if (state.value !== undefined) node.formValue = state.value;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -199,21 +266,25 @@ async function enrichFormControlStates(
|
|||||||
cdp: CdpRunner,
|
cdp: CdpRunner,
|
||||||
tabId: number,
|
tabId: number,
|
||||||
frameNodeGroups: CapturedNode[][],
|
frameNodeGroups: CapturedNode[][],
|
||||||
|
signal?: AbortSignal,
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
const hasControls = frameNodeGroups.some((nodes) =>
|
const hasControls = frameNodeGroups.some((nodes) =>
|
||||||
nodes.some((node) => isFormControlTag(node.tag)),
|
nodes.some((node) => isFormControlTag(node.tag)),
|
||||||
);
|
);
|
||||||
if (!hasControls) return;
|
if (!hasControls) return;
|
||||||
try {
|
try {
|
||||||
|
throwIfAborted(signal);
|
||||||
const result = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
|
const result = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
|
||||||
expression: formStateBatchExpression(MAX_FORM_ENRICH_CONTROLS),
|
expression: formStateBatchExpression(MAX_FORM_ENRICH_CONTROLS),
|
||||||
returnByValue: true,
|
returnByValue: true,
|
||||||
});
|
});
|
||||||
|
throwIfAborted(signal);
|
||||||
const frameStates = flattenFrameFormStates(runtimeValue<CapturedFrameFormState>(result));
|
const frameStates = flattenFrameFormStates(runtimeValue<CapturedFrameFormState>(result));
|
||||||
for (let i = 0; i < frameNodeGroups.length; i += 1) {
|
for (let i = 0; i < frameNodeGroups.length; i += 1) {
|
||||||
applyFormStates(frameNodeGroups[i], frameStates[i] ?? []);
|
applyFormStates(frameNodeGroups[i], frameStates[i] ?? []);
|
||||||
}
|
}
|
||||||
} catch {
|
} catch (error) {
|
||||||
|
if (isAbortError(error)) throw error;
|
||||||
// Best-effort enrichment. DOMSnapshot/AX data still carries the nodes.
|
// Best-effort enrichment. DOMSnapshot/AX data still carries the nodes.
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -225,11 +296,12 @@ interface RuntimeEvaluateReply {
|
|||||||
}
|
}
|
||||||
|
|
||||||
interface HoverCandidate {
|
interface HoverCandidate {
|
||||||
triggerSel: string;
|
backendNodeId: number;
|
||||||
affectedSub: string;
|
label?: string;
|
||||||
label: string;
|
|
||||||
x: number;
|
x: number;
|
||||||
y: number;
|
y: number;
|
||||||
|
score: number;
|
||||||
|
reasons: string[];
|
||||||
}
|
}
|
||||||
|
|
||||||
interface CdpDomNode {
|
interface CdpDomNode {
|
||||||
@@ -244,9 +316,9 @@ interface ParseDocumentResult {
|
|||||||
}
|
}
|
||||||
|
|
||||||
interface FrameContext {
|
interface FrameContext {
|
||||||
|
frameId?: string;
|
||||||
ownerFrameBackendNodeId: number | null;
|
ownerFrameBackendNodeId: number | null;
|
||||||
originInTop: { x: number; y: number };
|
projection: GeometryProjection | null;
|
||||||
clipRectInTop: Rect;
|
|
||||||
scrollX: number;
|
scrollX: number;
|
||||||
scrollY: number;
|
scrollY: number;
|
||||||
}
|
}
|
||||||
@@ -265,38 +337,6 @@ function devicePixelRatio(metrics: LayoutMetricsReply): number {
|
|||||||
return dpr >= 1 ? dpr : 1;
|
return dpr >= 1 ? dpr : 1;
|
||||||
}
|
}
|
||||||
|
|
||||||
function viewportRect(viewport: Viewport): Rect {
|
|
||||||
return { x: 0, y: 0, w: Math.max(0, viewport.width), h: Math.max(0, viewport.height) };
|
|
||||||
}
|
|
||||||
|
|
||||||
function intersectRects(a: Rect, b: Rect): Rect | null {
|
|
||||||
const x1 = Math.max(a.x, b.x);
|
|
||||||
const y1 = Math.max(a.y, b.y);
|
|
||||||
const x2 = Math.min(a.x + a.w, b.x + b.w);
|
|
||||||
const y2 = Math.min(a.y + a.h, b.y + b.h);
|
|
||||||
if (x2 <= x1 || y2 <= y1) return null;
|
|
||||||
return { x: x1, y: y1, w: x2 - x1, h: y2 - y1 };
|
|
||||||
}
|
|
||||||
|
|
||||||
function rectInTop(localRect: Rect, context: FrameContext): Rect | null {
|
|
||||||
const translated = {
|
|
||||||
x: context.originInTop.x + localRect.x,
|
|
||||||
y: context.originInTop.y + localRect.y,
|
|
||||||
w: localRect.w,
|
|
||||||
h: localRect.h,
|
|
||||||
};
|
|
||||||
return intersectRects(translated, context.clipRectInTop);
|
|
||||||
}
|
|
||||||
|
|
||||||
function unclipRectInTop(localRect: Rect, context: FrameContext): Rect {
|
|
||||||
return {
|
|
||||||
x: context.originInTop.x + localRect.x,
|
|
||||||
y: context.originInTop.y + localRect.y,
|
|
||||||
w: localRect.w,
|
|
||||||
h: localRect.h,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
function sparseIndexMap(sparse: SparseArray | undefined): Map<number, number> {
|
function sparseIndexMap(sparse: SparseArray | undefined): Map<number, number> {
|
||||||
const out = new Map<number, number>();
|
const out = new Map<number, number>();
|
||||||
if (!sparse?.index || !sparse.value) return out;
|
if (!sparse?.index || !sparse.value) return out;
|
||||||
@@ -322,17 +362,19 @@ function collectBackendIdsFromDomNode(node: CdpDomNode | undefined, out: Set<num
|
|||||||
}
|
}
|
||||||
|
|
||||||
const MAX_HOVER_PROBE_MS = 2_000;
|
const MAX_HOVER_PROBE_MS = 2_000;
|
||||||
const MAX_HOVER_TRIGGERS = 8;
|
const MAX_HOVER_TRIGGERS = 6;
|
||||||
const HOVER_SETTLE_MS = 200;
|
const MAX_HOVER_SURFACES = 3;
|
||||||
|
const HOVER_SETTLE_MS = 300;
|
||||||
|
const MAX_HOVER_SUB_ITEMS = 12;
|
||||||
|
|
||||||
function runtimeValue<T>(reply: RuntimeEvaluateReply): T | undefined {
|
function runtimeValue<T>(reply: RuntimeEvaluateReply): T | undefined {
|
||||||
return reply.result?.value as T | undefined;
|
return reply.result?.value as T | undefined;
|
||||||
}
|
}
|
||||||
|
|
||||||
function hoverCssScanExpression(): string {
|
function hoverCssTriggerScanExpression(): string {
|
||||||
return `(() => {
|
return `(() => {
|
||||||
const visibilityProps = ["display", "visibility", "opacity", "maxHeight", "height", "overflow"];
|
const visibilityProps = ["display", "visibility", "opacity", "maxHeight", "height", "overflow"];
|
||||||
const pairs = [];
|
const centres = [];
|
||||||
const seenRules = new Set();
|
const seenRules = new Set();
|
||||||
for (const sheet of Array.from(document.styleSheets)) {
|
for (const sheet of Array.from(document.styleSheets)) {
|
||||||
let rules;
|
let rules;
|
||||||
@@ -348,95 +390,90 @@ function hoverCssScanExpression(): string {
|
|||||||
const hoverIndex = part.indexOf(":hover");
|
const hoverIndex = part.indexOf(":hover");
|
||||||
if (hoverIndex < 0) continue;
|
if (hoverIndex < 0) continue;
|
||||||
const triggerSel = part.slice(0, hoverIndex).trim();
|
const triggerSel = part.slice(0, hoverIndex).trim();
|
||||||
const affectedSub = part.slice(hoverIndex + 6).trim();
|
|
||||||
if (!triggerSel) continue;
|
if (!triggerSel) continue;
|
||||||
const key = triggerSel + "||" + affectedSub;
|
if (seenRules.has(triggerSel)) continue;
|
||||||
if (seenRules.has(key)) continue;
|
seenRules.add(triggerSel);
|
||||||
seenRules.add(key);
|
let elements;
|
||||||
pairs.push({ triggerSel, affectedSub });
|
try { elements = Array.from(document.querySelectorAll(triggerSel)); } catch { continue; }
|
||||||
|
for (const el of elements) {
|
||||||
|
if (!(el instanceof HTMLElement)) continue;
|
||||||
|
const rect = el.getBoundingClientRect();
|
||||||
|
const style = getComputedStyle(el);
|
||||||
|
if (rect.width <= 0 || rect.height <= 0 || style.visibility === "hidden" || style.display === "none" || style.pointerEvents === "none") continue;
|
||||||
|
centres.push({ x: rect.left + rect.width / 2, y: rect.top + rect.height / 2 });
|
||||||
|
if (centres.length >= 24) return centres;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
return centres;
|
||||||
const candidates = [];
|
|
||||||
const seenLabels = new Set();
|
|
||||||
const isUnsafeTrigger = (el) => {
|
|
||||||
if (!(el instanceof HTMLElement)) return true;
|
|
||||||
const tag = el.tagName.toLowerCase();
|
|
||||||
if (["input", "textarea", "select", "option"].includes(tag)) return true;
|
|
||||||
if (el.isContentEditable) return true;
|
|
||||||
if (el.hasAttribute("disabled") || el.hasAttribute("inert")) return true;
|
|
||||||
if ((el.getAttribute("aria-disabled") || "").toLowerCase() === "true") return true;
|
|
||||||
return false;
|
|
||||||
};
|
|
||||||
for (const pair of pairs) {
|
|
||||||
let elements;
|
|
||||||
try { elements = Array.from(document.querySelectorAll(pair.triggerSel)); } catch { continue; }
|
|
||||||
for (const el of elements) {
|
|
||||||
if (!(el instanceof Element)) continue;
|
|
||||||
if (isUnsafeTrigger(el)) continue;
|
|
||||||
const rect = el.getBoundingClientRect();
|
|
||||||
const style = getComputedStyle(el);
|
|
||||||
if (rect.width <= 0 || rect.height <= 0 || style.visibility === "hidden" || style.display === "none" || style.pointerEvents === "none") continue;
|
|
||||||
const label = (el.textContent || "").replace(/\\s+/g, " ").trim();
|
|
||||||
if (!label || seenLabels.has(label)) continue;
|
|
||||||
seenLabels.add(label);
|
|
||||||
candidates.push({
|
|
||||||
triggerSel: pair.triggerSel,
|
|
||||||
affectedSub: pair.affectedSub,
|
|
||||||
label,
|
|
||||||
x: rect.left + rect.width / 2,
|
|
||||||
y: rect.top + rect.height / 2,
|
|
||||||
});
|
|
||||||
if (candidates.length >= ${MAX_HOVER_TRIGGERS}) return candidates;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return candidates;
|
|
||||||
})()`;
|
})()`;
|
||||||
}
|
}
|
||||||
|
|
||||||
function hoverCollectExpression(candidate: HoverCandidate): string {
|
interface HoverRuntimeItem {
|
||||||
return `(() => {
|
text: string;
|
||||||
const triggerSel = ${JSON.stringify(candidate.triggerSel)};
|
role: string;
|
||||||
const rawAffectedSub = ${JSON.stringify(candidate.affectedSub)};
|
tag: string;
|
||||||
const x = ${JSON.stringify(candidate.x)};
|
x: number;
|
||||||
const y = ${JSON.stringify(candidate.y)};
|
y: number;
|
||||||
const hit = document.elementFromPoint(x, y);
|
}
|
||||||
const trigger = hit instanceof Element ? hit.closest(triggerSel) : null;
|
|
||||||
if (!trigger) return [];
|
|
||||||
|
|
||||||
const affectedSub = rawAffectedSub.replace(/^[>+~]\\s*/, "").trim();
|
function hoverStateExpression(): string {
|
||||||
let targets = [];
|
return `(() => {
|
||||||
if (!affectedSub) {
|
|
||||||
targets = [trigger];
|
|
||||||
} else {
|
|
||||||
try { targets = Array.from(trigger.querySelectorAll(affectedSub)); } catch { targets = []; }
|
|
||||||
}
|
|
||||||
const items = [];
|
const items = [];
|
||||||
const seen = new Set();
|
const seen = new Set();
|
||||||
const push = (value) => {
|
const selectors = [
|
||||||
const text = String(value || "").replace(/\\s+/g, " ").trim();
|
"a",
|
||||||
|
"button",
|
||||||
|
"[role='menuitem']",
|
||||||
|
"[role='menuitemcheckbox']",
|
||||||
|
"[role='menuitemradio']",
|
||||||
|
"[role='option']",
|
||||||
|
"[role='tab']",
|
||||||
|
"[role='link']",
|
||||||
|
"[role='button']"
|
||||||
|
].join(",");
|
||||||
|
const push = (el) => {
|
||||||
|
if (!(el instanceof HTMLElement)) return;
|
||||||
|
const style = getComputedStyle(el);
|
||||||
|
const rect = el.getBoundingClientRect();
|
||||||
|
if (rect.width <= 0 || rect.height <= 0) return;
|
||||||
|
if (style.display === "none" || style.visibility === "hidden" || style.opacity === "0") return;
|
||||||
|
const text = String(
|
||||||
|
el.getAttribute("aria-label") ||
|
||||||
|
el.getAttribute("title") ||
|
||||||
|
el.textContent ||
|
||||||
|
""
|
||||||
|
).replace(/\\s+/g, " ").trim();
|
||||||
if (!text || seen.has(text)) return;
|
if (!text || seen.has(text)) return;
|
||||||
seen.add(text);
|
seen.add(text);
|
||||||
items.push(text);
|
items.push({
|
||||||
|
text,
|
||||||
|
role: (el.getAttribute("role") || "").toLowerCase(),
|
||||||
|
tag: el.tagName.toLowerCase(),
|
||||||
|
x: rect.left,
|
||||||
|
y: rect.top,
|
||||||
|
});
|
||||||
};
|
};
|
||||||
for (const target of targets) {
|
for (const el of Array.from(document.querySelectorAll(selectors))) push(el);
|
||||||
if (!(target instanceof Element)) continue;
|
return items.slice(0, 400);
|
||||||
const style = getComputedStyle(target);
|
|
||||||
if (style.display === "none" || style.visibility === "hidden" || style.opacity === "0") continue;
|
|
||||||
const clickable = target.querySelectorAll('a, button, [role="menuitem"], [role="option"]');
|
|
||||||
if (clickable.length > 0) {
|
|
||||||
for (const child of Array.from(clickable)) push(child.textContent);
|
|
||||||
} else {
|
|
||||||
push(target.textContent);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return items.slice(0, 12);
|
|
||||||
})()`;
|
})()`;
|
||||||
}
|
}
|
||||||
|
|
||||||
async function wait(ms: number): Promise<void> {
|
async function wait(ms: number, signal?: AbortSignal): Promise<void> {
|
||||||
await new Promise((resolve) => setTimeout(resolve, ms));
|
throwIfAborted(signal);
|
||||||
|
await new Promise<void>((resolve, reject) => {
|
||||||
|
const timer = setTimeout(() => {
|
||||||
|
signal?.removeEventListener("abort", onAbort);
|
||||||
|
resolve();
|
||||||
|
}, ms);
|
||||||
|
const onAbort = () => {
|
||||||
|
clearTimeout(timer);
|
||||||
|
signal?.removeEventListener("abort", onAbort);
|
||||||
|
reject(captureAbortError());
|
||||||
|
};
|
||||||
|
signal?.addEventListener("abort", onAbort, { once: true });
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
async function clearHover(cdp: CdpRunner, tabId: number): Promise<void> {
|
async function clearHover(cdp: CdpRunner, tabId: number): Promise<void> {
|
||||||
@@ -453,48 +490,232 @@ async function clearHover(cdp: CdpRunner, tabId: number): Promise<void> {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
async function probeHoverSurfaces(cdp: CdpRunner, tabId: number): Promise<CapturedSurfaceProbe[]> {
|
function capturedText(node: CapturedNode): string | undefined {
|
||||||
|
const value =
|
||||||
|
node.attrs["aria-label"] ?? node.attrs.title ?? node.attrs.alt ?? node.textContent ?? "";
|
||||||
|
const clean = value.replace(/\s+/g, " ").trim();
|
||||||
|
return clean || undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
function hasGraphicDescendant(
|
||||||
|
node: CapturedNode,
|
||||||
|
childrenByParentId: Map<number, CapturedNode[]>,
|
||||||
|
depth = 0,
|
||||||
|
): boolean {
|
||||||
|
if (depth > 3) return false;
|
||||||
|
for (const child of childrenByParentId.get(node.backendNodeId) ?? []) {
|
||||||
|
const tag = child.tag.toLowerCase();
|
||||||
|
if (["img", "svg", "use", "path", "i"].includes(tag)) return true;
|
||||||
|
if (hasGraphicDescendant(child, childrenByParentId, depth + 1)) return true;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
function roleOf(node: CapturedNode): string {
|
||||||
|
return (node.attrs.role ?? "").toLowerCase();
|
||||||
|
}
|
||||||
|
|
||||||
|
function scoreHoverCandidate(
|
||||||
|
node: CapturedNode,
|
||||||
|
childrenByParentId: Map<number, CapturedNode[]>,
|
||||||
|
cssHoverPoints: Array<{ x: number; y: number }>,
|
||||||
|
): HoverCandidate | null {
|
||||||
|
const rect = node.rect;
|
||||||
|
if (!rect) return null;
|
||||||
|
const label = capturedText(node);
|
||||||
|
const cssHoverMatch = cssHoverPoints.some(
|
||||||
|
(point) =>
|
||||||
|
point.x >= rect.x &&
|
||||||
|
point.x <= rect.x + rect.w &&
|
||||||
|
point.y >= rect.y &&
|
||||||
|
point.y <= rect.y + rect.h,
|
||||||
|
);
|
||||||
|
const decision = evaluateHoverTrigger({
|
||||||
|
tag: node.tag,
|
||||||
|
role: roleOf(node),
|
||||||
|
label,
|
||||||
|
attrs: node.attrs,
|
||||||
|
rect,
|
||||||
|
cursor: node.cursor,
|
||||||
|
pointerEvents: node.pointerEvents,
|
||||||
|
hasGraphicDescendant: hasGraphicDescendant(node, childrenByParentId),
|
||||||
|
cssHoverMatch,
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!decision.eligible) return null;
|
||||||
|
return {
|
||||||
|
backendNodeId: node.backendNodeId,
|
||||||
|
label,
|
||||||
|
x: rect.x + rect.w / 2,
|
||||||
|
y: rect.y + rect.h / 2,
|
||||||
|
score: decision.score,
|
||||||
|
reasons: decision.reasons,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildHoverCandidates(
|
||||||
|
nodes: CapturedNode[],
|
||||||
|
cssHoverPoints: Array<{ x: number; y: number }>,
|
||||||
|
): HoverCandidate[] {
|
||||||
|
const childrenByParentId = new Map<number, CapturedNode[]>();
|
||||||
|
const parentByBackendId = new Map<number, number | null>();
|
||||||
|
for (const node of nodes) {
|
||||||
|
parentByBackendId.set(node.backendNodeId, node.parentBackendNodeId);
|
||||||
|
if (node.parentBackendNodeId === null) continue;
|
||||||
|
const children = childrenByParentId.get(node.parentBackendNodeId) ?? [];
|
||||||
|
children.push(node);
|
||||||
|
childrenByParentId.set(node.parentBackendNodeId, children);
|
||||||
|
}
|
||||||
|
|
||||||
|
const candidates = nodes
|
||||||
|
.map((node) => scoreHoverCandidate(node, childrenByParentId, cssHoverPoints))
|
||||||
|
.filter((candidate): candidate is HoverCandidate => candidate !== null)
|
||||||
|
.sort((a, b) => b.score - a.score);
|
||||||
|
|
||||||
|
const deduped: HoverCandidate[] = [];
|
||||||
|
const seen = new Set<number>();
|
||||||
|
for (const candidate of candidates) {
|
||||||
|
if (seen.has(candidate.backendNodeId)) continue;
|
||||||
|
if (deduped.some((existing) => sameHoverCluster(existing, candidate, parentByBackendId))) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
seen.add(candidate.backendNodeId);
|
||||||
|
deduped.push(candidate);
|
||||||
|
if (deduped.length >= MAX_HOVER_TRIGGERS) break;
|
||||||
|
}
|
||||||
|
return deduped;
|
||||||
|
}
|
||||||
|
|
||||||
|
function sameHoverCluster(
|
||||||
|
a: HoverCandidate,
|
||||||
|
b: HoverCandidate,
|
||||||
|
parentByBackendId: Map<number, number | null>,
|
||||||
|
): boolean {
|
||||||
|
if (Math.hypot(a.x - b.x, a.y - b.y) <= 8) return true;
|
||||||
|
return (
|
||||||
|
isBackendAncestor(a.backendNodeId, b.backendNodeId, parentByBackendId) ||
|
||||||
|
isBackendAncestor(b.backendNodeId, a.backendNodeId, parentByBackendId)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function isBackendAncestor(
|
||||||
|
ancestorId: number,
|
||||||
|
nodeId: number,
|
||||||
|
parentByBackendId: Map<number, number | null>,
|
||||||
|
): boolean {
|
||||||
|
let parentId = parentByBackendId.get(nodeId);
|
||||||
|
let guard = 0;
|
||||||
|
while (parentId !== null && parentId !== undefined && guard < parentByBackendId.size) {
|
||||||
|
if (parentId === ancestorId) return true;
|
||||||
|
parentId = parentByBackendId.get(parentId);
|
||||||
|
guard += 1;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
function diffHoverItems(before: HoverRuntimeItem[], after: HoverRuntimeItem[]): string[] {
|
||||||
|
const beforeKeys = new Set(before.map((item) => item.text.toLowerCase()));
|
||||||
|
const out: string[] = [];
|
||||||
|
const seen = new Set<string>();
|
||||||
|
for (const item of after) {
|
||||||
|
const text = item.text.replace(/\s+/g, " ").trim();
|
||||||
|
const key = text.toLowerCase();
|
||||||
|
if (!text || beforeKeys.has(key) || seen.has(key)) continue;
|
||||||
|
seen.add(key);
|
||||||
|
out.push(text);
|
||||||
|
if (out.length >= MAX_HOVER_SUB_ITEMS) break;
|
||||||
|
}
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
function confidenceForHover(
|
||||||
|
candidate: HoverCandidate,
|
||||||
|
subItems: string[],
|
||||||
|
): "high" | "medium" | "low" {
|
||||||
|
if (subItems.length >= 2 && candidate.score >= 80) return "high";
|
||||||
|
if (subItems.length >= 2 || candidate.score >= 80) return "medium";
|
||||||
|
return "low";
|
||||||
|
}
|
||||||
|
|
||||||
|
async function probeHoverSurfaces(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
tabId: number,
|
||||||
|
nodes: CapturedNode[],
|
||||||
|
options: CaptureViewModelOptions,
|
||||||
|
): Promise<CapturedSurfaceProbe[]> {
|
||||||
const started = Date.now();
|
const started = Date.now();
|
||||||
try {
|
try {
|
||||||
const scan = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
|
throwIfAborted(options.signal);
|
||||||
expression: hoverCssScanExpression(),
|
const cssScan = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
|
||||||
|
expression: hoverCssTriggerScanExpression(),
|
||||||
returnByValue: true,
|
returnByValue: true,
|
||||||
});
|
});
|
||||||
const candidates = runtimeValue<HoverCandidate[]>(scan) ?? [];
|
throwIfAborted(options.signal);
|
||||||
|
const cssHoverPoints = runtimeValue<Array<{ x: number; y: number }>>(cssScan) ?? [];
|
||||||
|
const candidates = buildHoverCandidates(nodes, cssHoverPoints);
|
||||||
|
if (candidates.length === 0) return [];
|
||||||
|
|
||||||
const results: CapturedSurfaceProbe[] = [];
|
const results: CapturedSurfaceProbe[] = [];
|
||||||
const seen = new Set<string>();
|
const seen = new Set<number>();
|
||||||
for (const candidate of candidates.slice(0, MAX_HOVER_TRIGGERS)) {
|
throwIfAborted(options.signal);
|
||||||
if (Date.now() - started > MAX_HOVER_PROBE_MS) break;
|
await options.hoverProbeBypassOverlay?.(tabId, true).catch(() => undefined);
|
||||||
if (!candidate.label || seen.has(candidate.label)) continue;
|
try {
|
||||||
try {
|
throwIfAborted(options.signal);
|
||||||
await cdp.send(tabId, "Input.dispatchMouseEvent", {
|
for (const candidate of candidates.slice(0, MAX_HOVER_TRIGGERS)) {
|
||||||
type: "mouseMoved",
|
throwIfAborted(options.signal);
|
||||||
x: candidate.x,
|
if (Date.now() - started > MAX_HOVER_PROBE_MS) break;
|
||||||
y: candidate.y,
|
if (results.length >= MAX_HOVER_SURFACES) break;
|
||||||
});
|
if (seen.has(candidate.backendNodeId)) continue;
|
||||||
await wait(HOVER_SETTLE_MS);
|
try {
|
||||||
const collected = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
|
await clearHover(cdp, tabId);
|
||||||
expression: hoverCollectExpression(candidate),
|
throwIfAborted(options.signal);
|
||||||
returnByValue: true,
|
await wait(HOVER_SETTLE_MS, options.signal);
|
||||||
});
|
const baselineReply = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
|
||||||
const subItems = (runtimeValue<string[]>(collected) ?? [])
|
expression: hoverStateExpression(),
|
||||||
.map((item) => item.replace(/\s+/g, " ").trim())
|
returnByValue: true,
|
||||||
.filter(Boolean);
|
});
|
||||||
if (subItems.length === 0) continue;
|
throwIfAborted(options.signal);
|
||||||
seen.add(candidate.label);
|
const baselineItems = runtimeValue<HoverRuntimeItem[]>(baselineReply) ?? [];
|
||||||
results.push({
|
|
||||||
triggerLabel: candidate.label,
|
throwIfAborted(options.signal);
|
||||||
triggerAction: "hover",
|
await cdp.send(tabId, "Input.dispatchMouseEvent", {
|
||||||
subItems,
|
type: "mouseMoved",
|
||||||
});
|
x: candidate.x,
|
||||||
} catch {
|
y: candidate.y,
|
||||||
continue;
|
});
|
||||||
} finally {
|
throwIfAborted(options.signal);
|
||||||
await clearHover(cdp, tabId);
|
await wait(HOVER_SETTLE_MS, options.signal);
|
||||||
|
const collected = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
|
||||||
|
expression: hoverStateExpression(),
|
||||||
|
returnByValue: true,
|
||||||
|
});
|
||||||
|
throwIfAborted(options.signal);
|
||||||
|
const subItems = diffHoverItems(
|
||||||
|
baselineItems,
|
||||||
|
runtimeValue<HoverRuntimeItem[]>(collected) ?? [],
|
||||||
|
);
|
||||||
|
if (subItems.length === 0) continue;
|
||||||
|
seen.add(candidate.backendNodeId);
|
||||||
|
results.push({
|
||||||
|
triggerBackendNodeId: candidate.backendNodeId,
|
||||||
|
triggerPoint: { x: candidate.x, y: candidate.y },
|
||||||
|
triggerAction: "hover",
|
||||||
|
subItems,
|
||||||
|
confidence: confidenceForHover(candidate, subItems),
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
if (isAbortError(error)) throw error;
|
||||||
|
continue;
|
||||||
|
} finally {
|
||||||
|
await clearHover(cdp, tabId);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
} finally {
|
||||||
|
await options.hoverProbeBypassOverlay?.(tabId, false).catch(() => undefined);
|
||||||
}
|
}
|
||||||
return results;
|
return results;
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
|
if (isAbortError(err)) throw err;
|
||||||
console.debug("[bsk capture] hover surface probe failed", err);
|
console.debug("[bsk capture] hover surface probe failed", err);
|
||||||
return [];
|
return [];
|
||||||
}
|
}
|
||||||
@@ -507,13 +728,16 @@ async function probeHoverSurfaces(cdp: CdpRunner, tabId: number): Promise<Captur
|
|||||||
export async function collectOverlayExcludedBackendIds(
|
export async function collectOverlayExcludedBackendIds(
|
||||||
cdp: CdpRunner,
|
cdp: CdpRunner,
|
||||||
tabId: number,
|
tabId: number,
|
||||||
|
signal?: AbortSignal,
|
||||||
): Promise<Set<number>> {
|
): Promise<Set<number>> {
|
||||||
const excluded = new Set<number>();
|
const excluded = new Set<number>();
|
||||||
try {
|
try {
|
||||||
|
throwIfAborted(signal);
|
||||||
const doc = await cdp.send<{ root?: { nodeId?: number } }>(tabId, "DOM.getDocument", {
|
const doc = await cdp.send<{ root?: { nodeId?: number } }>(tabId, "DOM.getDocument", {
|
||||||
depth: 0,
|
depth: 0,
|
||||||
pierce: true,
|
pierce: true,
|
||||||
});
|
});
|
||||||
|
throwIfAborted(signal);
|
||||||
const rootNodeId = doc.root?.nodeId;
|
const rootNodeId = doc.root?.nodeId;
|
||||||
if (typeof rootNodeId !== "number") return excluded;
|
if (typeof rootNodeId !== "number") return excluded;
|
||||||
|
|
||||||
@@ -521,6 +745,7 @@ export async function collectOverlayExcludedBackendIds(
|
|||||||
nodeId: rootNodeId,
|
nodeId: rootNodeId,
|
||||||
selector: OVERLAY_HOST_SELECTOR,
|
selector: OVERLAY_HOST_SELECTOR,
|
||||||
});
|
});
|
||||||
|
throwIfAborted(signal);
|
||||||
if (typeof found.nodeId !== "number" || found.nodeId === 0) return excluded;
|
if (typeof found.nodeId !== "number" || found.nodeId === 0) return excluded;
|
||||||
|
|
||||||
const described = await cdp.send<{ node?: CdpDomNode }>(tabId, "DOM.describeNode", {
|
const described = await cdp.send<{ node?: CdpDomNode }>(tabId, "DOM.describeNode", {
|
||||||
@@ -528,8 +753,10 @@ export async function collectOverlayExcludedBackendIds(
|
|||||||
depth: -1,
|
depth: -1,
|
||||||
pierce: true,
|
pierce: true,
|
||||||
});
|
});
|
||||||
|
throwIfAborted(signal);
|
||||||
collectBackendIdsFromDomNode(described.node, excluded);
|
collectBackendIdsFromDomNode(described.node, excluded);
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
|
if (isAbortError(err)) throw err;
|
||||||
console.debug("[bsk capture] overlay exclusion fallback failed", err);
|
console.debug("[bsk capture] overlay exclusion fallback failed", err);
|
||||||
}
|
}
|
||||||
return excluded;
|
return excluded;
|
||||||
@@ -590,6 +817,10 @@ function parseDocumentNodes(
|
|||||||
const posCol = STYLE_COL.position;
|
const posCol = STYLE_COL.position;
|
||||||
const peCol = STYLE_COL["pointer-events"];
|
const peCol = STYLE_COL["pointer-events"];
|
||||||
const cursorCol = STYLE_COL.cursor;
|
const cursorCol = STYLE_COL.cursor;
|
||||||
|
const inputValues = sparseIndexMap(dn.inputValue);
|
||||||
|
const textValues = sparseIndexMap(dn.textValue);
|
||||||
|
const checkedInputs = new Set(dn.inputChecked?.index ?? []);
|
||||||
|
const selectedOptions = new Set(dn.optionSelected?.index ?? []);
|
||||||
|
|
||||||
// Collect visible text from #text child nodes so element CapturedNodes
|
// Collect visible text from #text child nodes so element CapturedNodes
|
||||||
// carry a textContent value usable as a button/link label fallback.
|
// carry a textContent value usable as a button/link label fallback.
|
||||||
@@ -640,7 +871,10 @@ function parseDocumentNodes(
|
|||||||
w: b[2] / dpr,
|
w: b[2] / dpr,
|
||||||
h: b[3] / dpr,
|
h: b[3] / dpr,
|
||||||
};
|
};
|
||||||
rect = rectInTop(localRect, context);
|
const bounds = context.projection
|
||||||
|
? projectRectToViewport(localRect, context.projection)
|
||||||
|
: null;
|
||||||
|
rect = bounds ? { x: bounds.x, y: bounds.y, w: bounds.width, h: bounds.height } : null;
|
||||||
}
|
}
|
||||||
paintOrder = dl?.paintOrders?.[li] ?? 0;
|
paintOrder = dl?.paintOrders?.[li] ?? 0;
|
||||||
const styleRow = dl?.styles?.[li] ?? [];
|
const styleRow = dl?.styles?.[li] ?? [];
|
||||||
@@ -655,9 +889,24 @@ function parseDocumentNodes(
|
|||||||
|
|
||||||
const textContent = nodeTextContent.get(n);
|
const textContent = nodeTextContent.get(n);
|
||||||
|
|
||||||
|
const rawFormValueIndex = tag === "textarea" ? textValues.get(n) : inputValues.get(n);
|
||||||
|
const rawFormValue =
|
||||||
|
rawFormValueIndex !== undefined ? str(strings, rawFormValueIndex) : undefined;
|
||||||
|
const sensitive = isSensitiveFormControl({ tag, attrs });
|
||||||
|
const formDefaultValue = attrs.value ?? "";
|
||||||
|
const formValue = sensitive ? undefined : rawFormValue;
|
||||||
|
const formState = snapshotFormState(
|
||||||
|
rawFormValue,
|
||||||
|
formDefaultValue,
|
||||||
|
Object.prototype.hasOwnProperty.call(attrs, "value"),
|
||||||
|
sensitive,
|
||||||
|
);
|
||||||
|
if (sensitive) delete attrs.value;
|
||||||
|
|
||||||
nodes.push({
|
nodes.push({
|
||||||
backendNodeId,
|
backendNodeId,
|
||||||
parentBackendNodeId,
|
parentBackendNodeId,
|
||||||
|
...(context.frameId ? { frameId: context.frameId } : {}),
|
||||||
ownerFrameBackendNodeId: context.ownerFrameBackendNodeId,
|
ownerFrameBackendNodeId: context.ownerFrameBackendNodeId,
|
||||||
tag,
|
tag,
|
||||||
attrs,
|
attrs,
|
||||||
@@ -668,6 +917,16 @@ function parseDocumentNodes(
|
|||||||
pointerEvents,
|
pointerEvents,
|
||||||
cursor,
|
cursor,
|
||||||
textContent,
|
textContent,
|
||||||
|
...(formValue !== undefined ? { formValue } : {}),
|
||||||
|
...(tag === "input" || tag === "textarea"
|
||||||
|
? {
|
||||||
|
formPlaceholder: attrs.placeholder ?? "",
|
||||||
|
...(!sensitive ? { formDefaultValue } : {}),
|
||||||
|
...(formState ? { formState } : {}),
|
||||||
|
}
|
||||||
|
: {}),
|
||||||
|
...(checkedInputs.has(n) ? { formValue: "true", formState: "filled" } : {}),
|
||||||
|
...(selectedOptions.has(n) ? { formValue: attrs.value ?? textContent ?? "" } : {}),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
return { nodes, excludedBackendNodeIds };
|
return { nodes, excludedBackendNodeIds };
|
||||||
@@ -675,6 +934,8 @@ function parseDocumentNodes(
|
|||||||
|
|
||||||
interface ParseFrameDocumentsResult {
|
interface ParseFrameDocumentsResult {
|
||||||
iframeNodes: CapturedIframeNodes;
|
iframeNodes: CapturedIframeNodes;
|
||||||
|
frameOwnerBackendNodeIds: Map<string, number>;
|
||||||
|
frameParentIds: Map<string, string>;
|
||||||
excludedBackendNodeIds: Set<number>;
|
excludedBackendNodeIds: Set<number>;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -688,10 +949,14 @@ function parseChildFrameDocuments(
|
|||||||
visited = new Set<number>(),
|
visited = new Set<number>(),
|
||||||
): ParseFrameDocumentsResult {
|
): ParseFrameDocumentsResult {
|
||||||
const iframeNodes: CapturedIframeNodes = new Map();
|
const iframeNodes: CapturedIframeNodes = new Map();
|
||||||
|
const frameOwnerBackendNodeIds = new Map<string, number>();
|
||||||
|
const frameParentIds = new Map<string, string>();
|
||||||
const excludedBackendNodeIds = new Set<number>();
|
const excludedBackendNodeIds = new Set<number>();
|
||||||
const parentDoc = documents[parentDocIndex];
|
const parentDoc = documents[parentDocIndex];
|
||||||
const cdi = sparseIndexMap(parentDoc?.nodes?.contentDocumentIndex);
|
const cdi = sparseIndexMap(parentDoc?.nodes?.contentDocumentIndex);
|
||||||
if (cdi.size === 0) return { iframeNodes, excludedBackendNodeIds };
|
if (cdi.size === 0) {
|
||||||
|
return { iframeNodes, frameOwnerBackendNodeIds, frameParentIds, excludedBackendNodeIds };
|
||||||
|
}
|
||||||
|
|
||||||
const parentBackendIds = parentDoc?.nodes?.backendNodeId ?? [];
|
const parentBackendIds = parentDoc?.nodes?.backendNodeId ?? [];
|
||||||
const parentNodeByBackendId = new Map(parentNodes.map((node) => [node.backendNodeId, node]));
|
const parentNodeByBackendId = new Map(parentNodes.map((node) => [node.backendNodeId, node]));
|
||||||
@@ -703,19 +968,15 @@ function parseChildFrameDocuments(
|
|||||||
const iframeBackendId = parentBackendIds[nodeArrayIdx];
|
const iframeBackendId = parentBackendIds[nodeArrayIdx];
|
||||||
if (iframeBackendId === undefined) continue;
|
if (iframeBackendId === undefined) continue;
|
||||||
const iframeNode = parentNodeByBackendId.get(iframeBackendId);
|
const iframeNode = parentNodeByBackendId.get(iframeBackendId);
|
||||||
if (!iframeNode?.localRect) continue;
|
const projection =
|
||||||
|
parentContext.projection && iframeNode?.localRect
|
||||||
const iframeRectInTop = unclipRectInTop(iframeNode.localRect, parentContext);
|
? childFrameProjection(parentContext.projection, iframeNode.localRect)
|
||||||
const childClip = intersectRects(iframeRectInTop, parentContext.clipRectInTop);
|
: null;
|
||||||
if (!childClip) {
|
|
||||||
iframeNodes.set(iframeBackendId, []);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
const childContext: FrameContext = {
|
const childContext: FrameContext = {
|
||||||
|
frameId: snapshotFrameId(childDoc, strings),
|
||||||
ownerFrameBackendNodeId: iframeBackendId,
|
ownerFrameBackendNodeId: iframeBackendId,
|
||||||
originInTop: { x: iframeRectInTop.x, y: iframeRectInTop.y },
|
projection,
|
||||||
clipRectInTop: childClip,
|
|
||||||
scrollX: childDoc.scrollOffsetX ?? 0,
|
scrollX: childDoc.scrollOffsetX ?? 0,
|
||||||
scrollY: childDoc.scrollOffsetY ?? 0,
|
scrollY: childDoc.scrollOffsetY ?? 0,
|
||||||
};
|
};
|
||||||
@@ -723,6 +984,10 @@ function parseChildFrameDocuments(
|
|||||||
nextVisited.add(childDocIndex);
|
nextVisited.add(childDocIndex);
|
||||||
const parsed = parseDocumentNodes(childDoc, strings, dpr, childContext);
|
const parsed = parseDocumentNodes(childDoc, strings, dpr, childContext);
|
||||||
iframeNodes.set(iframeBackendId, parsed.nodes);
|
iframeNodes.set(iframeBackendId, parsed.nodes);
|
||||||
|
if (childContext.frameId) {
|
||||||
|
frameOwnerBackendNodeIds.set(childContext.frameId, iframeBackendId);
|
||||||
|
if (parentContext.frameId) frameParentIds.set(childContext.frameId, parentContext.frameId);
|
||||||
|
}
|
||||||
for (const id of parsed.excludedBackendNodeIds) excludedBackendNodeIds.add(id);
|
for (const id of parsed.excludedBackendNodeIds) excludedBackendNodeIds.add(id);
|
||||||
|
|
||||||
const nested = parseChildFrameDocuments(
|
const nested = parseChildFrameDocuments(
|
||||||
@@ -737,10 +1002,16 @@ function parseChildFrameDocuments(
|
|||||||
for (const [nestedFrameId, nestedNodes] of nested.iframeNodes) {
|
for (const [nestedFrameId, nestedNodes] of nested.iframeNodes) {
|
||||||
iframeNodes.set(nestedFrameId, nestedNodes);
|
iframeNodes.set(nestedFrameId, nestedNodes);
|
||||||
}
|
}
|
||||||
|
for (const [frameId, ownerBackendNodeId] of nested.frameOwnerBackendNodeIds) {
|
||||||
|
frameOwnerBackendNodeIds.set(frameId, ownerBackendNodeId);
|
||||||
|
}
|
||||||
|
for (const [frameId, parentFrameId] of nested.frameParentIds) {
|
||||||
|
frameParentIds.set(frameId, parentFrameId);
|
||||||
|
}
|
||||||
for (const id of nested.excludedBackendNodeIds) excludedBackendNodeIds.add(id);
|
for (const id of nested.excludedBackendNodeIds) excludedBackendNodeIds.add(id);
|
||||||
}
|
}
|
||||||
|
|
||||||
return { iframeNodes, excludedBackendNodeIds };
|
return { iframeNodes, frameOwnerBackendNodeIds, frameParentIds, excludedBackendNodeIds };
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function captureViewModel(
|
export async function captureViewModel(
|
||||||
@@ -748,7 +1019,9 @@ export async function captureViewModel(
|
|||||||
tabId: number,
|
tabId: number,
|
||||||
options: CaptureViewModelOptions = {},
|
options: CaptureViewModelOptions = {},
|
||||||
): Promise<CapturedViewModel> {
|
): Promise<CapturedViewModel> {
|
||||||
|
throwIfAborted(options.signal);
|
||||||
const metrics = await cdp.send<LayoutMetricsReply>(tabId, "Page.getLayoutMetrics", {});
|
const metrics = await cdp.send<LayoutMetricsReply>(tabId, "Page.getLayoutMetrics", {});
|
||||||
|
throwIfAborted(options.signal);
|
||||||
const dpr = devicePixelRatio(metrics);
|
const dpr = devicePixelRatio(metrics);
|
||||||
const vpSrc = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
|
const vpSrc = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
|
||||||
const viewport: Viewport = {
|
const viewport: Viewport = {
|
||||||
@@ -759,11 +1032,13 @@ export async function captureViewModel(
|
|||||||
const scrollY = vpSrc.pageY ?? 0;
|
const scrollY = vpSrc.pageY ?? 0;
|
||||||
|
|
||||||
await cdp.send(tabId, "DOMSnapshot.enable", {});
|
await cdp.send(tabId, "DOMSnapshot.enable", {});
|
||||||
|
throwIfAborted(options.signal);
|
||||||
const snap = await cdp.send<SnapshotReply>(tabId, "DOMSnapshot.captureSnapshot", {
|
const snap = await cdp.send<SnapshotReply>(tabId, "DOMSnapshot.captureSnapshot", {
|
||||||
computedStyles: REQUESTED_STYLES,
|
computedStyles: REQUESTED_STYLES,
|
||||||
includePaintOrder: true,
|
includePaintOrder: true,
|
||||||
includeDOMRects: true,
|
includeDOMRects: true,
|
||||||
});
|
});
|
||||||
|
throwIfAborted(options.signal);
|
||||||
|
|
||||||
const strings = snap.strings ?? [];
|
const strings = snap.strings ?? [];
|
||||||
const documents = snap.documents ?? [];
|
const documents = snap.documents ?? [];
|
||||||
@@ -779,9 +1054,13 @@ export async function captureViewModel(
|
|||||||
}
|
}
|
||||||
|
|
||||||
const topContext: FrameContext = {
|
const topContext: FrameContext = {
|
||||||
|
frameId: snapshotFrameId(doc0, strings),
|
||||||
ownerFrameBackendNodeId: null,
|
ownerFrameBackendNodeId: null,
|
||||||
originInTop: { x: 0, y: 0 },
|
projection: {
|
||||||
clipRectInTop: viewportRect(viewport),
|
sourceClips: [],
|
||||||
|
edges: [],
|
||||||
|
topViewport: viewport,
|
||||||
|
},
|
||||||
scrollX,
|
scrollX,
|
||||||
scrollY,
|
scrollY,
|
||||||
};
|
};
|
||||||
@@ -795,9 +1074,35 @@ export async function captureViewModel(
|
|||||||
excludedBackendNodeIds.add(id);
|
excludedBackendNodeIds.add(id);
|
||||||
}
|
}
|
||||||
|
|
||||||
await enrichFormControlStates(cdp, tabId, [nodes, ...iframeNodes.values()]);
|
await enrichFormControlStates(cdp, tabId, [nodes, ...iframeNodes.values()], options.signal);
|
||||||
|
throwIfAborted(options.signal);
|
||||||
|
|
||||||
const surfaceProbes = options.conditionalSurfaceProbe ? await probeHoverSurfaces(cdp, tabId) : [];
|
const surfaceProbes = options.conditionalSurfaceProbe
|
||||||
|
? await probeHoverSurfaces(cdp, tabId, nodes, options)
|
||||||
|
: [];
|
||||||
|
throwIfAborted(options.signal);
|
||||||
|
|
||||||
return { nodes, viewport, iframeNodes, surfaceProbes, excludedBackendNodeIds };
|
const frameNodes = new Map<string, CapturedNode[]>();
|
||||||
|
if (topContext.frameId) frameNodes.set(topContext.frameId, nodes);
|
||||||
|
for (const iframeFrameNodes of iframeNodes.values()) {
|
||||||
|
const frameId = iframeFrameNodes.find((node) => node.frameId)?.frameId;
|
||||||
|
if (frameId) frameNodes.set(frameId, iframeFrameNodes);
|
||||||
|
}
|
||||||
|
for (const [frameId, ownerBackendNodeId] of frameParsed.frameOwnerBackendNodeIds) {
|
||||||
|
if (!frameNodes.has(frameId)) {
|
||||||
|
frameNodes.set(frameId, iframeNodes.get(ownerBackendNodeId) ?? []);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
nodes,
|
||||||
|
viewport,
|
||||||
|
iframeNodes,
|
||||||
|
frameNodes,
|
||||||
|
frameOwnerBackendNodeIds: frameParsed.frameOwnerBackendNodeIds,
|
||||||
|
frameParentIds: frameParsed.frameParentIds,
|
||||||
|
...(topContext.frameId ? { rootFrameId: topContext.frameId } : {}),
|
||||||
|
surfaceProbes,
|
||||||
|
excludedBackendNodeIds,
|
||||||
|
};
|
||||||
}
|
}
|
||||||
@@ -0,0 +1,179 @@
|
|||||||
|
import type { CdpFrame, CdpFrameGraph } from "@/browser-driver/frame-graph";
|
||||||
|
import { resolveFrameProjection } from "../frame-geometry";
|
||||||
|
import { type GeometryProjection, projectRectToViewport } from "../geometry";
|
||||||
|
import { type CdpRunner, cdpRunnerForTarget, sendToCdpTarget } from "../shared";
|
||||||
|
import { type CapturedNode, type CapturedViewModel, captureViewModel } from "./capture";
|
||||||
|
import {
|
||||||
|
buildFrameDocuments,
|
||||||
|
type FrameAxBatch,
|
||||||
|
type FrameDocument,
|
||||||
|
type FrameOwnedAxNode,
|
||||||
|
} from "./frame-document";
|
||||||
|
|
||||||
|
export type FrameAxNode = FrameOwnedAxNode;
|
||||||
|
export type CapturedFrameDocument<T extends FrameAxNode> = FrameDocument<T>;
|
||||||
|
|
||||||
|
function throwIfAborted(signal?: AbortSignal): void {
|
||||||
|
if (signal?.aborted) throw new DOMException("observation aborted", "AbortError");
|
||||||
|
}
|
||||||
|
|
||||||
|
function isAbortError(err: unknown): boolean {
|
||||||
|
return err instanceof Error && err.name === "AbortError";
|
||||||
|
}
|
||||||
|
|
||||||
|
async function discoverFrameGraph(cdp: CdpRunner, tabId: number): Promise<CdpFrameGraph | null> {
|
||||||
|
if (!cdp.getFrameGraph) return null;
|
||||||
|
try {
|
||||||
|
return await cdp.getFrameGraph(tabId);
|
||||||
|
} catch (err) {
|
||||||
|
console.debug("[bsk observation] frame graph capture failed", err);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function transformFrameNodes(
|
||||||
|
nodes: CapturedNode[],
|
||||||
|
projection: GeometryProjection,
|
||||||
|
): CapturedNode[] {
|
||||||
|
return nodes.map((node) => ({
|
||||||
|
...node,
|
||||||
|
rect: (() => {
|
||||||
|
const bounds = projectRectToViewport(node.rect, projection);
|
||||||
|
return bounds ? { x: bounds.x, y: bounds.y, w: bounds.width, h: bounds.height } : null;
|
||||||
|
})(),
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
function frameOwnerNode(captured: CapturedViewModel, frame: CdpFrame): CapturedNode | undefined {
|
||||||
|
if (frame.ownerBackendNodeId === undefined) return undefined;
|
||||||
|
const parentNodes = frame.parentFrameId
|
||||||
|
? captured.frameNodes?.get(frame.parentFrameId)
|
||||||
|
: captured.nodes;
|
||||||
|
return parentNodes?.find((node) => node.backendNodeId === frame.ownerBackendNodeId);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function captureMissingFrameDocuments(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
tabId: number,
|
||||||
|
graph: CdpFrameGraph,
|
||||||
|
captured: CapturedViewModel,
|
||||||
|
signal?: AbortSignal,
|
||||||
|
): Promise<void> {
|
||||||
|
if (!captured.frameNodes) captured.frameNodes = new Map();
|
||||||
|
if (!captured.frameOwnerBackendNodeIds) captured.frameOwnerBackendNodeIds = new Map();
|
||||||
|
if (!captured.frameParentIds) captured.frameParentIds = new Map();
|
||||||
|
|
||||||
|
for (const frame of graph.frames) {
|
||||||
|
if (!frame.target.sessionId || captured.frameNodes.has(frame.frameId)) continue;
|
||||||
|
const owner = frameOwnerNode(captured, frame);
|
||||||
|
if (!owner?.rect || frame.ownerBackendNodeId === undefined) continue;
|
||||||
|
try {
|
||||||
|
const child = await captureViewModel(cdpRunnerForTarget(cdp, frame.target), tabId, {
|
||||||
|
signal,
|
||||||
|
});
|
||||||
|
const projection = await resolveFrameProjection(cdp, graph, frame.frameId);
|
||||||
|
if (!projection) continue;
|
||||||
|
const rootFrameId = child.rootFrameId ?? frame.frameId;
|
||||||
|
const childFrames = child.frameNodes ?? new Map<string, CapturedNode[]>();
|
||||||
|
if (!childFrames.has(rootFrameId)) childFrames.set(rootFrameId, child.nodes);
|
||||||
|
if (rootFrameId !== frame.frameId && !childFrames.has(frame.frameId)) {
|
||||||
|
childFrames.set(
|
||||||
|
frame.frameId,
|
||||||
|
child.nodes.map((node) => ({ ...node, frameId: frame.frameId })),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
for (const [childFrameId, nodes] of childFrames) {
|
||||||
|
captured.frameNodes.set(childFrameId, transformFrameNodes(nodes, projection));
|
||||||
|
}
|
||||||
|
captured.iframeNodes.set(
|
||||||
|
frame.ownerBackendNodeId,
|
||||||
|
captured.frameNodes.get(frame.frameId) ?? [],
|
||||||
|
);
|
||||||
|
captured.frameOwnerBackendNodeIds.set(frame.frameId, frame.ownerBackendNodeId);
|
||||||
|
if (frame.parentFrameId) captured.frameParentIds.set(frame.frameId, frame.parentFrameId);
|
||||||
|
for (const [childFrameId, ownerBackendNodeId] of child.frameOwnerBackendNodeIds ?? []) {
|
||||||
|
captured.frameOwnerBackendNodeIds.set(childFrameId, ownerBackendNodeId);
|
||||||
|
}
|
||||||
|
for (const [childFrameId, parentFrameId] of child.frameParentIds ?? []) {
|
||||||
|
captured.frameParentIds.set(childFrameId, parentFrameId);
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
if (isAbortError(err)) throw err;
|
||||||
|
console.debug("[bsk observation] child frame DOM capture failed", {
|
||||||
|
frameId: frame.frameId,
|
||||||
|
err,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function captureAxTrees<T extends FrameAxNode>(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
tabId: number,
|
||||||
|
discoveredFrames: CdpFrame[],
|
||||||
|
captured: CapturedViewModel,
|
||||||
|
signal?: AbortSignal,
|
||||||
|
): Promise<FrameAxBatch<T>[]> {
|
||||||
|
let frames = [...discoveredFrames];
|
||||||
|
const capturedFrameIds = [...(captured.frameNodes?.keys() ?? [])];
|
||||||
|
if (frames.length === 0) {
|
||||||
|
const rootFrameId = captured.rootFrameId ?? capturedFrameIds[0] ?? "root";
|
||||||
|
frames = [{ frameId: rootFrameId, target: { tabId } }];
|
||||||
|
}
|
||||||
|
for (const frameId of capturedFrameIds) {
|
||||||
|
if (!frames.some((frame) => frame.frameId === frameId)) {
|
||||||
|
frames.push({ frameId, target: { tabId } });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let firstFailure: unknown;
|
||||||
|
const trees = await Promise.all(
|
||||||
|
frames.map(async (frame): Promise<FrameAxBatch<T>> => {
|
||||||
|
try {
|
||||||
|
throwIfAborted(signal);
|
||||||
|
await sendToCdpTarget(cdp, frame.target, "Accessibility.enable", {});
|
||||||
|
const result = await sendToCdpTarget<{ nodes?: T[] }>(
|
||||||
|
cdp,
|
||||||
|
frame.target,
|
||||||
|
"Accessibility.getFullAXTree",
|
||||||
|
frame.frameId === "root" ? {} : { frameId: frame.frameId },
|
||||||
|
);
|
||||||
|
throwIfAborted(signal);
|
||||||
|
return {
|
||||||
|
frame,
|
||||||
|
nodes: result.nodes ?? [],
|
||||||
|
};
|
||||||
|
} catch (err) {
|
||||||
|
if (isAbortError(err)) throw err;
|
||||||
|
firstFailure ??= err;
|
||||||
|
console.debug("[bsk observation] frame AX capture failed", {
|
||||||
|
frameId: frame.frameId,
|
||||||
|
err,
|
||||||
|
});
|
||||||
|
return { frame, nodes: [] };
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
const capturedNodeCount = [...captured.iframeNodes.values()].reduce(
|
||||||
|
(count, nodes) => count + nodes.length,
|
||||||
|
captured.nodes.length,
|
||||||
|
);
|
||||||
|
if (firstFailure && capturedNodeCount === 0 && trees.every((tree) => tree.nodes.length === 0)) {
|
||||||
|
throw firstFailure;
|
||||||
|
}
|
||||||
|
return trees;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function captureFrameData<T extends FrameAxNode>(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
tabId: number,
|
||||||
|
captured: CapturedViewModel,
|
||||||
|
signal?: AbortSignal,
|
||||||
|
): Promise<CapturedFrameDocument<T>[]> {
|
||||||
|
const graph = await discoverFrameGraph(cdp, tabId);
|
||||||
|
const frames = graph?.frames ?? [];
|
||||||
|
if (graph) await captureMissingFrameDocuments(cdp, tabId, graph, captured, signal);
|
||||||
|
throwIfAborted(signal);
|
||||||
|
const batches = await captureAxTrees<T>(cdp, tabId, frames, captured, signal);
|
||||||
|
return buildFrameDocuments(graph, batches, captured);
|
||||||
|
}
|
||||||
Loaded 100 of 208 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user