Compare commits

..
Author SHA1 Message Date
BB-fat 6b78211e79 test(dsh-plugin): drop redundant thumbnail read-back case
Coverage already lives in the attachment-store and ring tests; this
case only re-hit readThumbnail and was flaky on CI.
2026-08-19 03:57:08 +00:00
BB-fat eebf334b40 fix(dsh-plugin): give observation waitFor more room on CI
The thumbnail read-back test timed out at the 2s waitFor budget under
CI load even though capture completed shortly after.
2026-08-19 03:50:14 +00:00
BB-fat a284ffc5ae Merge pull request #112 from Tencent/fix/dsh-plugin-id-and-edge-store
Fix/dsh plugin id and edge store
2026-08-19 11:43:13 +08:00
drakezhangandCursor e720f1ffe5 ci(dsh-plugin): publish from the npm-publish environment
The publish job named its GitHub Environment NPM_TOKEN, which is the
secret name rather than a deployment target, so the repository's
Deployments list showed a bogus NPM_TOKEN entry next to chrome-web-store.
The environment is now npm-publish; the secret it holds keeps its name.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-08-19 11:20:26 +08:00
drakezhangandCursor 2b7e10ca2b fix(dsh-plugin-browserskill): identify the plugin by its published name
Renaming the package to @wxg-prc-cpg/browser-skill-dsh-plugin left two
identifiers behind on the old directory name, and both are resolved by
name rather than by path.

cordis.patch.yml is the specifier the loader imports, so dsh failed with
ERR_MODULE_NOT_FOUND on startup and took the whole plugin tree down with
it — installing the published package made dsh unusable. The client
bundle registers itself under the same kind of key, which the web shell
looks up by package name, so the screenshot toolview and the observation
overlay were unreachable too.

The bundle id now reads package.json instead of repeating the name, so it
cannot drift again. Quoting is required in the YAML because a leading '@'
is reserved in plain scalars.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-08-19 11:20:07 +08:00
drakezhangandCursor 1ef93463dd docs: list the Edge Add-ons build alongside the Chrome Web Store
The extension is now published to Edge Add-ons, but every install path
still pointed Edge users at the Chrome Web Store. Both READMEs and
AGENT_INSTALL.md now name a store per browser, and `bsk doctor` offers
both listings when no browser is connected.

Two Edge-specific gaps went with it: Edge blocks content-script injection
on its own storefront the same way Chrome does, so the borrow-confirmation
overlay must treat microsoftedge.microsoft.com/addons as non-injectable;
and the Chinese README still described the pre-install.ps1 manual Windows
setup.

Also give the DeepSeek Harness section its npm package link and the
profile-start step, so plugin users have a complete path.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-08-19 11:19:47 +08:00
Hannah Liu c7252529f4 Merge pull request #109 from Tencent/fix/oopif-support
fix(extension): support OOPIF and cross-frame geometry
2026-08-18 21:16:39 +08:00
Ljy-0827 0b65b4b295 fix(vom): oopif fix 2026-08-18 19:28:16 +08:00
Ljy-0827 94ec7e73fc fix(vom): iframe oopif geometry 2026-08-18 19:28:16 +08:00
Ljy-0827 42403e55f2 feat(extension): support OOPIF refs with unified frame geometry 2026-08-18 19:28:16 +08:00
Hannah Liu 22019cdf07 Merge pull request #105 from Tencent/fix/vom-iframe
fix(vom): vom support observe iframe
2026-08-18 19:28:13 +08:00
Ljy-0827 8d7c0aa66c fix(extension): support iframe content in VOM observations 2026-08-18 19:18:39 +08:00
BB-fat a9489282fa fix(dsh-plugin-browserskill): keep observation frames in a 2-slot ring
Stop writing overlay thumbnails into the durable attachment store. Each
session now holds only the current and previous PNG in process memory,
reuses the id when the hash is unchanged, and drops the rest on rotate,
remove, or dispose.
2026-08-18 10:15:27 +00:00
Zhang GH d4cddd6224 Merge pull request #99 from Tencent/fix/record-empty-fill
fix(record): preserve committed empty fill steps
2026-08-18 18:12:59 +08:00
Zhang GH 8e4de6cf4b Merge pull request #91 from Tencent/feat/dsh-plugin
feat(packages): add dsh-plugin-browserskill — DeepSeek Harness tools for bsk
2026-08-18 18:12:37 +08:00
BB-fat fea66ce404 fix(dsh-plugin-browserskill): send JSON content-type on overlay interrupt
The observation HTTP fence rejects POSTs without application/json, so the
Interrupt button always got 403.
2026-08-18 09:36:15 +00:00
BB-fat 8bae85127a docs: add DeepSeek Harness plugin section to the README
Document the dsh plugin highlights and the one-line install for the web profile.
2026-08-18 03:13:58 +00:00
“haonan”andCursor c8c5c4a7c4 fix(record): preserve committed empty fill steps
Allow intentional input clears to remain in exported traces instead of dropping every non-redacted blank fill.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-08-17 17:21:17 +08:00
57 changed files with 4122 additions and 644 deletions

No files matched your search

+2 -2
View File
@@ -66,8 +66,8 @@ jobs:
publish: publish:
needs: [resolve, guard] needs: [resolve, guard]
runs-on: ubuntu-latest runs-on: ubuntu-latest
# Secret NPM_TOKEN is stored on the GitHub Environment also named NPM_TOKEN. # Secret NPM_TOKEN is stored on the "npm-publish" GitHub Environment.
environment: NPM_TOKEN environment: npm-publish
permissions: permissions:
contents: read contents: read
steps: steps:
+9 -5
View File
@@ -34,13 +34,17 @@ where only `extension connected` fails is expected; go to Step 3.
## 3. Open the extension install page ## 3. Open the extension install page
If `extension connected` is `FAIL` (`0 browsers connected`), the user likely If `extension connected` is `FAIL` (`0 browsers connected`), the user likely
has not installed or enabled the browser extension yet. Open the has not installed or enabled the browser extension yet. Open the install page
[Chrome Web Store install page](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi) matching the user's browser —
in the user's browser, then tell the user: [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi)
for Chrome and other Chromium browsers,
[Edge Add-ons](https://microsoftedge.microsoft.com/addons/detail/browserskill/emacgiaaaiojkkpkddmmdfhmokgmnikg)
for Microsoft Edge — then tell the user:
> Install **BrowserSkill** from the > Install **BrowserSkill** from the
> [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi), > [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi)
> open the popup, and wait until it turns green. Reply when done. > (or [Edge Add-ons](https://microsoftedge.microsoft.com/addons/detail/browserskill/emacgiaaaiojkkpkddmmdfhmokgmnikg)
> on Edge), open the popup, and wait until it turns green. Reply when done.
If opening the page fails, give the user the same link. Then run `bsk doctor` If opening the page fails, give the user the same link. Then run `bsk doctor`
once more. All `ok`/`na` → tell the user it's ready. once more. All `ok`/`na` → tell the user it's ready.
+37 -5
View File
@@ -13,7 +13,7 @@
</p> </p>
**BrowserSkill** connects Cursor, Claude Code, Codex, OpenClaw, CodeBuddy, **BrowserSkill** connects Cursor, Claude Code, Codex, OpenClaw, CodeBuddy,
WorkBuddy, Pi, Hermes Agent, and other shell-capable AI agents to your already logged-in WorkBuddy, Pi, Hermes Agent, DeepSeek Harness, and other AI agents to your already logged-in
browser. browser.
Need the agent to touch a tab you already have open? It must borrow that tab Need the agent to touch a tab you already have open? It must borrow that tab
@@ -67,7 +67,8 @@ Set up browser-skill on this machine by following https://raw.githubusercontent.
<br> <br>
Install the CLI, then install the extension from the [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi). Install the CLI, then install the extension from the [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi)
or [Edge Add-ons](https://microsoftedge.microsoft.com/addons/detail/browserskill/emacgiaaaiojkkpkddmmdfhmokgmnikg).
#### 1. Install the `bsk` CLI #### 1. Install the `bsk` CLI
@@ -91,7 +92,14 @@ bsk --version
#### 2. Install the browser extension #### 2. Install the browser extension
Install BrowserSkill from the [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi). Install BrowserSkill from your browser's store:
| Browser | Store listing |
| --- | --- |
| Chrome | [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi) |
| Microsoft Edge | [Edge Add-ons](https://microsoftedge.microsoft.com/addons/detail/browserskill/emacgiaaaiojkkpkddmmdfhmokgmnikg) |
On other Chromium-based browsers, install the Chrome Web Store build.
#### 3. Install the skill #### 3. Install the skill
@@ -123,7 +131,8 @@ internal variants and install paths.
Other shell-capable agent harnesses are supported too. Copy Other shell-capable agent harnesses are supported too. Copy
[`skill/SKILL.md`](skill/SKILL.md) into your harness's skills directory as [`skill/SKILL.md`](skill/SKILL.md) into your harness's skills directory as
`browser-skill/SKILL.md` to install the skill manually. `browser-skill/SKILL.md` to install the skill manually. DeepSeek Harness uses a
dedicated plugin instead — see [DeepSeek Harness plugin](#deepseek-harness-plugin).
</details> </details>
@@ -133,6 +142,26 @@ Start a new Agent session and write a prompt that needs the browser, for example
/browser-skill open example.com and summarize what is on the page. /browser-skill open example.com and summarize what is on the page.
``` ```
## DeepSeek Harness plugin
Using [DeepSeek Harness](https://github.com/deepseek-ai/deepseek-harness) (`dsh`)?
BrowserSkill ships a first-class dsh plugin on npm as
[`@wxg-prc-cpg/browser-skill-dsh-plugin`](https://www.npmjs.com/package/@wxg-prc-cpg/browser-skill-dsh-plugin).
It injects native `browser_*` tools (no shelling out to `bsk`) and a live Web UI
overlay of each Agent Window.
Add it to a dsh profile, then start that profile:
```sh
dsh plugin --profile web add @wxg-prc-cpg/browser-skill-dsh-plugin
dsh --profile web
```
The plugin carries its own copy of the skill, so `bsk install-skill` is not needed
for dsh — but the `bsk` CLI and the browser extension are still prerequisites. See
the [plugin README](packages/dsh-plugin-browserskill/README.md) for the tool list,
configuration, and the observation overlay.
## How It Works ## How It Works
BrowserSkill is a local bridge between your agent harness and your browser. BrowserSkill is a local bridge between your agent harness and your browser.
@@ -166,7 +195,9 @@ flowchart TB
The agent never talks to the browser directly. It asks the `bsk` CLI to perform a The agent never talks to the browser directly. It asks the `bsk` CLI to perform a
browser task; the local daemon routes that request to the extension; the browser task; the local daemon routes that request to the extension; the
extension runs it in an Agent Window. extension runs it in an Agent Window. DeepSeek Harness takes the same path
through the [plugin](#deepseek-harness-plugin): the agent calls injected
`browser_*` tools, and the plugin invokes `bsk` on its behalf.
## For Developers ## For Developers
@@ -176,6 +207,7 @@ The repository is a Cargo + pnpm workspace:
- `crates/bsk-protocol` — shared wire types and JSON schemas - `crates/bsk-protocol` — shared wire types and JSON schemas
- `apps/extension` — browser extension - `apps/extension` — browser extension
- `packages/ui` and `packages/i18n` — shared extension UI support - `packages/ui` and `packages/i18n` — shared extension UI support
- `packages/dsh-plugin-browserskill` — DeepSeek Harness plugin (`@wxg-prc-cpg/browser-skill-dsh-plugin`)
## License ## License
+32 -7
View File
@@ -12,7 +12,7 @@
<a href="README.md">English</a> · 中文 <a href="README.md">English</a> · 中文
</p> </p>
**BrowserSkill** 把 Cursor、Claude Code、Codex、OpenClaw、CodeBuddy、WorkBuddy、Pi、Hermes Agent 等支持 Shell 的 AI Agent 连接到你已登录的浏览器。 **BrowserSkill** 把 Cursor、Claude Code、Codex、OpenClaw、CodeBuddy、WorkBuddy、Pi、Hermes Agent、DeepSeek Harness 等 AI Agent 连接到你已登录的浏览器。
需要 Agent 操作你已打开的标签页?必须显式借用该标签,任务结束后归还,其余浏览器窗口不受影响。 需要 Agent 操作你已打开的标签页?必须显式借用该标签,任务结束后归还,其余浏览器窗口不受影响。
@@ -54,7 +54,8 @@ BrowserSkill 由两个本地运行组件组成:`bsk` CLI/daemon 和浏览器
<br> <br>
先安装 CLI,再从 [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi) 安装浏览器扩展。 先安装 CLI,再从 [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi)
或 [Edge 加载项商店](https://microsoftedge.microsoft.com/addons/detail/browserskill/emacgiaaaiojkkpkddmmdfhmokgmnikg) 安装浏览器扩展。
#### 1. 安装 `bsk` CLI #### 1. 安装 `bsk` CLI
@@ -64,8 +65,11 @@ BrowserSkill 由两个本地运行组件组成:`bsk` CLI/daemon 和浏览器
curl -fsSL https://raw.githubusercontent.com/Tencent/BrowserSkill/main/install.sh | sh curl -fsSL https://raw.githubusercontent.com/Tencent/BrowserSkill/main/install.sh | sh
``` ```
**Windows**:从 [最新 CLI release](https://github.com/Tencent/BrowserSkill/releases/latest) **Windows**(PowerShell,安装到 `~/.local/bin`):
下载 `bsk-v<version>-x86_64-pc-windows-msvc.zip`,解压后将 `bsk.exe` 加入 `PATH`。
```powershell
irm https://raw.githubusercontent.com/Tencent/BrowserSkill/main/install.ps1 | iex
```
验证二进制: 验证二进制:
@@ -75,7 +79,14 @@ bsk --version
#### 2. 安装浏览器扩展 #### 2. 安装浏览器扩展
从 [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi) 安装 BrowserSkill。 在对应浏览器的商店安装 BrowserSkill:
| 浏览器 | 商店页面 |
| --- | --- |
| Chrome | [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi) |
| Microsoft Edge | [Edge 加载项商店](https://microsoftedge.microsoft.com/addons/detail/browserskill/emacgiaaaiojkkpkddmmdfhmokgmnikg) |
其他基于 Chromium 的浏览器,安装 Chrome Web Store 版本即可。
#### 3. 安装 skill #### 3. 安装 skill
@@ -102,7 +113,7 @@ bsk install-skill
用 <kbd>Space</kbd> 选择需要安装的 Agent harness,然后按 <kbd>Enter</kbd> 安装 skill。运行 `bsk install-skill --list` 可查看 internal 变体及安装路径。 用 <kbd>Space</kbd> 选择需要安装的 Agent harness,然后按 <kbd>Enter</kbd> 安装 skill。运行 `bsk install-skill --list` 可查看 internal 变体及安装路径。
其他支持 Shell 的 Agent harness 也可使用 BrowserSkill,但需手动将 [`skill/SKILL.md`](skill/SKILL.md) 复制到对应 skills 目录下的 `browser-skill/SKILL.md`。 其他支持 Shell 的 Agent harness 也可使用 BrowserSkill,但需手动将 [`skill/SKILL.md`](skill/SKILL.md) 复制到对应 skills 目录下的 `browser-skill/SKILL.md`。DeepSeek Harness 走独立插件,见 [DeepSeek Harness 插件](#deepseek-harness-插件)。
</details> </details>
@@ -112,6 +123,19 @@ bsk install-skill
/browser-skill open example.com and summarize what is on the page. /browser-skill open example.com and summarize what is on the page.
``` ```
## DeepSeek Harness 插件
在用 [DeepSeek Harness](https://github.com/deepseek-ai/deepseek-harness)(`dsh`)?BrowserSkill 提供了官方 dsh 插件,已发布到 npm:[`@wxg-prc-cpg/browser-skill-dsh-plugin`](https://www.npmjs.com/package/@wxg-prc-cpg/browser-skill-dsh-plugin)。它会注入原生 `browser_*` 工具(无需再通过 Shell 调用 `bsk`),并在 Web UI 中实时观察每个 Agent Window。
把它装进某个 dsh profile,然后启动该 profile:
```sh
dsh plugin --profile web add @wxg-prc-cpg/browser-skill-dsh-plugin
dsh --profile web
```
插件自带 skill,所以在 dsh 下无需执行 `bsk install-skill`;但 `bsk` CLI 和浏览器扩展仍是前置条件。工具清单、配置项与观察浮层见[插件 README](packages/dsh-plugin-browserskill/README.md)。
## 工作原理 ## 工作原理
BrowserSkill 是 Agent 运行时与浏览器之间的本地桥接层。 BrowserSkill 是 Agent 运行时与浏览器之间的本地桥接层。
@@ -143,7 +167,7 @@ flowchart TB
style UserWindows fill:#f8fafc,stroke:#cbd5e1,color:#334155 style UserWindows fill:#f8fafc,stroke:#cbd5e1,color:#334155
``` ```
Agent 不直接与浏览器通信。它通过 `bsk` CLI 下发浏览器任务;本地 daemon 把请求路由到扩展;扩展在 Agent Window 中执行。 Agent 不直接与浏览器通信。它通过 `bsk` CLI 下发浏览器任务;本地 daemon 把请求路由到扩展;扩展在 Agent Window 中执行。DeepSeek Harness 走同一条链路,只是经由 [插件](#deepseek-harness-插件):Agent 调用注入的 `browser_*` 工具,由插件代为执行 `bsk`。
## 面向开发者 ## 面向开发者
@@ -153,6 +177,7 @@ Agent 不直接与浏览器通信。它通过 `bsk` CLI 下发浏览器任务;
- `crates/bsk-protocol` — 共享协议类型与 JSON Schema - `crates/bsk-protocol` — 共享协议类型与 JSON Schema
- `apps/extension` — 浏览器扩展 - `apps/extension` — 浏览器扩展
- `packages/ui` 和 `packages/i18n` — 扩展 UI 共享支持 - `packages/ui` 和 `packages/i18n` — 扩展 UI 共享支持
- `packages/dsh-plugin-browserskill` — DeepSeek Harness 插件(`@wxg-prc-cpg/browser-skill-dsh-plugin`)
## 许可证 ## 许可证
@@ -1,5 +1,5 @@
import { describe, expect, it, vi } from "vitest"; import { describe, expect, it, vi } from "vitest";
import { type CdpDebuggerApi, ChromiumCdp } from "../chromium-cdp"; import { type CdpDebuggee, type CdpDebuggerApi, ChromiumCdp } from "../chromium-cdp";
function fakeChromeEvent<TArgs extends unknown[]>() { function fakeChromeEvent<TArgs extends unknown[]>() {
const listeners = new Set<(...args: TArgs) => void>(); const listeners = new Set<(...args: TArgs) => void>();
@@ -14,7 +14,7 @@ function fakeChromeEvent<TArgs extends unknown[]>() {
} }
function fakeApi() { function fakeApi() {
const onEvent = fakeChromeEvent<[chrome.debugger.Debuggee, string, unknown]>(); const onEvent = fakeChromeEvent<[CdpDebuggee, string, unknown]>();
const onDetach = fakeChromeEvent<[chrome.debugger.Debuggee, string]>(); const onDetach = fakeChromeEvent<[chrome.debugger.Debuggee, string]>();
const api: CdpDebuggerApi = { const api: CdpDebuggerApi = {
attach: vi.fn(async () => {}), attach: vi.fn(async () => {}),
@@ -29,6 +29,135 @@ function fakeApi() {
} }
describe("ChromiumCdp", () => { describe("ChromiumCdp", () => {
it("discovers multiple iframe targets and recursively routes nested OOPIF commands", async () => {
const { api, onEvent } = fakeApi();
(api.sendCommand as ReturnType<typeof vi.fn>).mockImplementation(
async (
target: chrome.debugger.Debuggee & { sessionId?: string },
method: string,
params?: { frameId?: string },
) => {
if (method === "DOM.getFrameOwner") {
return { backendNodeId: params?.frameId === "nested" ? 300 : 200 };
}
if (method !== "Page.getFrameTree") return {};
if (target.sessionId === "right-session") {
return {
frameTree: {
frame: { id: "right", url: "https://right.test" },
childFrames: [
{ frame: { id: "nested", parentId: "right", url: "https://nested.test" } },
],
},
};
}
if (target.sessionId === "nested-session") {
return { frameTree: { frame: { id: "nested", url: "https://nested.test" } } };
}
return {
frameTree: {
frame: { id: "main", url: "https://app.test" },
childFrames: [
{ frame: { id: "left", parentId: "main", url: "https://left.test" } },
{
frame: { id: "right", parentId: "main", url: "https://right.test" },
childFrames: [
{ frame: { id: "nested", parentId: "right", url: "https://nested.test" } },
],
},
],
},
};
},
);
const cdp = new ChromiumCdp(api);
await cdp.ensureAttached(4);
onEvent.fire({ tabId: 4 }, "Target.attachedToTarget", { sessionId: "right-session" });
onEvent.fire({ tabId: 4 } as chrome.debugger.Debuggee, "Target.attachedToTarget", {
sessionId: "nested-session",
});
const graph = await cdp.getFrameGraph(4);
expect(graph.frames).toHaveLength(4);
expect(graph.frames.find((frame) => frame.frameId === "left")?.target).toEqual({ tabId: 4 });
expect(graph.frames.find((frame) => frame.frameId === "right")?.target.sessionId).toBe(
"right-session",
);
expect(graph.frames.find((frame) => frame.frameId === "nested")?.target.sessionId).toBe(
"nested-session",
);
expect(graph.frames.find((frame) => frame.frameId === "right")?.ownerBackendNodeId).toBe(200);
expect(graph.frames.find((frame) => frame.frameId === "nested")?.ownerBackendNodeId).toBe(300);
expect(api.sendCommand).toHaveBeenCalledWith(
{ tabId: 4, sessionId: "right-session" },
"DOM.getFrameOwner",
{ frameId: "nested" },
);
expect(api.sendCommand).toHaveBeenCalledWith(
{ tabId: 4, sessionId: "nested-session" },
"Target.setAutoAttach",
expect.objectContaining({ flatten: true }),
);
});
it("waits for recursively attached iframe targets without a depth limit", async () => {
const { api, onEvent } = fakeApi();
const depth = 6;
const scheduled = new Set<string>();
const nestedTree = (index: number): Record<string, unknown> => ({
frame: {
id: index === 0 ? "main" : `frame-${index}`,
...(index > 0 ? { parentId: index === 1 ? "main" : `frame-${index - 1}` } : {}),
},
...(index < depth ? { childFrames: [nestedTree(index + 1)] } : {}),
});
(api.sendCommand as ReturnType<typeof vi.fn>).mockImplementation(
async (target: CdpDebuggee, method: string, params?: { frameId?: string }) => {
if (method === "Target.setAutoAttach") {
const parentIndex = target.sessionId
? Number(target.sessionId.replace("session-", ""))
: 0;
const nextIndex = parentIndex + 1;
const key = `${target.sessionId ?? "root"}:${nextIndex}`;
if (nextIndex <= depth && !scheduled.has(key)) {
scheduled.add(key);
setTimeout(() => {
onEvent.fire(
{ tabId: 4, ...(target.sessionId ? { sessionId: target.sessionId } : {}) },
"Target.attachedToTarget",
{
sessionId: `session-${nextIndex}`,
targetInfo: { type: "iframe" },
},
);
}, 5);
}
return {};
}
if (method === "Page.getFrameTree") {
if (!target.sessionId) return { frameTree: nestedTree(0) };
const index = Number(target.sessionId.replace("session-", ""));
return { frameTree: nestedTree(index) };
}
if (method === "DOM.getFrameOwner") {
return { backendNodeId: Number(params?.frameId?.replace("frame-", "")) + 100 };
}
return {};
},
);
const cdp = new ChromiumCdp(api);
const graph = await cdp.getFrameGraph(4);
expect(graph.frames).toHaveLength(depth + 1);
for (let index = 1; index <= depth; index += 1) {
expect(
graph.frames.find((frame) => frame.frameId === `frame-${index}`)?.target.sessionId,
).toBe(`session-${index}`);
}
});
it("coalesces concurrent attach calls for the same tab", async () => { it("coalesces concurrent attach calls for the same tab", async () => {
const { api } = fakeApi(); const { api } = fakeApi();
let releaseAttach!: () => void; let releaseAttach!: () => void;
@@ -0,0 +1,136 @@
import { describe, expect, it } from "vitest";
import { buildFrameGraph, type CdpFrameTreeNode, type CdpFrameTreeSource } from "../frame-graph";
describe("buildFrameGraph", () => {
it("keeps sibling frames distinct and routes nested OOPIFs to their child sessions", () => {
const graph = buildFrameGraph([
{
target: { tabId: 4 },
tree: {
frame: { id: "main", url: "https://app.test" },
childFrames: [
{ frame: { id: "left", parentId: "main", url: "https://left.test" } },
{
frame: { id: "right", parentId: "main", url: "https://right.test" },
childFrames: [
{ frame: { id: "nested", parentId: "right", url: "https://nested.test" } },
],
},
],
},
},
{
target: { tabId: 4, sessionId: "right-session" },
tree: {
frame: { id: "right", url: "https://right.test" },
childFrames: [{ frame: { id: "nested", parentId: "right", url: "https://nested.test" } }],
},
},
{
target: { tabId: 4, sessionId: "nested-session" },
tree: { frame: { id: "nested", url: "https://nested.test" } },
},
]);
expect(graph?.frames).toHaveLength(4);
expect(graph?.frames.find((frame) => frame.frameId === "left")?.target).toEqual({ tabId: 4 });
expect(graph?.frames.find((frame) => frame.frameId === "right")).toMatchObject({
parentFrameId: "main",
target: { tabId: 4, sessionId: "right-session" },
});
expect(graph?.frames.find((frame) => frame.frameId === "nested")).toMatchObject({
parentFrameId: "right",
target: { tabId: 4, sessionId: "nested-session" },
});
});
it("propagates each target boundary to its same-process descendants", () => {
const sources: CdpFrameTreeSource[] = [
{
target: { tabId: 4 },
tree: {
frame: { id: "main" },
childFrames: [
{
frame: { id: "oopif-a", parentId: "main" },
childFrames: [
{
frame: { id: "same-process-b", parentId: "oopif-a" },
childFrames: [
{
frame: { id: "oopif-c", parentId: "same-process-b" },
childFrames: [{ frame: { id: "same-process-d", parentId: "oopif-c" } }],
},
],
},
],
},
],
},
},
{
target: { tabId: 4, sessionId: "session-a" },
tree: {
frame: { id: "oopif-a" },
childFrames: [
{
frame: { id: "same-process-b", parentId: "oopif-a" },
childFrames: [{ frame: { id: "oopif-c", parentId: "same-process-b" } }],
},
],
},
},
{
target: { tabId: 4, sessionId: "session-c" },
tree: {
frame: { id: "oopif-c" },
childFrames: [{ frame: { id: "same-process-d", parentId: "oopif-c" } }],
},
},
];
const graph = buildFrameGraph(sources);
expect(graph?.frames.map((frame) => [frame.frameId, frame.target.sessionId])).toEqual([
["main", undefined],
["oopif-a", "session-a"],
["same-process-b", "session-a"],
["oopif-c", "session-c"],
["same-process-d", "session-c"],
]);
expect(graph?.frames.find((frame) => frame.frameId === "oopif-a")?.parentFrameId).toBe("main");
expect(graph?.frames.find((frame) => frame.frameId === "oopif-c")?.parentFrameId).toBe(
"same-process-b",
);
const reordered = buildFrameGraph([sources[2], sources[1], sources[0]]);
expect(reordered).toEqual(graph);
});
it("walks deeply nested frame trees without consuming the JavaScript call stack", () => {
const depth = 5_000;
const root: CdpFrameTreeNode = { frame: { id: "frame-0" } };
let parent = root;
for (let index = 1; index <= depth; index += 1) {
const child: CdpFrameTreeNode = {
frame: { id: `frame-${index}`, parentId: `frame-${index - 1}` },
};
parent.childFrames = [child];
parent = child;
}
const graph = buildFrameGraph([{ target: { tabId: 4 }, tree: root }]);
expect(graph?.frames).toHaveLength(depth + 1);
expect(graph?.frames.at(-1)?.frameId).toBe(`frame-${depth}`);
});
it("does not loop when an in-memory frame tree contains an object cycle", () => {
const root: CdpFrameTreeNode = { frame: { id: "main" } };
root.childFrames = [root];
expect(buildFrameGraph([{ target: { tabId: 4 }, tree: root }])?.frames).toEqual([
{ frameId: "main", target: { tabId: 4 } },
]);
});
});
+211 -12
View File
@@ -31,6 +31,15 @@ import type {
NetworkEntryKind, NetworkEntryKind,
NetworkResult, NetworkResult,
} from "@/transport/types"; } from "@/transport/types";
import {
buildFrameGraph,
type CdpFrameGraph,
type CdpFrameTreeNode,
type CdpFrameTreeSource,
type CdpTarget,
} from "./frame-graph";
export type CdpDebuggee = chrome.debugger.Debuggee & { sessionId?: string };
/** /**
* Minimal slice of `chrome.debugger` the rest of the extension * Minimal slice of `chrome.debugger` the rest of the extension
@@ -38,21 +47,15 @@ import type {
* fake without monkey-patching the real `chrome` global. * fake without monkey-patching the real `chrome` global.
*/ */
export interface CdpDebuggerApi { export interface CdpDebuggerApi {
attach(target: chrome.debugger.Debuggee, requiredVersion: string): Promise<void>; attach(target: CdpDebuggee, requiredVersion: string): Promise<void>;
detach(target: chrome.debugger.Debuggee): Promise<void>; detach(target: CdpDebuggee): Promise<void>;
sendCommand( sendCommand(target: CdpDebuggee, method: string, commandParams?: object): Promise<unknown>;
target: chrome.debugger.Debuggee,
method: string,
commandParams?: object,
): Promise<unknown>;
/** /**
* Fires for every CDP event (`Page.lifecycleEvent`, `DOM.documentUpdated`, * Fires for every CDP event (`Page.lifecycleEvent`, `DOM.documentUpdated`,
* …). The first callback argument is the source debuggee; the second * …). The first callback argument is the source debuggee; the second
* is the CDP method name; the third is the payload. * is the CDP method name; the third is the payload.
*/ */
onEvent: chrome.events.Event< onEvent: chrome.events.Event<(source: CdpDebuggee, method: string, params: unknown) => void>;
(source: chrome.debugger.Debuggee, method: string, params: unknown) => void
>;
/** /**
* Fires when Chrome unilaterally detaches us — most commonly because * Fires when Chrome unilaterally detaches us — most commonly because
* the tab navigated to a chrome:// URL or the user clicked * the tab navigated to a chrome:// URL or the user clicked
@@ -110,6 +113,8 @@ const MAX_CONSOLE_STACK_FRAMES = 20;
const MAX_NETWORK_BUFFER = 200; const MAX_NETWORK_BUFFER = 200;
const MAX_NETWORK_FIELD_LENGTH = 4096; const MAX_NETWORK_FIELD_LENGTH = 4096;
const MAX_NETWORK_REQUEST_META = 1024; const MAX_NETWORK_REQUEST_META = 1024;
const FRAME_DISCOVERY_TIMEOUT_MS = 1000;
const FRAME_DISCOVERY_QUIET_MS = 20;
interface ParsedDialogOpening { interface ParsedDialogOpening {
type: JavaScriptDialogType; type: JavaScriptDialogType;
@@ -131,6 +136,27 @@ interface NetworkRequestMeta {
truncated: boolean; truncated: boolean;
} }
interface FrameDiscoveryState {
sessions: Set<string>;
pending: Set<Promise<void>>;
generation: number;
}
async function settleBeforeDeadline(promises: Promise<void>[], deadline: number): Promise<boolean> {
if (promises.length === 0) return true;
const remaining = deadline - Date.now();
if (remaining <= 0) return false;
let timer: ReturnType<typeof setTimeout> | undefined;
const timedOut = await Promise.race([
Promise.allSettled(promises).then(() => false),
new Promise<boolean>((resolve) => {
timer = setTimeout(() => resolve(true), remaining);
}),
]);
if (timer) clearTimeout(timer);
return !timedOut;
}
/** /**
* Wrapper around `chrome.debugger` that owns the "attach once per * Wrapper around `chrome.debugger` that owns the "attach once per
* tabId" cache and exposes typed `send<T>()`. * tabId" cache and exposes typed `send<T>()`.
@@ -149,10 +175,12 @@ export class ChromiumCdp {
private readonly networkSequences = new Map<number, number>(); private readonly networkSequences = new Map<number, number>();
private readonly networkDomainsEnabledTabs = new Set<number>(); private readonly networkDomainsEnabledTabs = new Set<number>();
private readonly networkRequestMeta = new Map<number, Map<string, NetworkRequestMeta>>(); private readonly networkRequestMeta = new Map<number, Map<string, NetworkRequestMeta>>();
private readonly frameDiscovery = new Map<number, FrameDiscoveryState>();
private detachSubscription: { dispose(): void } | null = null; private detachSubscription: { dispose(): void } | null = null;
private dialogSubscription: { dispose(): void } | null = null; private dialogSubscription: { dispose(): void } | null = null;
private consoleSubscription: { dispose(): void } | null = null; private consoleSubscription: { dispose(): void } | null = null;
private networkSubscription: { dispose(): void } | null = null; private networkSubscription: { dispose(): void } | null = null;
private frameTargetSubscription: { dispose(): void } | null = null;
constructor(api: CdpDebuggerApi = chromeDebuggerApi) { constructor(api: CdpDebuggerApi = chromeDebuggerApi) {
this.api = api; this.api = api;
@@ -160,6 +188,7 @@ export class ChromiumCdp {
this.bindDialogHandler(); this.bindDialogHandler();
this.bindConsoleHandler(); this.bindConsoleHandler();
this.bindNetworkHandler(); this.bindNetworkHandler();
this.bindFrameTargetHandler();
} }
/** Attach to `tabId` if we haven't already in this driver. */ /** Attach to `tabId` if we haven't already in this driver. */
@@ -177,6 +206,9 @@ export class ChromiumCdp {
await this.enableConsoleDomains(tabId); await this.enableConsoleDomains(tabId);
await this.enableNetworkDomainBestEffort(tabId); await this.enableNetworkDomainBestEffort(tabId);
this.attachedTabs.add(tabId); this.attachedTabs.add(tabId);
await this.enableFrameDiscovery({ tabId }).catch((err) => {
console.debug("[bsk cdp] frame discovery unavailable", { tabId, err });
});
} catch (err) { } catch (err) {
// A CDP domain enable failed after the raw attach succeeded // A CDP domain enable failed after the raw attach succeeded
// (e.g. `Page.enable` rejects because the tab just navigated to // (e.g. `Page.enable` rejects because the tab just navigated to
@@ -222,6 +254,73 @@ export class ChromiumCdp {
} }
} }
async sendToTarget<T = unknown>(target: CdpTarget, method: string, params?: object): Promise<T> {
if (!this.attachedTabs.has(target.tabId)) {
await this.ensureAttached(target.tabId);
}
try {
return (await this.api.sendCommand(target, method, params ?? {})) as T;
} catch (err) {
throw normalizeError(err);
}
}
async getFrameGraph(tabId: number): Promise<CdpFrameGraph> {
await this.ensureAttached(tabId);
await this.enableFrameDiscovery({ tabId }).catch(() => {});
await this.drainFrameAttachTasks(tabId);
const sources: CdpFrameTreeSource[] = [];
const root = await this.sendToTarget<{ frameTree?: CdpFrameTreeNode }>(
{ tabId },
"Page.getFrameTree",
{},
);
if (root.frameTree) sources.push({ target: { tabId }, tree: root.frameTree });
const sessions = [...(this.frameDiscovery.get(tabId)?.sessions ?? [])];
const childTrees = await Promise.all(
sessions.map(async (sessionId): Promise<CdpFrameTreeSource | null> => {
const target = { tabId, sessionId };
try {
const reply = await this.sendToTarget<{ frameTree?: CdpFrameTreeNode }>(
target,
"Page.getFrameTree",
{},
);
return reply.frameTree ? { target, tree: reply.frameTree } : null;
} catch {
return null;
}
}),
);
for (const source of childTrees) {
if (source) sources.push(source);
}
const graph = buildFrameGraph(sources);
if (!graph) throw new Error("Page.getFrameTree returned no root frame");
const frameById = new Map(graph.frames.map((frame) => [frame.frameId, frame]));
await Promise.all(
graph.frames.map(async (frame) => {
if (!frame.parentFrameId) return;
const parent = frameById.get(frame.parentFrameId);
if (!parent) return;
try {
const owner = await this.sendToTarget<{ backendNodeId?: number }>(
parent.target,
"DOM.getFrameOwner",
{ frameId: frame.frameId },
);
if (owner.backendNodeId !== undefined) frame.ownerBackendNodeId = owner.backendNodeId;
} catch {
// The frame may have navigated between tree capture and owner lookup.
}
}),
);
return graph;
}
/** Return a cursor marking the current dialog sequence for `tabId`. */ /** Return a cursor marking the current dialog sequence for `tabId`. */
dialogCursor(tabId: number): DialogCursor { dialogCursor(tabId: number): DialogCursor {
return this.dialogSequences.get(tabId) ?? 0; return this.dialogSequences.get(tabId) ?? 0;
@@ -332,6 +431,7 @@ export class ChromiumCdp {
this.clearDialogState(tabId); this.clearDialogState(tabId);
this.clearConsoleState(tabId); this.clearConsoleState(tabId);
this.clearNetworkState(tabId); this.clearNetworkState(tabId);
this.clearFrameState(tabId);
try { try {
await this.api.detach({ tabId }); await this.api.detach({ tabId });
} catch (err) { } catch (err) {
@@ -354,7 +454,7 @@ export class ChromiumCdp {
} }
/** Subscribe to all CDP events. Returned disposable removes the listener. */ /** Subscribe to all CDP events. Returned disposable removes the listener. */
onEvent(handler: (source: chrome.debugger.Debuggee, method: string, params: unknown) => void): { onEvent(handler: (source: CdpDebuggee, method: string, params: unknown) => void): {
dispose(): void; dispose(): void;
} { } {
this.api.onEvent.addListener(handler); this.api.onEvent.addListener(handler);
@@ -378,6 +478,7 @@ export class ChromiumCdp {
this.networkSequences.clear(); this.networkSequences.clear();
this.networkDomainsEnabledTabs.clear(); this.networkDomainsEnabledTabs.clear();
this.networkRequestMeta.clear(); this.networkRequestMeta.clear();
this.frameDiscovery.clear();
await Promise.all( await Promise.all(
tabs.map(async (tabId) => { tabs.map(async (tabId) => {
try { try {
@@ -428,9 +529,104 @@ export class ChromiumCdp {
} }
} }
private async enableFrameDiscovery(target: CdpTarget): Promise<void> {
await this.api.sendCommand(target, "Target.setAutoAttach", {
autoAttach: true,
waitForDebuggerOnStart: false,
flatten: true,
filter: [{ type: "iframe", exclude: false }],
});
}
private bindFrameTargetHandler(): void {
if (this.frameTargetSubscription) return;
const listener = (source: CdpDebuggee, method: string, params: unknown) => {
const tabId = source.tabId;
if (typeof tabId !== "number") return;
const raw = (params ?? {}) as Record<string, unknown>;
const sessionId = typeof raw.sessionId === "string" ? raw.sessionId : undefined;
if (!sessionId) return;
if (method === "Target.detachedFromTarget") {
const state = this.frameDiscovery.get(tabId);
if (state?.sessions.delete(sessionId)) state.generation += 1;
return;
}
if (method !== "Target.attachedToTarget") return;
const targetInfo = raw.targetInfo as { type?: string } | undefined;
if (targetInfo?.type && targetInfo.type !== "iframe") return;
const state = this.frameDiscoveryState(tabId);
if (state.sessions.has(sessionId)) return;
state.sessions.add(sessionId);
state.generation += 1;
const task = this.initializeFrameTarget({ tabId, sessionId });
state.pending.add(task);
void task.finally(() => {
state.pending.delete(task);
});
};
this.api.onEvent.addListener(listener);
this.frameTargetSubscription = {
dispose: () => this.api.onEvent.removeListener(listener),
};
}
private async initializeFrameTarget(target: CdpTarget): Promise<void> {
try {
await this.enableFrameDiscovery(target);
} catch (err) {
const state = this.frameDiscovery.get(target.tabId);
if (state?.sessions.delete(target.sessionId as string)) state.generation += 1;
console.debug("[bsk cdp] child frame target initialization failed", { target, err });
}
}
private async drainFrameAttachTasks(tabId: number): Promise<void> {
const deadline = Date.now() + FRAME_DISCOVERY_TIMEOUT_MS;
while (Date.now() < deadline) {
const state = this.frameDiscovery.get(tabId);
const generation = state?.generation ?? 0;
if (state?.pending.size && !(await settleBeforeDeadline([...state.pending], deadline))) break;
// Target.setAutoAttach may enqueue the next attachedToTarget event after
// its command promise settles. Wait for a short quiet window, then finish
// only if neither the state object nor its generation changed.
const quietTime = Math.min(FRAME_DISCOVERY_QUIET_MS, deadline - Date.now());
if (quietTime <= 0) break;
await new Promise<void>((resolve) => setTimeout(resolve, quietTime));
const current = this.frameDiscovery.get(tabId);
if (
current === state &&
(current?.generation ?? 0) === generation &&
!current?.pending.size
) {
return;
}
}
console.debug("[bsk cdp] frame discovery did not reach quiescence before timeout", { tabId });
}
private frameDiscoveryState(tabId: number): FrameDiscoveryState {
const existing = this.frameDiscovery.get(tabId);
if (existing) return existing;
const created: FrameDiscoveryState = {
sessions: new Set(),
pending: new Set(),
generation: 0,
};
this.frameDiscovery.set(tabId, created);
return created;
}
private clearFrameState(tabId: number): void {
this.frameDiscovery.delete(tabId);
}
private bindDialogHandler(): void { private bindDialogHandler(): void {
if (this.dialogSubscription) return; if (this.dialogSubscription) return;
const listener = (source: chrome.debugger.Debuggee, method: string, params: unknown) => { const listener = (source: CdpDebuggee, method: string, params: unknown) => {
if (method !== "Page.javascriptDialogOpening") return; if (method !== "Page.javascriptDialogOpening") return;
const tabId = source.tabId; const tabId = source.tabId;
if (typeof tabId !== "number") return; if (typeof tabId !== "number") return;
@@ -614,6 +810,7 @@ export class ChromiumCdp {
this.clearDialogState(source.tabId); this.clearDialogState(source.tabId);
this.clearConsoleState(source.tabId); this.clearConsoleState(source.tabId);
this.clearNetworkState(source.tabId); this.clearNetworkState(source.tabId);
this.clearFrameState(source.tabId);
} }
}; };
this.api.onDetach.addListener(listener); this.api.onDetach.addListener(listener);
@@ -632,6 +829,8 @@ export class ChromiumCdp {
this.consoleSubscription = null; this.consoleSubscription = null;
this.networkSubscription?.dispose(); this.networkSubscription?.dispose();
this.networkSubscription = null; this.networkSubscription = null;
this.frameTargetSubscription?.dispose();
this.frameTargetSubscription = null;
} }
/** Detach tabs only when no other live session has claimed them. */ /** Detach tabs only when no other live session has claimed them. */
@@ -0,0 +1,107 @@
export interface CdpTarget {
tabId: number;
sessionId?: string;
}
export interface CdpFrame {
frameId: string;
parentFrameId?: string;
ownerBackendNodeId?: number;
url?: string;
target: CdpTarget;
}
export interface CdpFrameGraph {
rootFrameId: string;
frames: CdpFrame[];
}
export interface CdpFrameTreeNode {
frame: {
id: string;
parentId?: string;
url?: string;
};
childFrames?: CdpFrameTreeNode[];
}
export interface CdpFrameTreeSource {
target: CdpTarget;
tree: CdpFrameTreeNode;
}
export function cdpTargetKey(target: CdpTarget): string {
return `${target.tabId}:${target.sessionId ?? "root"}`;
}
function mergeFrame(
frames: Map<string, CdpFrame>,
order: string[],
node: CdpFrameTreeNode,
target: CdpTarget,
): void {
const existing = frames.get(node.frame.id);
if (!existing) order.push(node.frame.id);
const parentFrameId = node.frame.parentId || existing?.parentFrameId;
const url = node.frame.url || existing?.url;
frames.set(node.frame.id, {
frameId: node.frame.id,
...(parentFrameId ? { parentFrameId } : {}),
...(url ? { url } : {}),
target,
});
}
function walkFrameTree(
source: CdpFrameTreeSource,
targetByRootFrameId: ReadonlyMap<string, CdpTarget>,
frames: Map<string, CdpFrame>,
order: string[],
): void {
const stack: Array<{ node: CdpFrameTreeNode; inheritedTarget: CdpTarget }> = [
{ node: source.tree, inheritedTarget: source.target },
];
const expanded = new WeakSet<CdpFrameTreeNode>();
while (stack.length > 0) {
const current = stack.pop();
if (!current || expanded.has(current.node)) continue;
expanded.add(current.node);
const target = targetByRootFrameId.get(current.node.frame.id) ?? current.inheritedTarget;
mergeFrame(frames, order, current.node, target);
const children = current.node.childFrames ?? [];
for (let index = children.length - 1; index >= 0; index -= 1) {
stack.push({ node: children[index], inheritedTarget: target });
}
}
}
export function buildFrameGraph(sources: CdpFrameTreeSource[]): CdpFrameGraph | null {
const rootSource = sources.find((source) => source.target.sessionId === undefined) ?? sources[0];
if (!rootSource) return null;
const frames = new Map<string, CdpFrame>();
const order: string[] = [];
const targetByRootFrameId = new Map(
sources.map((source) => [source.tree.frame.id, source.target] as const),
);
// The top-level source is authoritative if malformed input reports the same
// frame as both a root target and a child target.
targetByRootFrameId.set(rootSource.tree.frame.id, rootSource.target);
walkFrameTree(rootSource, targetByRootFrameId, frames, order);
for (const source of sources) {
if (source === rootSource) continue;
walkFrameTree(source, targetByRootFrameId, frames, order);
}
return {
rootFrameId: rootSource.tree.frame.id,
frames: order.flatMap((frameId) => {
const frame = frames.get(frameId);
return frame ? [frame] : [];
}),
};
}
@@ -8,6 +8,7 @@ import {
useRef, useRef,
useState, useState,
} from "react"; } from "react";
import type { HelpHighlightRect } from "@/lib/help-bridge";
import logoUrl from "../../assets/logo.png"; import logoUrl from "../../assets/logo.png";
export interface HelpRequestData { export interface HelpRequestData {
@@ -19,6 +20,10 @@ export interface HelpRequestData {
displayMode?: "full" | "compact"; displayMode?: "full" | "compact";
/** CSS selectors to scroll to + flash-highlight. */ /** CSS selectors to scroll to + flash-highlight. */
selectors: string[]; selectors: string[];
/** Pre-resolved top-viewport rectangles, used for cross-frame targets. */
rects?: HelpHighlightRect[];
/** Re-resolve cross-frame rectangles after scroll, resize, or layout changes. */
refreshRects?: () => Promise<HelpHighlightRect[] | undefined>;
onContinue: (note: string) => void; onContinue: (note: string) => void;
onCancel: () => void; onCancel: () => void;
} }
@@ -49,6 +54,7 @@ const PANEL_WIDTH = 420;
const FALLBACK_PANEL_H = 180; const FALLBACK_PANEL_H = 180;
const FALLBACK_PANEL_H_COLLAPSED = 44; const FALLBACK_PANEL_H_COLLAPSED = 44;
const EMPTY_SELECTORS: string[] = []; const EMPTY_SELECTORS: string[] = [];
const EMPTY_RECTS: HelpHighlightRect[] = [];
function clamp(value: number, min: number, max: number): number { function clamp(value: number, min: number, max: number): number {
return Math.max(min, Math.min(max, value)); return Math.max(min, Math.min(max, value));
@@ -179,6 +185,8 @@ export function HelpRequestOverlay({ request }: Props) {
const { t } = useTranslation("extension"); const { t } = useTranslation("extension");
const isCompact = request?.displayMode === "compact"; const isCompact = request?.displayMode === "compact";
const effectiveSelectors = isCompact ? EMPTY_SELECTORS : (request?.selectors ?? EMPTY_SELECTORS); const effectiveSelectors = isCompact ? EMPTY_SELECTORS : (request?.selectors ?? EMPTY_SELECTORS);
const [liveRects, setLiveRects] = useState<HelpHighlightRect[] | null>(null);
const effectiveRects = isCompact ? EMPTY_RECTS : (liveRects ?? request?.rects ?? EMPTY_RECTS);
const [note, setNote] = useState(""); const [note, setNote] = useState("");
const [collapsed, setCollapsed] = useState(false); const [collapsed, setCollapsed] = useState(false);
const [boxes, setBoxes] = useState<Box[]>([]); const [boxes, setBoxes] = useState<Box[]>([]);
@@ -201,6 +209,7 @@ export function HelpRequestOverlay({ request }: Props) {
// Reset the note and collapse state whenever a new request appears. // Reset the note and collapse state whenever a new request appears.
useEffect(() => { useEffect(() => {
setNote(""); setNote("");
setLiveRects(null);
setCollapsed(false); setCollapsed(false);
placementRef.current = null; placementRef.current = null;
setDragPos(null); setDragPos(null);
@@ -209,6 +218,32 @@ export function HelpRequestOverlay({ request }: Props) {
dragStartRef.current = null; dragStartRef.current = null;
}, [request?.id]); }, [request?.id]);
useEffect(() => {
if (!request || isCompact || !request.refreshRects) return;
let disposed = false;
let resolving = false;
const refresh = async () => {
if (resolving) return;
resolving = true;
try {
const rects = await request.refreshRects?.();
if (!disposed && rects) setLiveRects(rects);
} finally {
resolving = false;
}
};
const onViewportChange = () => void refresh();
window.addEventListener("scroll", onViewportChange, true);
window.addEventListener("resize", onViewportChange);
const interval = window.setInterval(onViewportChange, 500);
return () => {
disposed = true;
window.removeEventListener("scroll", onViewportChange, true);
window.removeEventListener("resize", onViewportChange);
window.clearInterval(interval);
};
}, [request, isCompact]);
// Scroll the first matched target into view once per distinct request id. // Scroll the first matched target into view once per distinct request id.
useLayoutEffect(() => { useLayoutEffect(() => {
if (!request) return; if (!request) return;
@@ -231,7 +266,7 @@ export function HelpRequestOverlay({ request }: Props) {
return; return;
} }
const update = () => { const update = () => {
const nextBoxes = measure(effectiveSelectors); const nextBoxes = [...measure(effectiveSelectors), ...effectiveRects];
setBoxes((prev) => (boxesEqual(prev, nextBoxes) ? prev : nextBoxes)); setBoxes((prev) => (boxesEqual(prev, nextBoxes) ? prev : nextBoxes));
}; };
update(); update();
@@ -243,7 +278,7 @@ export function HelpRequestOverlay({ request }: Props) {
window.removeEventListener("resize", update); window.removeEventListener("resize", update);
window.clearInterval(interval); window.clearInterval(interval);
}; };
}, [request, effectiveSelectors]); }, [request, effectiveSelectors, effectiveRects]);
const onHeaderPointerDown = useCallback( const onHeaderPointerDown = useCallback(
(e: ReactPointerEvent<HTMLDivElement>) => { (e: ReactPointerEvent<HTMLDivElement>) => {
@@ -305,7 +340,7 @@ export function HelpRequestOverlay({ request }: Props) {
return; return;
} }
if (userMovedRef.current) return; if (userMovedRef.current) return;
const measured = boxes.length > 0 ? boxes : measure(effectiveSelectors); const measured = boxes.length > 0 ? boxes : [...measure(effectiveSelectors), ...effectiveRects];
const anchor = unionBox(measured); const anchor = unionBox(measured);
const banner = bannerRef.current; const banner = bannerRef.current;
if (!anchor || !banner) { if (!anchor || !banner) {
@@ -329,7 +364,7 @@ export function HelpRequestOverlay({ request }: Props) {
} }
const nextPos = placePanel(anchor, panelW, panelH, placementRef.current.placement); const nextPos = placePanel(anchor, panelW, panelH, placementRef.current.placement);
setPanelPos((prev) => (panelPosEqual(prev, nextPos) ? prev : nextPos)); setPanelPos((prev) => (panelPosEqual(prev, nextPos) ? prev : nextPos));
}, [request, boxes, collapsed, isCompact, effectiveSelectors]); }, [request, boxes, collapsed, isCompact, effectiveSelectors, effectiveRects]);
if (!request) return null; if (!request) return null;
@@ -1,6 +1,6 @@
import { i18n } from "@browser-skill/i18n"; import { i18n } from "@browser-skill/i18n";
import { I18nextProvider } from "@browser-skill/i18n/react"; import { I18nextProvider } from "@browser-skill/i18n/react";
import { cleanup, fireEvent, render, screen } from "@testing-library/react"; import { cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react";
import { createElement } from "react"; import { createElement } from "react";
import { afterEach, describe, expect, it, vi } from "vitest"; import { afterEach, describe, expect, it, vi } from "vitest";
import { type HelpRequestData, HelpRequestOverlay } from "../HelpRequestOverlay"; import { type HelpRequestData, HelpRequestOverlay } from "../HelpRequestOverlay";
@@ -38,6 +38,36 @@ describe("HelpRequestOverlay", () => {
expect(screen.getByText("Please complete the captcha")).toBeTruthy(); expect(screen.getByText("Please complete the captcha")).toBeTruthy();
}); });
it("renders explicit viewport rectangles for cross-frame targets", () => {
const { container } = renderOverlay(
baseRequest({ rects: [{ top: 30, left: 40, width: 120, height: 50 }] }),
);
const highlight = container.querySelector<HTMLElement>("[data-slot='help-highlight']");
expect(highlight?.style.top).toBe("30px");
expect(highlight?.style.left).toBe("40px");
expect(highlight?.style.width).toBe("120px");
expect(highlight?.style.height).toBe("50px");
});
it("re-resolves cross-frame rectangles after the viewport changes", async () => {
const refreshRects = vi.fn(async () => [{ top: 80, left: 90, width: 140, height: 60 }]);
const { container } = renderOverlay(
baseRequest({
rects: [{ top: 30, left: 40, width: 120, height: 50 }],
refreshRects,
}),
);
window.dispatchEvent(new Event("scroll"));
await waitFor(() => {
const highlight = container.querySelector<HTMLElement>("[data-slot='help-highlight']");
expect(highlight?.style.top).toBe("80px");
expect(highlight?.style.left).toBe("90px");
});
expect(refreshRects).toHaveBeenCalledTimes(1);
});
it("keeps the inactive render stable", () => { it("keeps the inactive render stable", () => {
const { container, rerender } = renderOverlay(null); const { container, rerender } = renderOverlay(null);
@@ -0,0 +1,64 @@
import { describe, expect, it, vi } from "vitest";
import { createHelpRequestData } from "../help-request";
describe("createHelpRequestData", () => {
it("gives live delivery and recovery the same rectangle refresh mechanism", async () => {
const finish = vi.fn();
const query = vi.fn(async () => ({
active: true,
request: {
requestId: "help-1",
prompt: "complete the challenge",
selectors: [],
rects: [{ top: 80, left: 90, width: 140, height: 60 }],
timeoutMs: 1_000,
},
}));
const request = createHelpRequestData(
{
requestId: "help-1",
prompt: "complete the challenge",
selectors: [],
rects: [{ top: 10, left: 20, width: 30, height: 40 }],
timeoutMs: 1_000,
},
{ finish, query },
);
await expect(request.refreshRects?.()).resolves.toEqual([
{ top: 80, left: 90, width: 140, height: 60 },
]);
request.onContinue(" done ");
request.onCancel();
expect(query).toHaveBeenCalledOnce();
expect(finish).toHaveBeenNthCalledWith(1, "help-1", "continued", " done ");
expect(finish).toHaveBeenNthCalledWith(2, "help-1", "cancelled");
});
it("ignores rectangle replies for a replaced request", async () => {
const request = createHelpRequestData(
{
requestId: "help-1",
prompt: "complete the challenge",
selectors: [],
timeoutMs: 1_000,
},
{
finish: vi.fn(),
query: vi.fn(async () => ({
active: true,
request: {
requestId: "help-2",
prompt: "new challenge",
selectors: [],
rects: [{ top: 1, left: 2, width: 3, height: 4 }],
timeoutMs: 1_000,
},
})),
},
);
await expect(request.refreshRects?.()).resolves.toBeUndefined();
});
});
@@ -0,0 +1,31 @@
import type { HelpQueryResponse, HelpRequestMessage } from "@/lib/help-bridge";
import type { HelpRequestData } from "./HelpRequestOverlay";
export interface HelpRequestDataDeps {
finish(requestId: string, outcome: "continued" | "cancelled", note?: string): void;
query(): Promise<HelpQueryResponse | undefined>;
}
/** Build the content-side representation shared by live delivery and recovery. */
export function createHelpRequestData(
message: Omit<HelpRequestMessage, "type">,
deps: HelpRequestDataDeps,
): HelpRequestData {
return {
id: message.requestId,
prompt: message.prompt,
...(message.title ? { title: message.title } : {}),
...(message.displayMode ? { displayMode: message.displayMode } : {}),
selectors: message.selectors,
rects: message.rects,
refreshRects: async () => {
const response = await deps.query();
return response?.active && response.request?.requestId === message.requestId
? response.request.rects
: undefined;
},
onContinue: (note: string) =>
deps.finish(message.requestId, "continued", note.trim() ? note : undefined),
onCancel: () => deps.finish(message.requestId, "cancelled"),
};
}
+14 -21
View File
@@ -7,6 +7,7 @@ import { BorrowConfirmationOverlay } from "@/content/BorrowConfirmationOverlay";
import { ControlOverlay } from "@/content/ControlOverlay"; import { ControlOverlay } from "@/content/ControlOverlay";
import { createCaptureSuppressController } from "@/content/capture-suppress"; import { createCaptureSuppressController } from "@/content/capture-suppress";
import { HelpRequestOverlay } from "@/content/HelpRequestOverlay"; import { HelpRequestOverlay } from "@/content/HelpRequestOverlay";
import { createHelpRequestData } from "@/content/help-request";
import overlayCss from "@/content/overlay.css?inline"; import overlayCss from "@/content/overlay.css?inline";
import { OverlayController, shouldShowAgentControlOverlay } from "@/content/overlay-controller"; import { OverlayController, shouldShowAgentControlOverlay } from "@/content/overlay-controller";
import { RecordOverlay } from "@/content/RecordOverlay"; import { RecordOverlay } from "@/content/RecordOverlay";
@@ -344,16 +345,7 @@ export default defineContentScript({
if (isHelpRequestMessage(message)) { if (isHelpRequestMessage(message)) {
const helpMsg = message as HelpRequestMessage; const helpMsg = message as HelpRequestMessage;
const previousHelp = overlays.setAgentHelpRequest({ const previousHelp = mountHelpRequest(helpMsg);
id: helpMsg.requestId,
prompt: helpMsg.prompt,
...(helpMsg.title ? { title: helpMsg.title } : {}),
...(helpMsg.displayMode ? { displayMode: helpMsg.displayMode } : {}),
selectors: helpMsg.selectors,
onContinue: (note: string) =>
void sendHelpFinish(helpMsg.requestId, "continued", note.trim() ? note : undefined),
onCancel: () => void sendHelpFinish(helpMsg.requestId, "cancelled"),
});
if (previousHelp && previousHelp.id !== helpMsg.requestId) { if (previousHelp && previousHelp.id !== helpMsg.requestId) {
void sendHelpFinish(previousHelp.id, "cancelled"); void sendHelpFinish(previousHelp.id, "cancelled");
} }
@@ -406,17 +398,18 @@ export default defineContentScript({
}); });
} }
function mountHelpRequest(helpMsg: Omit<HelpRequestMessage, "type">): void { function mountHelpRequest(helpMsg: Omit<HelpRequestMessage, "type">) {
overlays.setAgentHelpRequest({ return overlays.setAgentHelpRequest(
id: helpMsg.requestId, createHelpRequestData(helpMsg, {
prompt: helpMsg.prompt, finish: (requestId, outcome, note) => {
...(helpMsg.title ? { title: helpMsg.title } : {}), void sendHelpFinish(requestId, outcome, note);
...(helpMsg.displayMode ? { displayMode: helpMsg.displayMode } : {}), },
selectors: helpMsg.selectors, query: () =>
onContinue: (note: string) => chrome.runtime.sendMessage({ type: HELP_QUERY }) as Promise<
void sendHelpFinish(helpMsg.requestId, "continued", note.trim() ? note : undefined), HelpQueryResponse | undefined
onCancel: () => void sendHelpFinish(helpMsg.requestId, "cancelled"), >,
}); }),
);
} }
async function queryActiveHelpWithRetry(): Promise<void> { async function queryActiveHelpWithRetry(): Promise<void> {
@@ -32,6 +32,7 @@ describe("help-bridge", () => {
prompt: "log in", prompt: "log in",
displayMode: "compact", displayMode: "compact",
selectors: ["#login"], selectors: ["#login"],
rects: [{ top: 10, left: 20, width: 100, height: 40 }],
timeoutMs: 1000, timeoutMs: 1000,
}), }),
).toBe(true); ).toBe(true);
@@ -104,6 +104,27 @@ describe("reduceTraceSteps", () => {
}); });
}); });
it("keeps committed empty fill steps", () => {
const { steps } = reduceTraceSteps(
[
{
op: "fill",
target: { tag: "input", role: "textbox", name: "Search query" },
value: "",
page_url: "https://example.com/search",
},
],
"https://example.com/search",
);
expect(steps).toEqual([
expect.objectContaining({
op: "fill",
value: "",
}),
]);
});
it("maps select navigated_to onto effect.navigated_to (page id)", () => { it("maps select navigated_to onto effect.navigated_to (page id)", () => {
const { pages, steps } = reduceTraceSteps( const { pages, steps } = reduceTraceSteps(
[ [
+19
View File
@@ -16,6 +16,13 @@ export const HELP_ACK = "bsk-help-ack";
export const HELP_QUERY = "bsk-help-query"; export const HELP_QUERY = "bsk-help-query";
export const HELP_FINISH = "bsk-help-finish"; export const HELP_FINISH = "bsk-help-finish";
export interface HelpHighlightRect {
top: number;
left: number;
width: number;
height: number;
}
export interface HelpRequestMessage { export interface HelpRequestMessage {
type: typeof HELP_REQUEST; type: typeof HELP_REQUEST;
requestId: string; requestId: string;
@@ -26,6 +33,8 @@ export interface HelpRequestMessage {
displayMode?: "full" | "compact"; displayMode?: "full" | "compact";
/** CSS selectors to scroll to + flash-highlight (may be empty). */ /** CSS selectors to scroll to + flash-highlight (may be empty). */
selectors: string[]; selectors: string[];
/** Top-viewport rectangles for targets that cannot be represented by a root selector. */
rects?: HelpHighlightRect[];
timeoutMs: number; timeoutMs: number;
} }
@@ -74,6 +83,16 @@ export function isHelpRequestMessage(msg: unknown): msg is HelpRequestMessage {
(m.displayMode === undefined || m.displayMode === "full" || m.displayMode === "compact") && (m.displayMode === undefined || m.displayMode === "full" || m.displayMode === "compact") &&
Array.isArray(m.selectors) && Array.isArray(m.selectors) &&
m.selectors.every((selector) => typeof selector === "string") && m.selectors.every((selector) => typeof selector === "string") &&
(m.rects === undefined ||
(Array.isArray(m.rects) &&
m.rects.every(
(rect) =>
typeof rect === "object" &&
rect !== null &&
["top", "left", "width", "height"].every((key) =>
Number.isFinite((rect as Record<string, unknown>)[key]),
),
))) &&
typeof m.timeoutMs === "number" typeof m.timeoutMs === "number"
); );
} }
-1
View File
@@ -18,7 +18,6 @@ export function shouldRecordPress(
} }
function shouldIncludeDraft(step: DraftTraceStep): boolean { function shouldIncludeDraft(step: DraftTraceStep): boolean {
if (step.op === "fill" && !(step.value ?? "").trim() && !step.redacted) return false;
if (step.op === "press" && !shouldRecordPress(step.key, step.modifiers)) return false; if (step.op === "press" && !shouldRecordPress(step.key, step.modifiers)) return false;
return true; return true;
} }
@@ -33,4 +33,19 @@ describe("RefStore", () => {
expect(s.resolve("@e11")).toBe(11); expect(s.resolve("@e11")).toBe(11);
expect(s.size()).toBe(2); expect(s.size()).toBe(2);
}); });
it("preserves the child CDP session as part of ref identity", () => {
const s = new RefStore();
s.set("e1", 42, {
tabId: 7,
frameId: "child-frame",
cdpSessionId: "child-session",
});
expect(s.resolveEntry("e1")).toMatchObject({
backendNodeId: 42,
tabId: 7,
frameId: "child-frame",
cdpSessionId: "child-session",
});
});
}); });
@@ -1,9 +1,10 @@
/** /**
* Per-session map from `@e<N>` snapshot refs to CDP `backendNodeId`. * Per-session map from `@e<N>` snapshot refs to a CDP node address.
* *
* Each fresh `tool.snapshot` resets the store: M6 will call * Each fresh `tool.snapshot` resets the store: M6 will call
* `replace(...)` with the new ref → backendNodeId pairs. Tools like * `replace(...)` with the new ref → node address pairs. A node address
* `tool.click` consume the store via `resolve("@e1")`. * includes the owning flat CDP session/frame when the element lives in
* an OOPIF; tools resolve live geometry from that identity at call time.
* *
* Refs are session-scoped (§7): looking up a ref in the wrong session * Refs are session-scoped (§7): looking up a ref in the wrong session
* returns `null`, never silently leaks. Storing values in different * returns `null`, never silently leaks. Storing values in different
@@ -15,10 +16,19 @@ export type BackendNodeId = number;
export interface RefEntry { export interface RefEntry {
backendNodeId: BackendNodeId; backendNodeId: BackendNodeId;
tabId: number | null; tabId: number | null;
frameId?: string;
cdpSessionId?: string;
generation: number; generation: number;
} }
type RefInput = BackendNodeId | { backendNodeId: BackendNodeId; tabId: number }; export type RefInput =
| BackendNodeId
| {
backendNodeId: BackendNodeId;
tabId: number;
frameId?: string;
cdpSessionId?: string;
};
export class RefStore { export class RefStore {
private readonly map = new Map<string, RefEntry>(); private readonly map = new Map<string, RefEntry>();
@@ -44,7 +54,7 @@ export class RefStore {
} }
/** /**
* Replace the entire store with a new ref → backendNodeId mapping. * Replace the entire store with a new ref → CDP node identity mapping.
* Used after every fresh `tool.snapshot`. * Used after every fresh `tool.snapshot`.
*/ */
replace(entries: Iterable<readonly [string, RefInput]>): void { replace(entries: Iterable<readonly [string, RefInput]>): void {
@@ -53,10 +63,20 @@ export class RefStore {
for (const [ref, input] of entries) this.map.set(normaliseRef(ref), this.entry(input)); for (const [ref, input] of entries) this.map.set(normaliseRef(ref), this.entry(input));
} }
set(ref: string, id: BackendNodeId, opts: { tabId?: number } = {}): void { set(
ref: string,
id: BackendNodeId,
opts: {
tabId?: number;
frameId?: string;
cdpSessionId?: string;
} = {},
): void {
this.map.set(normaliseRef(ref), { this.map.set(normaliseRef(ref), {
backendNodeId: id, backendNodeId: id,
tabId: opts.tabId ?? null, tabId: opts.tabId ?? null,
...(opts.frameId ? { frameId: opts.frameId } : {}),
...(opts.cdpSessionId ? { cdpSessionId: opts.cdpSessionId } : {}),
generation: this.generation, generation: this.generation,
}); });
} }
@@ -80,6 +100,8 @@ export class RefStore {
return { return {
backendNodeId: input.backendNodeId, backendNodeId: input.backendNodeId,
tabId: input.tabId, tabId: input.tabId,
...(input.frameId ? { frameId: input.frameId } : {}),
...(input.cdpSessionId ? { cdpSessionId: input.cdpSessionId } : {}),
generation: this.generation, generation: this.generation,
}; };
} }
@@ -612,4 +612,14 @@ describe("isInjectableContentScriptUrl", () => {
).toBe(false); ).toBe(false);
expect(isInjectableContentScriptUrl("https://chromewebstore.google.com/")).toBe(false); expect(isInjectableContentScriptUrl("https://chromewebstore.google.com/")).toBe(false);
}); });
it("rejects the Edge Add-ons store", () => {
expect(
isInjectableContentScriptUrl(
"https://microsoftedge.microsoft.com/addons/detail/browserskill/emacgiaaaiojkkpkddmmdfhmokgmnikg",
),
).toBe(false);
// Non-store paths on the same host stay injectable.
expect(isInjectableContentScriptUrl("https://microsoftedge.microsoft.com/")).toBe(true);
});
}); });
@@ -1,24 +1,54 @@
import { describe, expect, it } from "vitest"; import { describe, expect, it } from "vitest";
import { nodeBoundingRect, quadBoundingRect } from "../element-geometry"; import { nodeContentRegion } from "../element-geometry";
import type { CdpRunner } from "../shared"; import type { CdpRunner } from "../shared";
describe("quadBoundingRect", () => { describe("nodeContentRegion", () => {
it("computes axis-aligned bounds from an 8-double quad", () => { it("rejects invalid content quads before using the box model fallback", async () => {
expect(quadBoundingRect([10, 20, 110, 20, 110, 60, 10, 60])).toEqual({ const send = async (_tabId: number, method: string) => {
x: 10, if (method === "DOM.getContentQuads") return { quads: [[0, 0, 0, 0, 0, 0, 0, 0]] };
y: 20, if (method === "DOM.getBoxModel") {
width: 100, return { model: { content: [5, 10, 25, 10, 25, 30, 5, 30] } };
height: 40, }
throw new Error(`unexpected CDP call ${method}`);
};
await expect(nodeContentRegion({ send: send as CdpRunner["send"] }, 7, 555)).resolves.toEqual([
[
{ x: 5, y: 10 },
{ x: 25, y: 10 },
{ x: 25, y: 30 },
{ x: 5, y: 30 },
],
]);
});
it("preserves every visible content fragment", async () => {
const send = async () => ({
quads: [
[10, 20, 60, 20, 60, 40, 10, 40],
[15, 50, 90, 50, 90, 80, 15, 80],
],
}); });
const cdp: CdpRunner = {
send: send as CdpRunner["send"],
};
await expect(nodeContentRegion(cdp, 7, 555)).resolves.toEqual([
[
{ x: 10, y: 20 },
{ x: 60, y: 20 },
{ x: 60, y: 40 },
{ x: 10, y: 40 },
],
[
{ x: 15, y: 50 },
{ x: 90, y: 50 },
{ x: 90, y: 80 },
{ x: 15, y: 80 },
],
]);
}); });
it("returns null for degenerate quads", () => {
expect(quadBoundingRect([0, 0, 0, 0, 0, 0, 0, 0])).toBeNull();
expect(quadBoundingRect([1, 2, 3])).toBeNull();
});
});
describe("nodeBoundingRect", () => {
it("falls back to visible descendant bounds for zero-size containers", async () => { it("falls back to visible descendant bounds for zero-size containers", async () => {
const calls: Array<{ method: string; params?: object }> = []; const calls: Array<{ method: string; params?: object }> = [];
const send = async (_tabId: number, method: string, params?: object) => { const send = async (_tabId: number, method: string, params?: object) => {
@@ -38,12 +68,14 @@ describe("nodeBoundingRect", () => {
}; };
const cdp = { send: send as CdpRunner["send"] }; const cdp = { send: send as CdpRunner["send"] };
await expect(nodeBoundingRect(cdp, 7, 555)).resolves.toEqual({ await expect(nodeContentRegion(cdp, 7, 555)).resolves.toEqual([
x: 242, [
y: 468, { x: 242, y: 468 },
width: 769, { x: 1011, y: 468 },
height: 180, { x: 1011, y: 648 },
}); { x: 242, y: 648 },
],
]);
expect(calls.map((c) => c.method)).toEqual([ expect(calls.map((c) => c.method)).toEqual([
"DOM.getContentQuads", "DOM.getContentQuads",
"DOM.getBoxModel", "DOM.getBoxModel",
@@ -0,0 +1,286 @@
import { describe, expect, it, vi } from "vitest";
import { resolveFrameProjection, resolveNodeGeometry } from "../frame-geometry";
import {
clipPolygon,
polygonArea,
polygonBounds,
projectAndClipRegion,
projectUnitPoint,
type Quad,
rectPolygon,
regionBounds,
} from "../geometry";
import type { CdpRunner } from "../shared";
describe("frame geometry projection", () => {
it("keeps region bounds separate from polygon area", () => {
const region = [
rectPolygon({ x: 0, y: 0, w: 10, h: 10 }),
rectPolygon({ x: 20, y: 5, w: 5, h: 5 }),
];
expect(regionBounds(region)).toEqual({ x: 0, y: 0, width: 25, height: 10 });
expect(polygonArea(region[0])).toBe(100);
});
it("maps through the iframe content box instead of its border box", () => {
const contentQuad: Quad = [
{ x: 204, y: 306 },
{ x: 604, y: 306 },
{ x: 604, y: 506 },
{ x: 204, y: 506 },
];
const projected = projectAndClipRegion(
[rectPolygon({ x: 10, y: 20, w: 100, h: 40 })],
[{ sourceViewport: { width: 200, height: 100 }, destinationQuad: contentQuad }],
{ width: 1000, height: 800 },
);
expect(polygonBounds(projected.flat())).toEqual({ x: 224, y: 346, width: 200, height: 80 });
});
it("uses a projective mapping for perspective-transformed iframe quads", () => {
const quad: Quad = [
{ x: 0, y: 0 },
{ x: 200, y: 20 },
{ x: 180, y: 140 },
{ x: 20, y: 100 },
];
expect(projectUnitPoint({ x: 0, y: 0 }, quad)).toEqual(quad[0]);
expect(projectUnitPoint({ x: 1, y: 0 }, quad)).toEqual(quad[1]);
expect(projectUnitPoint({ x: 1, y: 1 }, quad)).toEqual(quad[2]);
expect(projectUnitPoint({ x: 0, y: 1 }, quad)).toEqual(quad[3]);
});
it("clips against the iframe polygon rather than its bounding box", () => {
const frameQuad: Quad = [
{ x: 50, y: 0 },
{ x: 100, y: 50 },
{ x: 50, y: 100 },
{ x: 0, y: 50 },
];
expect(clipPolygon(rectPolygon({ x: 0, y: 0, w: 10, h: 10 }), frameQuad)).toEqual([]);
});
it("does not reapply same-target frame transforms around an OOPIF", async () => {
const send = vi.fn(async (_tabId, method, params) => {
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 800, clientHeight: 600 } };
}
if (method === "DOM.getBoxModel") {
const backendNodeId = (params as { backendNodeId?: number }).backendNodeId;
return backendNodeId === 20
? { model: { content: [110, 70, 310, 70, 310, 170, 110, 170] } }
: { model: { content: [100, 50, 400, 50, 400, 250, 100, 250] } };
}
throw new Error(`unexpected root command ${method}`);
});
const cdp: CdpRunner = {
send: send as CdpRunner["send"],
sendToTarget: vi.fn(async (_target, method) => {
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
}
throw new Error(`unexpected child command ${method}`);
}) as CdpRunner["sendToTarget"],
};
const projection = await resolveFrameProjection(
cdp,
{
rootFrameId: "main",
frames: [
{ frameId: "main", target: { tabId: 4 } },
{
frameId: "same-process-parent",
parentFrameId: "main",
ownerBackendNodeId: 10,
target: { tabId: 4 },
},
{
frameId: "oopif",
parentFrameId: "same-process-parent",
ownerBackendNodeId: 20,
target: { tabId: 4, sessionId: "oopif-session" },
},
],
},
"oopif",
);
expect(projection?.edges).toEqual([
{
sourceViewport: { width: 200, height: 100 },
destinationQuad: [
{ x: 110, y: 70 },
{ x: 310, y: 70 },
{ x: 310, y: 170 },
{ x: 110, y: 170 },
],
destinationClips: [
[
{ x: 100, y: 50 },
{ x: 400, y: 50 },
{ x: 400, y: 250 },
{ x: 100, y: 250 },
],
],
},
]);
});
it("clips live OOPIF geometry at every frame boundary", async () => {
const cdp: CdpRunner = {
send: vi.fn(async (_tabId, method) => {
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 800, clientHeight: 600 } };
}
if (method === "DOM.getBoxModel") {
return { model: { content: [100, 100, 300, 100, 300, 200, 100, 200] } };
}
throw new Error(`unexpected root command ${method}`);
}) as CdpRunner["send"],
sendToTarget: vi.fn(async (_target, method) => {
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
}
if (method === "DOM.getContentQuads") {
return { quads: [[-50, -20, 250, -20, 250, 120, -50, 120]] };
}
throw new Error(`unexpected child command ${method}`);
}) as CdpRunner["sendToTarget"],
getFrameGraph: vi.fn(async () => ({
rootFrameId: "main",
frames: [
{ frameId: "main", target: { tabId: 4 } },
{
frameId: "child",
parentFrameId: "main",
ownerBackendNodeId: 10,
target: { tabId: 4, sessionId: "child-session" },
},
],
})),
};
const geometry = await resolveNodeGeometry(cdp, 4, {
target: { tabId: 4, sessionId: "child-session" },
frameId: "child",
backendNodeId: 101,
});
expect(geometry).toMatchObject({
topBounds: { x: 100, y: 100, width: 200, height: 100 },
actionPoint: { x: 200, y: 150 },
});
});
it("reuses one frame graph while scrolling and projecting live geometry", async () => {
const getFrameGraph = vi.fn(async () => ({
rootFrameId: "main",
frames: [
{ frameId: "main", target: { tabId: 4 } },
{
frameId: "child",
parentFrameId: "main",
ownerBackendNodeId: 10,
target: { tabId: 4, sessionId: "child-session" },
},
],
}));
const cdp: CdpRunner = {
send: vi.fn(async (_tabId, method) => {
if (method === "DOM.scrollIntoViewIfNeeded") return {};
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 800, clientHeight: 600 } };
}
if (method === "DOM.getBoxModel") {
return { model: { content: [100, 100, 300, 100, 300, 200, 100, 200] } };
}
throw new Error(`unexpected root command ${method}`);
}) as CdpRunner["send"],
sendToTarget: vi.fn(async (_target, method) => {
if (method === "DOM.scrollIntoViewIfNeeded") return {};
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
}
if (method === "DOM.getContentQuads") {
return { quads: [[0, 0, 20, 0, 20, 20, 0, 20]] };
}
throw new Error(`unexpected child command ${method}`);
}) as CdpRunner["sendToTarget"],
getFrameGraph,
};
await expect(
resolveNodeGeometry(
cdp,
4,
{
target: { tabId: 4, sessionId: "child-session" },
frameId: "child",
backendNodeId: 101,
},
{ scrollIntoView: true },
),
).resolves.not.toMatchObject({ code: expect.any(String) });
expect(getFrameGraph).toHaveBeenCalledOnce();
});
it("clips live geometry to same-target iframe ancestors", async () => {
const cdp: CdpRunner = {
send: vi.fn(async (_tabId, method) => {
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 800, clientHeight: 600 } };
}
if (method === "DOM.getBoxModel") {
return { model: { content: [100, 100, 300, 100, 300, 200, 100, 200] } };
}
if (method === "DOM.getContentQuads") {
return { quads: [[50, 50, 350, 50, 350, 250, 50, 250]] };
}
throw new Error(`unexpected ${method}`);
}) as CdpRunner["send"],
getFrameGraph: vi.fn(async () => ({
rootFrameId: "main",
frames: [
{ frameId: "main", target: { tabId: 4 } },
{
frameId: "child",
parentFrameId: "main",
ownerBackendNodeId: 10,
target: { tabId: 4 },
},
],
})),
};
const geometry = await resolveNodeGeometry(cdp, 4, {
target: { tabId: 4 },
frameId: "child",
backendNodeId: 101,
});
expect(geometry).toMatchObject({
topBounds: { x: 100, y: 100, width: 200, height: 100 },
actionPoint: { x: 200, y: 150 },
});
});
it("fails closed when an OOPIF frame graph is unavailable", async () => {
const cdp: CdpRunner = {
send: vi.fn(),
sendToTarget: vi.fn(async (_target, method) => {
if (method === "DOM.getContentQuads") {
return { quads: [[0, 0, 20, 0, 20, 20, 0, 20]] };
}
throw new Error(`unexpected ${method}`);
}) as CdpRunner["sendToTarget"],
};
await expect(
resolveNodeGeometry(cdp, 4, {
target: { tabId: 4, sessionId: "child-session" },
frameId: "child",
backendNodeId: 101,
}),
).resolves.toMatchObject({ code: "cdp_failed" });
});
});
@@ -1,5 +1,6 @@
import { afterEach, describe, expect, it, vi } from "vitest"; import { afterEach, describe, expect, it, vi } from "vitest";
import { SessionManager } from "@/session-manager/manager"; import { SessionManager } from "@/session-manager/manager";
import { RefStore } from "@/session-manager/ref-store";
import type { RequestHelpParams } from "@/transport/types"; import type { RequestHelpParams } from "@/transport/types";
import { handleRequestHelp, type RequestHelpDeps, resetHelpLifecycleForTests } from "../human-loop"; import { handleRequestHelp, type RequestHelpDeps, resetHelpLifecycleForTests } from "../human-loop";
@@ -50,11 +51,10 @@ function installHelpLifecycleChrome() {
} }
function fakeManager(sessionId: string, agentWindowId: number, tabId: number) { function fakeManager(sessionId: string, agentWindowId: number, tabId: number) {
const refStore = new RefStore();
const mgr = { const mgr = {
get: (id: string) => get: (id: string) =>
id === sessionId id === sessionId ? { sessionId, agentWindowId, refStore, borrowedTabs: new Map() } : null,
? { sessionId, agentWindowId, refStore: { resolve: () => null }, borrowedTabs: new Map() }
: null,
findByWindowId: (wid: number) => (wid === agentWindowId ? { sessionId } : null), findByWindowId: (wid: number) => (wid === agentWindowId ? { sessionId } : null),
} as unknown as SessionManager; } as unknown as SessionManager;
return mgr; return mgr;
@@ -339,26 +339,23 @@ describe("handleRequestHelp", () => {
ac.abort(); ac.abort();
await expect(pending).resolves.toMatchObject({ code: "cancelled" }); await expect(pending).resolves.toMatchObject({ code: "cancelled" });
expect(cdpSend).toHaveBeenCalledWith( expect(cdpSend).not.toHaveBeenCalledWith(
5, expect.anything(),
"DOM.querySelectorAll", "DOM.removeAttribute",
expect.objectContaining({ selector: "[data-bsk-help]" }), expect.anything(),
);
expect(cdpSend).toHaveBeenCalledWith(
6,
"DOM.querySelectorAll",
expect.objectContaining({ selector: "[data-bsk-help]" }),
); );
}); });
it("tags ref targets via CDP and reports them matched", async () => { it("resolves ref targets to explicit viewport rectangles without mutating the page", async () => {
const refStore = new RefStore();
refStore.set("e1", 42, { tabId: 5 });
const mgr = { const mgr = {
get: (id: string) => get: (id: string) =>
id === "abcd" id === "abcd"
? { ? {
sessionId: "abcd", sessionId: "abcd",
agentWindowId: 99, agentWindowId: 99,
refStore: { resolve: () => 42 }, refStore,
borrowedTabs: new Map(), borrowedTabs: new Map(),
} }
: null, : null,
@@ -366,7 +363,17 @@ describe("handleRequestHelp", () => {
} as unknown as SessionManager; } as unknown as SessionManager;
const deps = baseDeps({ const deps = baseDeps({
cdp: { cdp: {
send: vi.fn(async () => ({ object: { objectId: "obj-1" } })), send: vi.fn(async (_tabId, method) => {
if (method === "DOM.getDocument") return { root: { nodeId: 1 } };
if (method === "DOM.scrollIntoViewIfNeeded") return {};
if (method === "DOM.getContentQuads") {
return { quads: [[10, 20, 110, 20, 110, 60, 10, 60]] };
}
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
}
throw new Error(`unexpected ${method}`);
}),
} as unknown as RequestHelpDeps["cdp"], } as unknown as RequestHelpDeps["cdp"],
}); });
const res = await handleRequestHelp( const res = await handleRequestHelp(
@@ -375,14 +382,97 @@ describe("handleRequestHelp", () => {
deps, deps,
); );
const sentMsg = (deps.sendToTab as ReturnType<typeof vi.fn>).mock.calls[0][1]; const sentMsg = (deps.sendToTab as ReturnType<typeof vi.fn>).mock.calls[0][1];
expect(sentMsg.selectors).toContain('[data-bsk-help="0"]'); expect(sentMsg.selectors).toEqual([]);
expect(sentMsg.rects).toEqual([{ top: 20, left: 10, width: 100, height: 40 }]);
expect(res).toMatchObject({ expect(res).toMatchObject({
outcome: "continued", outcome: "continued",
tab_id: 5, tab_id: 5,
resolved_targets: [{ matched: true, ref: "@e1" }], resolved_targets: [
{
matched: true,
ref: "@e1",
},
],
}); });
}); });
it("projects an OOPIF ref through its live content quad for the help overlay", async () => {
const mgr = {
get: (id: string) =>
id === "abcd"
? {
sessionId: "abcd",
agentWindowId: 99,
refStore: {
resolveEntry: () => ({
backendNodeId: 42,
tabId: 5,
frameId: "child",
cdpSessionId: "child-session",
generation: 1,
}),
},
borrowedTabs: new Map(),
}
: null,
findByWindowId: (wid: number) => (wid === 99 ? { sessionId: "abcd" } : null),
} as unknown as SessionManager;
const send = vi.fn(async (_tabId, method) => {
if (method === "DOM.getDocument") return { root: { nodeId: 1 } };
if (method === "DOM.scrollIntoViewIfNeeded") return {};
if (method === "DOM.getBoxModel") {
return { model: { content: [204, 306, 604, 306, 604, 506, 204, 506] } };
}
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
}
throw new Error(`unexpected root ${method}`);
});
const sendToTarget = vi.fn(async (_target, method) => {
if (method === "DOM.scrollIntoViewIfNeeded") return {};
if (method === "DOM.getContentQuads") {
return { quads: [[10, 20, 110, 20, 110, 60, 10, 60]] };
}
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
}
throw new Error(`unexpected child ${method}`);
});
const deps = baseDeps({
cdp: {
send,
sendToTarget,
getFrameGraph: vi.fn(async () => ({
rootFrameId: "main",
frames: [
{ frameId: "main", target: { tabId: 5 } },
{
frameId: "child",
parentFrameId: "main",
ownerBackendNodeId: 99,
target: { tabId: 5, sessionId: "child-session" },
},
],
})),
} as unknown as NonNullable<RequestHelpDeps["cdp"]>,
});
const res = await handleRequestHelp(
mgr,
baseParams({ tab_id: 5, targets: [{ ref: "@e1" }] }),
deps,
);
const sentMsg = (deps.sendToTab as ReturnType<typeof vi.fn>).mock.calls[0][1];
expect(sentMsg.rects).toEqual([{ top: 346, left: 224, width: 200, height: 80 }]);
expect(res).toMatchObject({ resolved_targets: [{ matched: true, ref: "@e1" }] });
expect(sendToTarget).toHaveBeenCalledWith(
{ tabId: 5, sessionId: "child-session" },
"DOM.getContentQuads",
{ backendNodeId: 42 },
);
});
it("reports ref target unmatched when ref does not resolve", async () => { it("reports ref target unmatched when ref does not resolve", async () => {
const deps = baseDeps(); const deps = baseDeps();
const res = await handleRequestHelp( const res = await handleRequestHelp(
@@ -398,16 +488,15 @@ describe("handleRequestHelp", () => {
}); });
it("reports ref target unmatched when ref is for another tab", async () => { it("reports ref target unmatched when ref is for another tab", async () => {
const refStore = new RefStore();
refStore.set("e1", 42, { tabId: 4 });
const mgr = { const mgr = {
get: (id: string) => get: (id: string) =>
id === "abcd" id === "abcd"
? { ? {
sessionId: "abcd", sessionId: "abcd",
agentWindowId: 99, agentWindowId: 99,
refStore: { refStore,
resolve: (ref: string, opts: { tabId?: number }) =>
ref === "e1" && opts.tabId === 4 ? 42 : null,
},
borrowedTabs: new Map(), borrowedTabs: new Map(),
} }
: null, : null,
@@ -30,6 +30,9 @@ function makeFakeCdp(handlers: Record<string, (params: unknown) => unknown>) {
const sendImpl = async (tabId: number, method: string, params?: object) => { const sendImpl = async (tabId: number, method: string, params?: object) => {
sent.push({ tabId, method, params }); sent.push({ tabId, method, params });
const h = handlers[method]; const h = handlers[method];
if (!h && method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
}
if (!h) throw new Error(`unexpected CDP call ${method}`); if (!h) throw new Error(`unexpected CDP call ${method}`);
return h(params); return h(params);
}; };
@@ -164,6 +167,63 @@ describe("handleClick", () => {
}); });
}); });
it("resolves frame refs in their CDP session and dispatches input in top coordinates", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
ctx.refStore.set("e3", 1234, {
tabId: 4,
frameId: "child-frame",
cdpSessionId: "child-session",
});
const fake = makeFakeCdp({
"Input.dispatchMouseEvent": () => ({}),
"DOM.scrollIntoViewIfNeeded": () => ({}),
"DOM.getBoxModel": () => ({
model: { content: [204, 306, 604, 306, 604, 506, 204, 506] },
}),
});
fake.cdp.getFrameGraph = vi.fn(async () => ({
rootFrameId: "main",
frames: [
{ frameId: "main", target: { tabId: 4 } },
{
frameId: "child-frame",
parentFrameId: "main",
ownerBackendNodeId: 99,
target: { tabId: 4, sessionId: "child-session" },
},
],
}));
const targetCalls: Array<{ sessionId?: string; method: string }> = [];
fake.cdp.sendToTarget = vi.fn(async (target, method) => {
targetCalls.push({ sessionId: target.sessionId, method });
if (method === "DOM.scrollIntoViewIfNeeded") return {};
if (method === "DOM.getContentQuads") {
return { quads: [[10, 20, 110, 20, 110, 60, 10, 60]] };
}
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
}
throw new Error(`unexpected child CDP call ${method}`);
}) as CdpRunner["sendToTarget"];
const res = await handleClick(
sm,
{ session_id: "aa11", ref: "@e3" },
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
);
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
// The iframe content box starts at (204, 306) after its border and is scaled 2x.
expect(res).toMatchObject({ x: 324, y: 386 });
expect(targetCalls).toEqual([
{ sessionId: "child-session", method: "DOM.scrollIntoViewIfNeeded" },
{ sessionId: "child-session", method: "DOM.getContentQuads" },
{ sessionId: "child-session", method: "Page.getLayoutMetrics" },
]);
expect(fake.sent.filter((call) => call.method === "Input.dispatchMouseEvent")).toHaveLength(3);
});
it("enables overlay bypass before mouse events when overlay blocks the click point", async () => { it("enables overlay bypass before mouse events when overlay blocks the click point", async () => {
const order: string[] = []; const order: string[] = [];
const bypassOverlay = vi.fn(async (_tabId: number, enabled: boolean) => { const bypassOverlay = vi.fn(async (_tabId: number, enabled: boolean) => {
@@ -5,6 +5,7 @@ import { OVERLAY_HOST_MARKER_ATTR, OVERLAY_HOST_NAME } from "@/lib/overlay-bridg
import { SessionManager } from "@/session-manager/manager"; import { SessionManager } from "@/session-manager/manager";
import type { CdpRunner } from "@/tools/shared"; import type { CdpRunner } from "@/tools/shared";
import { import {
buildFrameVomScene,
buildVomScene, buildVomScene,
type CdpAxNode, type CdpAxNode,
handleGetHtml, handleGetHtml,
@@ -62,6 +63,9 @@ function makeFakeCdp(handlers: Record<string, (params?: object) => unknown>) {
const send = vi.fn(async (_tabId: number, method: string, params?: object) => { const send = vi.fn(async (_tabId: number, method: string, params?: object) => {
sent.push({ method, params }); sent.push({ method, params });
const handler = handlers[method]; const handler = handlers[method];
if (!handler && method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
}
if (!handler) throw new Error(`unexpected CDP call ${method}`); if (!handler) throw new Error(`unexpected CDP call ${method}`);
return handler(params); return handler(params);
}); });
@@ -508,6 +512,292 @@ describe("handleScreenshot overlay suppression", () => {
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
describe("buildVomScene", () => { describe("buildVomScene", () => {
it("merges multiple sibling and nested frames without mixing their targets", () => {
const node = (
backendNodeId: number,
parentBackendNodeId: number | null,
frameId: string,
tag: string,
attrs: Record<string, string> = {},
): CapturedNode => ({
backendNodeId,
parentBackendNodeId,
frameId,
tag,
attrs,
rect: { x: 0, y: 0, w: 100, h: 30 },
localRect: { x: 0, y: 0, w: 100, h: 30 },
paintOrder: 1,
position: "static",
pointerEvents: "auto",
});
const mainNodes = [
node(1, null, "main", "body"),
node(10, 1, "main", "iframe", { title: "First" }),
node(20, 1, "main", "iframe", { title: "Second" }),
];
const firstNodes = [
node(100, null, "first", "body"),
node(101, 100, "first", "button", { "aria-label": "First action" }),
];
const secondNodes = [
node(200, null, "second", "body"),
node(201, 200, "second", "h2"),
node(210, 200, "second", "iframe", { title: "Nested" }),
];
const nestedNodes = [
node(300, null, "nested", "body"),
node(301, 300, "nested", "input", { placeholder: "Nested field" }),
];
const captured: CapturedViewModel = {
viewport: { width: 1200, height: 800 },
nodes: mainNodes,
iframeNodes: new Map([
[10, firstNodes],
[20, secondNodes],
[210, nestedNodes],
]),
frameNodes: new Map([
["main", mainNodes],
["first", firstNodes],
["second", secondNodes],
["nested", nestedNodes],
]),
frameOwnerBackendNodeIds: new Map([
["first", 10],
["second", 20],
["nested", 210],
]),
rootFrameId: "main",
excludedBackendNodeIds: new Set(),
};
const axNode = (
nodeId: string,
backendDOMNodeId: number,
role: string,
parentId?: string,
name?: string,
): CdpAxNode => ({
nodeId,
backendDOMNodeId,
role: { type: "role", value: role },
...(parentId ? { parentId } : {}),
...(name ? { name: { type: "computedString", value: name } } : {}),
});
const result = buildFrameVomScene(
[
{
frameId: "main",
contextScopeId: "main",
target: { tabId: 7 },
domNodes: mainNodes,
axNodes: [
axNode("root", 1, "RootWebArea"),
axNode("frame-1", 10, "Iframe", "root", "First"),
axNode("frame-2", 20, "Iframe", "root", "Second"),
],
},
{
frameId: "first",
contextScopeId: "first",
parentFrameId: "main",
ownerBackendNodeId: 10,
target: { tabId: 7, sessionId: "session-first" },
domNodes: firstNodes,
axNodes: [
axNode("root", 100, "RootWebArea"),
axNode("button", 101, "button", "root", "First action"),
],
},
{
frameId: "second",
contextScopeId: "second",
parentFrameId: "main",
ownerBackendNodeId: 20,
target: { tabId: 7, sessionId: "session-second" },
domNodes: secondNodes,
axNodes: [
axNode("root", 200, "RootWebArea"),
axNode("heading", 201, "heading", "root", "Second heading"),
axNode("nested-frame", 210, "Iframe", "root", "Nested"),
],
},
{
frameId: "nested",
contextScopeId: "nested",
parentFrameId: "second",
ownerBackendNodeId: 210,
target: { tabId: 7, sessionId: "session-nested" },
domNodes: nestedNodes,
axNodes: [
axNode("root", 300, "RootWebArea"),
axNode("input", 301, "textbox", "root", "Nested field"),
],
},
],
captured,
);
const rendered = renderVom(result);
expect(rendered.text).toContain('Iframe "First"');
expect(rendered.text).toContain('@e1 button "First action"');
expect(rendered.text).toContain('heading "Second heading"');
expect(rendered.text).toContain('Iframe "Nested"');
expect(rendered.text).toContain('textbox "Nested field"');
expect(rendered.refs.find((ref) => ref.backendNodeId === 101)?.frameId).toBe("first");
expect(rendered.refs.find((ref) => ref.backendNodeId === 301)?.frameId).toBe("nested");
const firstAction = result.nodes.find((item) => item.backendNodeId === 101);
const nestedInput = result.nodes.find((item) => item.backendNodeId === 301);
expect(firstAction).toMatchObject({ frameId: "first", contextScopeId: "first" });
expect(nestedInput).toMatchObject({ frameId: "nested", contextScopeId: "nested" });
expect(nestedInput?.parentId).toBe(result.nodes.find((item) => item.backendNodeId === 210)?.id);
});
it("does not place a child document at the page root when its frame boundary is unresolved", () => {
const childNode: CapturedNode = {
backendNodeId: 101,
parentBackendNodeId: null,
frameId: "child",
tag: "button",
attrs: {},
rect: { x: 0, y: 0, w: 100, h: 30 },
paintOrder: 1,
position: "static",
pointerEvents: "auto",
};
const captured: CapturedViewModel = {
viewport: { width: 800, height: 600 },
nodes: [],
iframeNodes: new Map(),
frameNodes: new Map([["child", [childNode]]]),
rootFrameId: "main",
excludedBackendNodeIds: new Set(),
};
const result = buildFrameVomScene(
[
{
frameId: "main",
contextScopeId: "main",
target: { tabId: 7 },
domNodes: [],
axNodes: [],
},
{
frameId: "child",
contextScopeId: "child",
parentFrameId: "main",
target: { tabId: 7 },
domNodes: [childNode],
axNodes: [
{
nodeId: "button",
frameId: "child",
backendDOMNodeId: 101,
role: { type: "role", value: "button" },
name: { type: "computedString", value: "Child action" },
},
],
},
],
captured,
);
expect(result.nodes).toEqual([]);
});
it("keeps AX-only frame content attached to its iframe boundary", () => {
const mainNodes: CapturedNode[] = [
{
backendNodeId: 1,
parentBackendNodeId: null,
frameId: "main",
tag: "body",
attrs: {},
rect: { x: 0, y: 0, w: 800, h: 600 },
paintOrder: 0,
position: "static",
pointerEvents: "auto",
},
{
backendNodeId: 10,
parentBackendNodeId: 1,
frameId: "main",
tag: "iframe",
attrs: { title: "Remote" },
rect: { x: 100, y: 100, w: 400, h: 300 },
paintOrder: 1,
position: "static",
pointerEvents: "auto",
},
];
const captured: CapturedViewModel = {
viewport: { width: 800, height: 600 },
nodes: mainNodes,
iframeNodes: new Map(),
frameNodes: new Map([
["main", mainNodes],
["child", []],
]),
rootFrameId: "main",
excludedBackendNodeIds: new Set(),
};
const result = buildFrameVomScene(
[
{
frameId: "main",
contextScopeId: "main",
target: { tabId: 7 },
domNodes: mainNodes,
axNodes: [
{
nodeId: "main-root",
backendDOMNodeId: 1,
role: { type: "role", value: "RootWebArea" },
},
{
nodeId: "frame-owner",
parentId: "main-root",
backendDOMNodeId: 10,
role: { type: "role", value: "Iframe" },
name: { type: "computedString", value: "Remote" },
},
],
},
{
frameId: "child",
contextScopeId: "child",
parentFrameId: "main",
ownerBackendNodeId: 10,
target: { tabId: 7, sessionId: "child-session" },
domNodes: [],
axNodes: [
{
nodeId: "child-root",
backendDOMNodeId: 100,
role: { type: "role", value: "RootWebArea" },
},
{
nodeId: "child-button",
parentId: "child-root",
backendDOMNodeId: 101,
role: { type: "role", value: "button" },
name: { type: "computedString", value: "AX fallback action" },
},
],
},
],
captured,
);
const childButton = result.nodes.find((node) => node.backendNodeId === 101);
expect(childButton).toMatchObject({ frameId: "child", contextScopeId: "child" });
expect(childButton?.parentId).toBe(result.nodes.find((node) => node.backendNodeId === 10)?.id);
expect(renderVom(result).text).toContain('@e1 button "AX fallback action"');
});
it("joins AX semantics with captured geometry by backendDOMNodeId", () => { it("joins AX semantics with captured geometry by backendDOMNodeId", () => {
const axNodes: CdpAxNode[] = [ const axNodes: CdpAxNode[] = [
{ {
@@ -816,6 +1106,7 @@ describe("buildVomScene", () => {
parentId: "2", parentId: "2",
role: { type: "role", value: "textbox" }, role: { type: "role", value: "textbox" },
name: { type: "x", value: "输入密码" }, name: { type: "x", value: "输入密码" },
value: { value: "iframe-secret" },
backendDOMNodeId: 202, backendDOMNodeId: 202,
properties: [{ name: "inputType", value: { value: "password" } }], properties: [{ name: "inputType", value: { value: "password" } }],
}, },
@@ -870,6 +1161,10 @@ describe("buildVomScene", () => {
expect(passwordNodes).toHaveLength(1); expect(passwordNodes).toHaveLength(1);
expect(passwordNodes[0]).toEqual(expect.objectContaining({ id: 202, sensitive: true })); expect(passwordNodes[0]).toEqual(expect.objectContaining({ id: 202, sensitive: true }));
expect(passwordNodes[0].value).toBeUndefined();
const rendered = renderVom(buildVomScene(axNodes, captured)).text;
expect(rendered).toContain('textbox "输入密码" [filled] ="•••"');
expect(rendered).not.toContain("iframe-secret");
}); });
it("keeps unnamed iframe controls but skips unnamed iframe links", () => { it("keeps unnamed iframe controls but skips unnamed iframe links", () => {
@@ -3123,6 +3418,32 @@ describe("handleGetHtml", () => {
expect(deps.send).toHaveBeenCalledTimes(1); expect(deps.send).toHaveBeenCalledTimes(1);
}); });
it("reads a frame ref through its child CDP session", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
ctx.refStore.set("e7", 4242, {
tabId: 4,
frameId: "child-frame",
cdpSessionId: "child-session",
});
const deps = makeDeps({});
const sendToTarget = vi.fn(async (target, method, params) => {
expect(target).toEqual({ tabId: 4, sessionId: "child-session" });
expect(method).toBe("DOM.getOuterHTML");
expect(params).toEqual({ backendNodeId: 4242 });
return { outerHTML: "<button>frame</button>" };
});
(deps.cdp as CdpRunner).sendToTarget = sendToTarget as unknown as NonNullable<
CdpRunner["sendToTarget"]
>;
const res = await handleGetHtml(sm, { session_id: "aa11", ref: "@e7" }, deps);
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
expect(res.html).toBe("<button>frame</button>");
expect(deps.send).not.toHaveBeenCalled();
});
it("returns not_found when a ref belongs to another tab", async () => { it("returns not_found when a ref belongs to another tab", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11"); const ctx = await sm.start("aa11");
@@ -46,6 +46,25 @@ describe("lookupSnapshotRef", () => {
}); });
describe("resolveSnapshotRef", () => { describe("resolveSnapshotRef", () => {
it("preserves the frame and child session needed to route iframe refs", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
ctx.refStore.set("e3", 1234, {
tabId: 4,
frameId: "child-frame",
cdpSessionId: "child-session",
});
const expected = {
backendNodeId: 1234,
refKey: "e3",
frameId: "child-frame",
cdpSessionId: "child-session",
};
expect(lookupSnapshotRef(ctx, "@e3", 4)).toEqual(expected);
expect(resolveSnapshotRef(ctx, "@e3", 4)).toEqual(expected);
});
it("returns not_found for unknown ref", async () => { it("returns not_found for unknown ref", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11"); const ctx = await sm.start("aa11");
@@ -250,9 +250,10 @@ const DEFAULT_NOTIFICATION_COPY: BorrowNotificationCopy = {
// URL injection-eligibility test // URL injection-eligibility test
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
const CHROME_WEB_STORE_RES = [ const EXTENSION_STORE_RES = [
/^https:\/\/chrome\.google\.com\/webstore/i, /^https:\/\/chrome\.google\.com\/webstore/i,
/^https:\/\/chromewebstore\.google\.com/i, /^https:\/\/chromewebstore\.google\.com/i,
/^https:\/\/microsoftedge\.microsoft\.com\/addons/i,
]; ];
/** /**
@@ -264,14 +265,15 @@ const CHROME_WEB_STORE_RES = [
* - `ftp://` no longer hosts content scripts reliably in modern Chrome. * - `ftp://` no longer hosts content scripts reliably in modern Chrome.
* - `about:` / `chrome:` / `chrome-extension:` / `edge:` / `devtools:` / * - `about:` / `chrome:` / `chrome-extension:` / `edge:` / `devtools:` /
* `view-source:` / `data:` / `blob:` are blocked by the platform. * `view-source:` / `data:` / `blob:` are blocked by the platform.
* - The Chrome Web Store (both legacy and new domains) is also blocked. * - Extension storefronts are blocked by their own browser: the Chrome Web
* Store (legacy and new domains) under Chrome, and Edge Add-ons under Edge.
* *
* Mirrors the implicit scheme list documented in * Mirrors the implicit scheme list documented in
* https://developer.chrome.com/docs/extensions/develop/concepts/match-patterns. * https://developer.chrome.com/docs/extensions/develop/concepts/match-patterns.
*/ */
export function isInjectableContentScriptUrl(url: string | undefined): boolean { export function isInjectableContentScriptUrl(url: string | undefined): boolean {
if (!url) return false; if (!url) return false;
for (const re of CHROME_WEB_STORE_RES) { for (const re of EXTENSION_STORE_RES) {
if (re.test(url)) return false; if (re.test(url)) return false;
} }
return /^https?:\/\//i.test(url); return /^https?:\/\//i.test(url);
+41 -96
View File
@@ -1,9 +1,15 @@
// Shared CDP helpers for scrolling nodes into view and computing // Resolve and scroll nodes inside one CDP target. Cross-frame projection
// viewport-space bounding rectangles. Used by interaction tools (click // belongs to frame-geometry.ts.
// centre) and observation tools (element screenshot clip).
import type { RpcError } from "@/transport/types"; import type { RpcError } from "@/transport/types";
import { rpcError } from "./errors"; import { rpcError } from "./errors";
import {
parseCdpQuad,
polygonBounds,
type Region,
rectPolygon,
type ViewportRect,
} from "./geometry";
import { type CdpRunner, isRpcError } from "./shared"; import { type CdpRunner, isRpcError } from "./shared";
const ELEMENT_NOT_VISIBLE_MESSAGE = const ELEMENT_NOT_VISIBLE_MESSAGE =
@@ -13,45 +19,6 @@ function elementNotVisibleError(): RpcError {
return rpcError("permission_denied", "element_not_visible", ELEMENT_NOT_VISIBLE_MESSAGE); return rpcError("permission_denied", "element_not_visible", ELEMENT_NOT_VISIBLE_MESSAGE);
} }
/** Viewport-space axis-aligned bounding box for CDP `Page.captureScreenshot` clip. */
export interface ViewportRect {
x: number;
y: number;
width: number;
height: number;
}
/**
* Compute the axis-aligned bounding box of an 8-double CDP content
* quad / box-model polygon. Exported for unit tests.
*/
export function quadBoundingRect(quad: number[]): ViewportRect | null {
if (quad.length !== 8) return null;
const xs = [quad[0], quad[2], quad[4], quad[6]];
const ys = [quad[1], quad[3], quad[5], quad[7]];
const minX = Math.min(...xs);
const maxX = Math.max(...xs);
const minY = Math.min(...ys);
const maxY = Math.max(...ys);
const width = maxX - minX;
const height = maxY - minY;
if (width <= 0 || height <= 0) return null;
return { x: minX, y: minY, width, height };
}
function rectToQuad(rect: ViewportRect): number[] {
return [
rect.x,
rect.y,
rect.x + rect.width,
rect.y,
rect.x + rect.width,
rect.y + rect.height,
rect.x,
rect.y + rect.height,
];
}
/** /**
* Resolve `backendNodeId` → CDP `objectId` so we can invoke * Resolve `backendNodeId` → CDP `objectId` so we can invoke
* `Runtime.callFunctionOn` against the live JS object. * `Runtime.callFunctionOn` against the live JS object.
@@ -112,48 +79,24 @@ export async function scrollNodeIntoView(
} }
} }
/** function validCdpRegion(quads: number[][] | undefined): Region {
* Viewport-space bounding box for `backendNodeId`. Prefers return (quads ?? []).flatMap((raw) => {
* `DOM.getContentQuads`, then `DOM.getBoxModel`, then the union of const polygon = parseCdpQuad(raw);
* visible descendant client rects for zero-size overflow containers. return polygon && polygonBounds(polygon) ? [polygon] : [];
*/ });
export async function nodeBoundingRect( }
async function resolveVisibleContentRegion(
cdp: CdpRunner, cdp: CdpRunner,
tabId: number, tabId: number,
backendNodeId: number, backendNodeId: number,
): Promise<ViewportRect | RpcError> { ): Promise<Region | RpcError> {
const polygon = await visibleContentPolygon(cdp, tabId, backendNodeId);
if (isRpcError(polygon)) return polygon;
// visibleContentPolygon only returns quads with positive area.
return quadBoundingRect(polygon)!;
}
/**
* Compute the centroid of a CDP content quad. Quads are reported as
* 8 doubles in clockwise order: (x1, y1, x2, y2, x3, y3, x4, y4).
*/
export function quadCentre(quad: number[]): { x: number; y: number } {
const x = (quad[0] + quad[4]) / 2;
const y = (quad[1] + quad[5]) / 2;
return { x, y };
}
/** Centre of a `DOM.getBoxModel` `content` polygon (also 8 doubles). */
export function boxCentre(box: number[]): { x: number; y: number } {
return quadCentre(box);
}
async function visibleContentPolygon(
cdp: CdpRunner,
tabId: number,
backendNodeId: number,
): Promise<number[] | RpcError> {
try { try {
const quads = await cdp.send<{ quads?: number[][] }>(tabId, "DOM.getContentQuads", { const quads = await cdp.send<{ quads?: number[][] }>(tabId, "DOM.getContentQuads", {
backendNodeId, backendNodeId,
}); });
const quad = quads.quads?.find((q) => q.length === 8 && quadBoundingRect(q) !== null); const visible = validCdpRegion(quads.quads);
if (quad) return quad; if (visible.length > 0) return visible;
} catch (err) { } catch (err) {
console.debug("[bsk element-geometry] getContentQuads failed", err); console.debug("[bsk element-geometry] getContentQuads failed", err);
} }
@@ -162,15 +105,34 @@ async function visibleContentPolygon(
backendNodeId, backendNodeId,
}); });
const content = box.model?.content; const content = box.model?.content;
if (content && content.length === 8 && quadBoundingRect(content)) return content; const fallback = validCdpRegion(content ? [content] : undefined);
if (fallback.length > 0) return fallback;
} catch (err) { } catch (err) {
console.debug("[bsk element-geometry] getBoxModel failed", err); console.debug("[bsk element-geometry] getBoxModel failed", err);
} }
const descendantRect = await descendantBoundingRect(cdp, tabId, backendNodeId); const descendantRect = await descendantBoundingRect(cdp, tabId, backendNodeId);
if (descendantRect) return rectToQuad(descendantRect); if (descendantRect) {
return [
rectPolygon({
x: descendantRect.x,
y: descendantRect.y,
w: descendantRect.width,
h: descendantRect.height,
}),
];
}
return elementNotVisibleError(); return elementNotVisibleError();
} }
/** Resolve every visible content region in the node's target-local viewport. */
export async function nodeContentRegion(
cdp: CdpRunner,
tabId: number,
backendNodeId: number,
): Promise<Region | RpcError> {
return resolveVisibleContentRegion(cdp, tabId, backendNodeId);
}
async function descendantBoundingRect( async function descendantBoundingRect(
cdp: CdpRunner, cdp: CdpRunner,
tabId: number, tabId: number,
@@ -241,20 +203,3 @@ function parseViewportRect(value: unknown): ViewportRect | null {
if (![x, y, width, height].every(Number.isFinite) || width <= 0 || height <= 0) return null; if (![x, y, width, height].every(Number.isFinite) || width <= 0 || height <= 0) return null;
return { x, y, width, height }; return { x, y, width, height };
} }
/**
* Get the click point for `backendNodeId`. Prefers
* `DOM.getContentQuads` (richer for rotated / transformed elements);
* falls back to `DOM.getBoxModel`, then visible descendant bounds.
* Returns `permission_denied` with "element not visible" when every
* geometry path comes back empty / fails.
*/
export async function nodeCentre(
cdp: CdpRunner,
tabId: number,
backendNodeId: number,
): Promise<{ x: number; y: number } | RpcError> {
const polygon = await visibleContentPolygon(cdp, tabId, backendNodeId);
if (isRpcError(polygon)) return polygon;
return quadCentre(polygon);
}
+290
View File
@@ -0,0 +1,290 @@
import { type CdpFrame, type CdpFrameGraph, type CdpTarget } from "@/browser-driver/frame-graph";
import type { RpcError } from "@/transport/types";
import { nodeContentRegion, scrollNodeIntoView } from "./element-geometry";
import {
clipPolygon,
type GeometryProjection,
type Point,
type Polygon,
type ProjectiveEdge,
parseCdpQuad,
polygonArea,
polygonCentroid,
projectRegionToViewport,
type Quad,
type Region,
rectPolygon,
regionBounds,
type Size,
type ViewportRect,
} from "./geometry";
import { type CdpRunner, cdpRunnerForTarget, isRpcError, sendToCdpTarget } from "./shared";
export interface NodeAddress {
target: CdpTarget;
backendNodeId: number;
frameId?: string;
}
export interface ResolvedNodeGeometry {
topVisibleRegions: Region;
topBounds: ViewportRect;
actionPoint: Point;
}
function geometryError(message: string): RpcError {
return { code: "cdp_failed", message };
}
function frameMap(graph: CdpFrameGraph): Map<string, CdpFrame> {
return new Map(graph.frames.map((frame) => [frame.frameId, frame]));
}
function targetRootFrame(byId: Map<string, CdpFrame>, frame: CdpFrame): CdpFrame | null {
const seen = new Set<string>();
let current = frame;
while (current.parentFrameId) {
if (seen.has(current.frameId)) return null;
seen.add(current.frameId);
const parent = byId.get(current.parentFrameId);
if (!parent || parent.target.sessionId !== current.target.sessionId) break;
current = parent;
}
return current;
}
function frameAncestry(graph: CdpFrameGraph, frameId: string): CdpFrame[] | null {
const byId = frameMap(graph);
const path: CdpFrame[] = [];
const seen = new Set<string>();
let current = byId.get(frameId);
if (!current) return null;
while (current.parentFrameId) {
if (seen.has(current.frameId)) return null;
seen.add(current.frameId);
path.push(current);
const parent = byId.get(current.parentFrameId);
if (!parent) return null;
current = parent;
}
return path;
}
async function targetViewport(cdp: CdpRunner, target: CdpTarget): Promise<Size | null> {
const metrics = await sendToCdpTarget<{
cssLayoutViewport?: { clientWidth?: number; clientHeight?: number };
layoutViewport?: { clientWidth?: number; clientHeight?: number };
}>(cdp, target, "Page.getLayoutMetrics", {});
const viewport = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
const width = viewport.clientWidth ?? 0;
const height = viewport.clientHeight ?? 0;
return width > 0 && height > 0 ? { width, height } : null;
}
async function ownerContentQuad(
cdp: CdpRunner,
parent: CdpFrame,
ownerBackendNodeId: number,
): Promise<Quad | null> {
const result = await sendToCdpTarget<{ model?: { content?: number[] } }>(
cdp,
parent.target,
"DOM.getBoxModel",
{ backendNodeId: ownerBackendNodeId },
);
return parseCdpQuad(result.model?.content);
}
async function sameTargetFrameClips(
cdp: CdpRunner,
byId: Map<string, CdpFrame>,
frame: CdpFrame,
targetRoot: CdpFrame,
): Promise<Polygon[] | null> {
const clips: Polygon[] = [];
let current = frame;
const seen = new Set<string>();
while (current.frameId !== targetRoot.frameId) {
if (seen.has(current.frameId) || !current.parentFrameId) return null;
seen.add(current.frameId);
const parent = byId.get(current.parentFrameId);
if (!parent || current.ownerBackendNodeId === undefined) return null;
const clip = await ownerContentQuad(cdp, parent, current.ownerBackendNodeId);
if (!clip) return null;
clips.push(clip);
current = parent;
}
return clips;
}
export async function resolveFrameProjection(
cdp: CdpRunner,
graph: CdpFrameGraph,
frameId: string,
): Promise<GeometryProjection | null> {
const byId = frameMap(graph);
const frame = byId.get(frameId);
if (!frame) return null;
let targetRoot = targetRootFrame(byId, frame);
if (!targetRoot) return null;
const sourceViewport = await targetViewport(cdp, targetRoot.target);
if (!sourceViewport) return null;
const sourceClips = await sameTargetFrameClips(cdp, byId, frame, targetRoot);
if (!sourceClips) return null;
const edges: ProjectiveEdge[] = [];
while (targetRoot.parentFrameId) {
const parent = byId.get(targetRoot.parentFrameId);
if (!parent || targetRoot.ownerBackendNodeId === undefined) return null;
const destinationQuad = await ownerContentQuad(cdp, parent, targetRoot.ownerBackendNodeId);
if (!destinationQuad) return null;
const source =
edges.length === 0 ? sourceViewport : await targetViewport(cdp, targetRoot.target);
if (!source) return null;
const parentTargetRoot = targetRootFrame(byId, parent);
if (!parentTargetRoot) return null;
const destinationClips = await sameTargetFrameClips(cdp, byId, parent, parentTargetRoot);
if (!destinationClips) return null;
edges.push({ sourceViewport: source, destinationQuad, destinationClips });
targetRoot = parentTargetRoot;
}
const topViewport =
edges.length === 0 ? sourceViewport : await targetViewport(cdp, targetRoot.target);
return topViewport ? { sourceClips, edges, topViewport } : null;
}
async function loadFrameGraph(cdp: CdpRunner, tabId: number): Promise<CdpFrameGraph | null> {
if (!cdp.getFrameGraph) return null;
try {
return await cdp.getFrameGraph(tabId);
} catch (error) {
console.debug("[bsk frame-geometry] frame graph resolution failed", error);
return null;
}
}
async function scrollFrameOwners(
cdp: CdpRunner,
tabId: number,
graph: CdpFrameGraph,
frameId: string,
): Promise<RpcError | null> {
const byId = frameMap(graph);
const ancestry = frameAncestry(graph, frameId);
if (!ancestry) return geometryError(`could not resolve frame ancestry for ${frameId}`);
for (const child of [...ancestry].reverse()) {
const parent = child.parentFrameId ? byId.get(child.parentFrameId) : undefined;
if (!parent || child.ownerBackendNodeId === undefined) {
return geometryError(`could not resolve frame owner for ${child.frameId}`);
}
const error = await scrollNodeIntoView(
cdpRunnerForTarget(cdp, parent.target),
tabId,
child.ownerBackendNodeId,
);
if (error) return error;
}
return null;
}
async function scrollElementWithFrameGraph(
cdp: CdpRunner,
tabId: number,
target: CdpTarget,
backendNodeId: number,
frameId: string | undefined,
graph: CdpFrameGraph | null,
): Promise<RpcError | null> {
if (frameId) {
if (!graph) return geometryError(`could not resolve frame graph for ${frameId}`);
const error = await scrollFrameOwners(cdp, tabId, graph, frameId);
if (error) return error;
} else if (target.sessionId) {
return geometryError("an OOPIF node address requires frameId");
}
return scrollNodeIntoView(cdpRunnerForTarget(cdp, target), tabId, backendNodeId);
}
export async function scrollElementAndFramesIntoView(
cdp: CdpRunner,
tabId: number,
target: CdpTarget,
backendNodeId: number,
frameId?: string,
): Promise<RpcError | null> {
const graph = frameId ? await loadFrameGraph(cdp, tabId) : null;
return scrollElementWithFrameGraph(cdp, tabId, target, backendNodeId, frameId, graph);
}
function largestRegion(regions: Region): Polygon | null {
let largest: { polygon: Polygon; area: number } | null = null;
for (const polygon of regions) {
const area = polygonArea(polygon);
if (area <= 0) continue;
if (!largest || area > largest.area) largest = { polygon, area };
}
return largest?.polygon ?? null;
}
export async function resolveNodeGeometry(
cdp: CdpRunner,
tabId: number,
address: NodeAddress,
options: { scrollIntoView?: boolean } = {},
): Promise<ResolvedNodeGeometry | RpcError> {
try {
if (address.target.sessionId && !address.frameId) {
return geometryError("an OOPIF node address requires frameId");
}
const graph = address.frameId ? await loadFrameGraph(cdp, tabId) : null;
if (address.frameId && !graph) {
return geometryError(`could not resolve frame graph for ${address.frameId}`);
}
if (options.scrollIntoView) {
const scrollError = await scrollElementWithFrameGraph(
cdp,
tabId,
address.target,
address.backendNodeId,
address.frameId,
graph,
);
if (scrollError) return scrollError;
}
const localRegion = await nodeContentRegion(
cdpRunnerForTarget(cdp, address.target),
tabId,
address.backendNodeId,
);
if (isRpcError(localRegion)) return localRegion;
let topVisibleRegions: Region;
if (address.frameId && graph) {
const projection = await resolveFrameProjection(cdp, graph, address.frameId);
if (!projection)
return geometryError(`could not resolve frame geometry for ${address.frameId}`);
topVisibleRegions = projectRegionToViewport(localRegion, projection);
} else {
const viewport = await targetViewport(cdp, address.target);
if (!viewport) return geometryError("could not resolve top viewport geometry");
topVisibleRegions = localRegion
.map((polygon) =>
clipPolygon(polygon, rectPolygon({ x: 0, y: 0, w: viewport.width, h: viewport.height })),
)
.filter((polygon) => polygon.length >= 3);
}
const topBounds = regionBounds(topVisibleRegions);
const actionRegion = largestRegion(topVisibleRegions);
const actionPoint = actionRegion ? polygonCentroid(actionRegion) : null;
if (!topBounds || !actionPoint) {
return { code: "permission_denied", message: "element not visible" };
}
return { topVisibleRegions, topBounds, actionPoint };
} catch (error) {
return geometryError(error instanceof Error ? error.message : String(error));
}
}
+266
View File
@@ -0,0 +1,266 @@
export interface Point {
x: number;
y: number;
}
export type Polygon = Point[];
export type Region = Polygon[];
export type Quad = [Point, Point, Point, Point];
export interface Size {
width: number;
height: number;
}
export interface ViewportRect {
x: number;
y: number;
width: number;
height: number;
}
export interface ProjectiveEdge {
sourceViewport: Size;
destinationQuad: Quad;
destinationClips?: Polygon[];
}
export interface GeometryProjection {
sourceClips: Polygon[];
edges: ProjectiveEdge[];
topViewport: Size;
}
export function parseCdpQuad(raw: number[] | undefined): Quad | null {
if (!raw || raw.length !== 8 || !raw.every(Number.isFinite)) return null;
return [
{ x: raw[0], y: raw[1] },
{ x: raw[2], y: raw[3] },
{ x: raw[4], y: raw[5] },
{ x: raw[6], y: raw[7] },
];
}
export function rectPolygon(rect: { x: number; y: number; w: number; h: number }): Quad {
return [
{ x: rect.x, y: rect.y },
{ x: rect.x + rect.w, y: rect.y },
{ x: rect.x + rect.w, y: rect.y + rect.h },
{ x: rect.x, y: rect.y + rect.h },
];
}
export function viewportPolygon(viewport: Size): Quad {
return rectPolygon({ x: 0, y: 0, w: viewport.width, h: viewport.height });
}
/** Project a normalized unit-square point into an arbitrary content quad. */
export function projectUnitPoint(point: Point, quad: Quad): Point {
const [p0, p1, p2, p3] = quad;
const dx1 = p1.x - p2.x;
const dx2 = p3.x - p2.x;
const dx3 = p0.x - p1.x + p2.x - p3.x;
const dy1 = p1.y - p2.y;
const dy2 = p3.y - p2.y;
const dy3 = p0.y - p1.y + p2.y - p3.y;
let a: number;
let b: number;
let c: number;
let d: number;
let e: number;
let f: number;
let g = 0;
let h = 0;
if (Math.abs(dx3) < 1e-9 && Math.abs(dy3) < 1e-9) {
a = p1.x - p0.x;
b = p3.x - p0.x;
c = p0.x;
d = p1.y - p0.y;
e = p3.y - p0.y;
f = p0.y;
} else {
const denominator = dx1 * dy2 - dx2 * dy1;
if (Math.abs(denominator) < 1e-9) {
return {
x: p0.x + (p1.x - p0.x) * point.x + (p3.x - p0.x) * point.y,
y: p0.y + (p1.y - p0.y) * point.x + (p3.y - p0.y) * point.y,
};
}
g = (dx3 * dy2 - dx2 * dy3) / denominator;
h = (dx1 * dy3 - dx3 * dy1) / denominator;
a = p1.x - p0.x + g * p1.x;
b = p3.x - p0.x + h * p3.x;
c = p0.x;
d = p1.y - p0.y + g * p1.y;
e = p3.y - p0.y + h * p3.y;
f = p0.y;
}
const scale = g * point.x + h * point.y + 1;
return {
x: (a * point.x + b * point.y + c) / scale,
y: (d * point.x + e * point.y + f) / scale,
};
}
export function projectPolygon(polygon: Polygon, edge: ProjectiveEdge): Polygon {
const { width, height } = edge.sourceViewport;
if (width <= 0 || height <= 0) return [];
return polygon.map((point) =>
projectUnitPoint({ x: point.x / width, y: point.y / height }, edge.destinationQuad),
);
}
function polygonSignedArea(points: Polygon): number {
let area = 0;
for (let index = 0; index < points.length; index += 1) {
const current = points[index];
const next = points[(index + 1) % points.length];
area += current.x * next.y - next.x * current.y;
}
return area / 2;
}
function edgeIntersection(start: Point, end: Point, clipStart: Point, clipEnd: Point): Point {
const subjectX = end.x - start.x;
const subjectY = end.y - start.y;
const clipX = clipEnd.x - clipStart.x;
const clipY = clipEnd.y - clipStart.y;
const denominator = subjectX * clipY - subjectY * clipX;
if (Math.abs(denominator) < 1e-9) return end;
const offsetX = clipStart.x - start.x;
const offsetY = clipStart.y - start.y;
const scale = (offsetX * clipY - offsetY * clipX) / denominator;
return { x: start.x + scale * subjectX, y: start.y + scale * subjectY };
}
/** Clip a polygon to a convex polygon, preserving every resulting vertex. */
export function clipPolygon(subject: Polygon, clip: Polygon): Polygon {
if (subject.length < 3 || clip.length < 3) return [];
const orientation = polygonSignedArea(clip) >= 0 ? 1 : -1;
let output = [...subject];
for (let edge = 0; edge < clip.length && output.length > 0; edge += 1) {
const clipStart = clip[edge];
const clipEnd = clip[(edge + 1) % clip.length];
const input = output;
output = [];
const inside = (point: Point) =>
orientation *
((clipEnd.x - clipStart.x) * (point.y - clipStart.y) -
(clipEnd.y - clipStart.y) * (point.x - clipStart.x)) >=
-1e-9;
let start = input[input.length - 1];
for (const end of input) {
const startInside = inside(start);
const endInside = inside(end);
if (endInside) {
if (!startInside) output.push(edgeIntersection(start, end, clipStart, clipEnd));
output.push(end);
} else if (startInside) {
output.push(edgeIntersection(start, end, clipStart, clipEnd));
}
start = end;
}
}
return output;
}
export function polygonBounds(points: Polygon): ViewportRect | null {
if (points.length < 3) return null;
const xs = points.map((point) => point.x);
const ys = points.map((point) => point.y);
const x = Math.min(...xs);
const y = Math.min(...ys);
const right = Math.max(...xs);
const bottom = Math.max(...ys);
if (right <= x || bottom <= y) return null;
return { x, y, width: right - x, height: bottom - y };
}
export function regionBounds(region: Region): ViewportRect | null {
return polygonBounds(region.flat());
}
export function polygonArea(points: Polygon): number {
return Math.abs(polygonSignedArea(points));
}
export function polygonCentroid(points: Polygon): Point | null {
if (points.length < 3) return null;
let twiceArea = 0;
let x = 0;
let y = 0;
for (let index = 0; index < points.length; index += 1) {
const current = points[index];
const next = points[(index + 1) % points.length];
const cross = current.x * next.y - next.x * current.y;
twiceArea += cross;
x += (current.x + next.x) * cross;
y += (current.y + next.y) * cross;
}
if (Math.abs(twiceArea) < 1e-9) return null;
return { x: x / (3 * twiceArea), y: y / (3 * twiceArea) };
}
export function projectAndClipRegion(
region: Region,
edges: ProjectiveEdge[],
topViewport: Size,
): Region {
let projected = region;
for (const edge of edges) {
const sourceClip = viewportPolygon(edge.sourceViewport);
projected = projected
.map((polygon) => clipPolygon(polygon, sourceClip))
.filter((polygon) => polygon.length >= 3)
.map((polygon) =>
[edge.destinationQuad, ...(edge.destinationClips ?? [])].reduce(
(current, clip) => clipPolygon(current, clip),
projectPolygon(polygon, edge),
),
)
.filter((polygon) => polygon.length >= 3);
}
const topClip = viewportPolygon(topViewport);
return projected
.map((polygon) => clipPolygon(polygon, topClip))
.filter((polygon) => polygon.length >= 3);
}
export function projectRegionToViewport(region: Region, projection: GeometryProjection): Region {
const clipped = region
.map((polygon) =>
projection.sourceClips.reduce((current, clip) => clipPolygon(current, clip), polygon),
)
.filter((polygon) => polygon.length >= 3);
return projectAndClipRegion(clipped, projection.edges, projection.topViewport);
}
export function projectRectToViewport(
rect: { x: number; y: number; w: number; h: number } | null,
projection: GeometryProjection,
): ViewportRect | null {
if (!rect) return null;
return regionBounds(projectRegionToViewport([rectPolygon(rect)], projection));
}
export function childFrameProjection(
parent: GeometryProjection,
ownerRectInParent: { x: number; y: number; w: number; h: number },
): GeometryProjection {
const destinationQuad = rectPolygon(ownerRectInParent);
return {
sourceClips: [],
edges: [
{
sourceViewport: { width: ownerRectInParent.w, height: ownerRectInParent.h },
destinationQuad,
destinationClips: parent.sourceClips,
},
...parent.edges,
],
topViewport: parent.topViewport,
};
}
+62 -68
View File
@@ -8,6 +8,7 @@ import {
HELP_REQUEST, HELP_REQUEST,
type HelpCancelMessage, type HelpCancelMessage,
type HelpFinishMessage, type HelpFinishMessage,
type HelpHighlightRect,
type HelpQueryResponse, type HelpQueryResponse,
type HelpRequestMessage, type HelpRequestMessage,
isHelpAckMessage, isHelpAckMessage,
@@ -25,6 +26,7 @@ import type {
ResolvedTarget, ResolvedTarget,
RpcError, RpcError,
} from "@/transport/types"; } from "@/transport/types";
import { resolveNodeGeometry } from "./frame-geometry";
import { import {
type CdpRunner, type CdpRunner,
type ChromeTabsApi, type ChromeTabsApi,
@@ -37,7 +39,6 @@ import {
import { lookupSnapshotRef } from "./snapshot-ref"; import { lookupSnapshotRef } from "./snapshot-ref";
const DEFAULT_HELP_TIMEOUT_MS = 300_000; const DEFAULT_HELP_TIMEOUT_MS = 300_000;
const HELP_ATTR = "data-bsk-help";
const HELP_SEND_RETRIES = 3; const HELP_SEND_RETRIES = 3;
const HELP_SEND_RETRY_DELAY_MS = 350; const HELP_SEND_RETRY_DELAY_MS = 350;
const HELP_REARM_DEBOUNCE_MS = 150; const HELP_REARM_DEBOUNCE_MS = 150;
@@ -74,6 +75,7 @@ interface ActiveHelpRequest {
title?: string; title?: string;
targets: HelpTarget[]; targets: HelpTarget[];
selectors: string[]; selectors: string[];
rects: HelpHighlightRect[];
timeoutMs: number; timeoutMs: number;
notificationId: string; notificationId: string;
resolvedTargets?: ResolvedTarget[]; resolvedTargets?: ResolvedTarget[];
@@ -134,6 +136,7 @@ function helpRequestMessage(help: ActiveHelpRequest, tabId: number): HelpRequest
...(help.title ? { title: help.title } : {}), ...(help.title ? { title: help.title } : {}),
displayMode: isPrimaryTab ? "full" : "compact", displayMode: isPrimaryTab ? "full" : "compact",
selectors: isPrimaryTab ? help.selectors : [], selectors: isPrimaryTab ? help.selectors : [],
rects: isPrimaryTab ? help.rects : [],
timeoutMs: help.timeoutMs, timeoutMs: help.timeoutMs,
}; };
} }
@@ -165,33 +168,6 @@ async function sendHelpRequestWithAck(
throw lastError ?? new Error("failed to show help overlay"); throw lastError ?? new Error("failed to show help overlay");
} }
async function tagRefTarget(
cdp: CdpRunner,
tabId: number,
backendNodeId: number,
index: number,
): Promise<string | null> {
let objectId: string | undefined;
try {
const resolved = await cdp.send<{ object?: { objectId?: string } }>(tabId, "DOM.resolveNode", {
backendNodeId,
});
objectId = resolved.object?.objectId;
if (!objectId) return null;
await cdp.send(tabId, "Runtime.callFunctionOn", {
objectId,
functionDeclaration: `function(){ this.setAttribute("${HELP_ATTR}", "${index}"); }`,
});
return `[${HELP_ATTR}="${index}"]`;
} catch {
return null;
} finally {
if (objectId) {
await cdp.send(tabId, "Runtime.releaseObject", { objectId }).catch(() => {});
}
}
}
async function selectorExists( async function selectorExists(
cdp: CdpRunner, cdp: CdpRunner,
tabId: number, tabId: number,
@@ -209,36 +185,22 @@ async function selectorExists(
} }
} }
async function clearRefTags(cdp: CdpRunner | undefined, tabId: number): Promise<void> {
if (!cdp) return;
try {
const doc = await cdp.send<{ root?: { nodeId?: number } }>(tabId, "DOM.getDocument", {
depth: 0,
});
const rootId = doc.root?.nodeId;
if (rootId === undefined) return;
const { nodeIds } = await cdp.send<{ nodeIds: number[] }>(tabId, "DOM.querySelectorAll", {
nodeId: rootId,
selector: `[${HELP_ATTR}]`,
});
for (const nodeId of nodeIds ?? []) {
await cdp.send(tabId, "DOM.removeAttribute", { nodeId, name: HELP_ATTR }).catch(() => {});
}
} catch {
// Best-effort cleanup.
}
}
async function resolveHelpTargets( async function resolveHelpTargets(
ctx: ReturnType<SessionManager["get"]>, ctx: ReturnType<SessionManager["get"]>,
tabId: number, tabId: number,
targets: HelpTarget[], targets: HelpTarget[],
deps: RequestHelpDeps, deps: RequestHelpDeps,
): Promise<{ selectors: string[]; resolvedTargets?: ResolvedTarget[] }> { options: { scrollIntoView: boolean },
): Promise<{
selectors: string[];
rects: HelpHighlightRect[];
resolvedTargets?: ResolvedTarget[];
}> {
const selectors: string[] = []; const selectors: string[] = [];
const rects: HelpHighlightRect[] = [];
const resolved: ResolvedTarget[] = []; const resolved: ResolvedTarget[] = [];
let rootNodeId: number | undefined; let rootNodeId: number | undefined;
if (deps.cdp && targets.length > 0) { if (deps.cdp && targets.some((target) => Boolean(target.selector))) {
try { try {
const doc = await deps.cdp.send<{ root?: { nodeId?: number } }>(tabId, "DOM.getDocument", { const doc = await deps.cdp.send<{ root?: { nodeId?: number } }>(tabId, "DOM.getDocument", {
depth: 0, depth: 0,
@@ -262,28 +224,55 @@ async function resolveHelpTargets(
resolved.push({ matched, selector: tgt.selector }); resolved.push({ matched, selector: tgt.selector });
} else if (tgt.ref) { } else if (tgt.ref) {
const looked = ctx ? lookupSnapshotRef(ctx, tgt.ref, tabId) : null; const looked = ctx ? lookupSnapshotRef(ctx, tgt.ref, tabId) : null;
const backendNodeId = looked?.backendNodeId ?? null; let rect: HelpHighlightRect | null = null;
let sel: string | null = null; if (looked && deps.cdp) {
if (backendNodeId !== null && deps.cdp) { const target = {
sel = await tagRefTarget(deps.cdp, tabId, backendNodeId, i); tabId,
...(looked.cdpSessionId ? { sessionId: looked.cdpSessionId } : {}),
};
const geometry = await resolveNodeGeometry(
deps.cdp,
tabId,
{
target,
backendNodeId: looked.backendNodeId,
...(looked.frameId ? { frameId: looked.frameId } : {}),
},
{ scrollIntoView: options.scrollIntoView },
);
if (!isRpcError(geometry)) {
rect = {
top: geometry.topBounds.y,
left: geometry.topBounds.x,
width: geometry.topBounds.width,
height: geometry.topBounds.height,
};
}
} }
if (sel) selectors.push(sel); if (rect) rects.push(rect);
resolved.push({ matched: sel !== null, ref: tgt.ref }); resolved.push({
matched: rect !== null,
ref: tgt.ref,
});
} }
} }
return { selectors, resolvedTargets: resolved.length > 0 ? resolved : undefined }; return { selectors, rects, resolvedTargets: resolved.length > 0 ? resolved : undefined };
} }
async function refreshHelpTargets(help: ActiveHelpRequest): Promise<void> { async function refreshHelpTargets(
await clearRefTags(help.deps.cdp, help.primaryTabId); help: ActiveHelpRequest,
const { selectors, resolvedTargets } = await resolveHelpTargets( options: { scrollIntoView: boolean } = { scrollIntoView: true },
): Promise<void> {
const { selectors, rects, resolvedTargets } = await resolveHelpTargets(
help.ctx, help.ctx,
help.primaryTabId, help.primaryTabId,
help.targets, help.targets,
help.deps, help.deps,
options,
); );
help.selectors = selectors; help.selectors = selectors;
help.rects = rects;
help.resolvedTargets = resolvedTargets; help.resolvedTargets = resolvedTargets;
} }
@@ -294,12 +283,7 @@ async function cleanupHelp(help: ActiveHelpRequest): Promise<void> {
const tabsToCancel = new Set([help.primaryTabId, ...help.overlayTabIds]); const tabsToCancel = new Set([help.primaryTabId, ...help.overlayTabIds]);
await Promise.all( await Promise.all(
[...tabsToCancel].map((tabId) => [...tabsToCancel].map((tabId) =>
Promise.all([ help.deps.sendToTab(tabId, { type: HELP_CANCEL, requestId: help.requestId }).catch(() => {}),
clearRefTags(help.deps.cdp, tabId),
help.deps
.sendToTab(tabId, { type: HELP_CANCEL, requestId: help.requestId })
.catch(() => {}),
]),
), ),
); );
} }
@@ -476,7 +460,9 @@ function attachHelpRuntimeListener(deps: RequestHelpDeps): () => void {
sendResponse({ active: false }); sendResponse({ active: false });
return; return;
} }
if (tabId === help.primaryTabId) await refreshHelpTargets(help); if (tabId === help.primaryTabId) {
await refreshHelpTargets(help, { scrollIntoView: false });
}
help.overlayTabIds.add(tabId); help.overlayTabIds.add(tabId);
sendResponse({ sendResponse({
active: true, active: true,
@@ -486,6 +472,7 @@ function attachHelpRuntimeListener(deps: RequestHelpDeps): () => void {
...(help.title ? { title: help.title } : {}), ...(help.title ? { title: help.title } : {}),
displayMode: tabId === help.primaryTabId ? "full" : "compact", displayMode: tabId === help.primaryTabId ? "full" : "compact",
selectors: tabId === help.primaryTabId ? help.selectors : [], selectors: tabId === help.primaryTabId ? help.selectors : [],
rects: tabId === help.primaryTabId ? help.rects : [],
timeoutMs: help.timeoutMs, timeoutMs: help.timeoutMs,
}, },
}); });
@@ -686,7 +673,13 @@ export async function handleRequestHelp(
if (deps.autoAttachLifecycle !== false) ensureHelpLifecycleListeners(deps); if (deps.autoAttachLifecycle !== false) ensureHelpLifecycleListeners(deps);
const tabId = target.tabId; const tabId = target.tabId;
const initialTargets = params.targets ?? []; const initialTargets = params.targets ?? [];
const { selectors, resolvedTargets } = await resolveHelpTargets(ctx, tabId, initialTargets, deps); const { selectors, rects, resolvedTargets } = await resolveHelpTargets(
ctx,
tabId,
initialTargets,
deps,
{ scrollIntoView: true },
);
await deps.windows.update(target.windowId, { focused: true }).catch(() => {}); await deps.windows.update(target.windowId, { focused: true }).catch(() => {});
await deps.activateTab(tabId).catch(() => {}); await deps.activateTab(tabId).catch(() => {});
@@ -729,6 +722,7 @@ export async function handleRequestHelp(
...(params.title ? { title: params.title } : {}), ...(params.title ? { title: params.title } : {}),
targets: initialTargets, targets: initialTargets,
selectors, selectors,
rects,
timeoutMs, timeoutMs,
notificationId, notificationId,
resolvedTargets, resolvedTargets,
+88 -56
View File
@@ -3,14 +3,15 @@
// //
// All interaction tools: // All interaction tools:
// 1. Resolve target tab (sandbox: must be inside Agent Window). // 1. Resolve target tab (sandbox: must be inside Agent Window).
// 2. Resolve target element by `ref` (RefStore.resolve with tabId // 2. Resolve target element by `ref` (compound frame/session identity)
// binding) or `selector` (DOM.querySelector + describeNode). // or `selector` (DOM.querySelector + describeNode).
// 3. Scroll the node into view, then dispatch the appropriate // 3. Scroll the node and its frame owners into view, project its live
// `Input.*` CDP events. // content quad into the top viewport, then dispatch `Input.*` events.
// 4. Honour `AbortSignal` so canceled calls don't issue follow-up CDP // 4. Honour `AbortSignal` so canceled calls don't issue follow-up CDP
// commands. // commands.
import { ChromiumCdp } from "@/browser-driver/chromium-cdp"; import { ChromiumCdp } from "@/browser-driver/chromium-cdp";
import type { CdpTarget } from "@/browser-driver/frame-graph";
import type { SessionContext, SessionManager } from "@/session-manager/manager"; import type { SessionContext, SessionManager } from "@/session-manager/manager";
import type { import type {
ClickParams, ClickParams,
@@ -28,17 +29,13 @@ import type {
SelectResult, SelectResult,
} from "@/transport/types"; } from "@/transport/types";
import { attachDialogs, markDialogCursor } from "./dialogs"; import { attachDialogs, markDialogCursor } from "./dialogs";
import { import { backendNodeToObject } from "./element-geometry";
backendNodeToObject,
boxCentre,
nodeCentre,
quadCentre,
scrollNodeIntoView,
} from "./element-geometry";
import { rpcError } from "./errors"; import { rpcError } from "./errors";
import { resolveNodeGeometry, scrollElementAndFramesIntoView } from "./frame-geometry";
import { import {
type CdpRunner, type CdpRunner,
type ChromeTabsApi, type ChromeTabsApi,
cdpRunnerForTarget,
chromeTabsApi, chromeTabsApi,
enforceAgentWindow, enforceAgentWindow,
isRpcError, isRpcError,
@@ -137,7 +134,16 @@ async function resolveBackendNode(
target: { tabId: number }, target: { tabId: number },
params: { ref?: string; selector?: string }, params: { ref?: string; selector?: string },
toolName: string, toolName: string,
): Promise<{ backendNodeId: number; usedRef?: string; usedSelector?: string } | RpcError> { ): Promise<
| {
backendNodeId: number;
cdpTarget: CdpTarget;
frameId?: string;
usedRef?: string;
usedSelector?: string;
}
| RpcError
> {
const hasRef = typeof params.ref === "string" && params.ref.length > 0; const hasRef = typeof params.ref === "string" && params.ref.length > 0;
const hasSelector = typeof params.selector === "string" && params.selector.length > 0; const hasSelector = typeof params.selector === "string" && params.selector.length > 0;
if (hasRef && hasSelector) { if (hasRef && hasSelector) {
@@ -155,7 +161,15 @@ async function resolveBackendNode(
if (hasRef) { if (hasRef) {
const resolved = resolveSnapshotRef(ctx, params.ref as string, target.tabId); const resolved = resolveSnapshotRef(ctx, params.ref as string, target.tabId);
if (isRpcError(resolved)) return resolved; if (isRpcError(resolved)) return resolved;
return { backendNodeId: resolved.backendNodeId, usedRef: resolved.refKey }; return {
backendNodeId: resolved.backendNodeId,
cdpTarget: {
tabId: target.tabId,
...(resolved.cdpSessionId ? { sessionId: resolved.cdpSessionId } : {}),
},
...(resolved.frameId ? { frameId: resolved.frameId } : {}),
usedRef: resolved.refKey,
};
} }
// selector path // selector path
try { try {
@@ -192,7 +206,7 @@ async function resolveBackendNode(
message: "DOM.describeNode returned no backendNodeId", message: "DOM.describeNode returned no backendNodeId",
}; };
} }
return { backendNodeId, usedSelector: params.selector }; return { backendNodeId, cdpTarget: { tabId: target.tabId }, usedSelector: params.selector };
} catch (err) { } catch (err) {
return { return {
code: "cdp_failed", code: "cdp_failed",
@@ -228,19 +242,19 @@ export async function handleClick(
return { code: "cancelled", message: "click aborted" }; return { code: "cancelled", message: "click aborted" };
} }
try { deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId); const geometry = await resolveNodeGeometry(
const scrollErr = await scrollNodeIntoView(deps.cdp, target.tabId, node.backendNodeId); deps.cdp,
if (scrollErr) return scrollErr; target.tabId,
} catch (err) { {
return { target: node.cdpTarget,
code: "cdp_failed", backendNodeId: node.backendNodeId,
message: err instanceof Error ? err.message : String(err), ...(node.frameId ? { frameId: node.frameId } : {}),
}; },
} { scrollIntoView: true },
);
const centre = await nodeCentre(deps.cdp, target.tabId, node.backendNodeId); if (isRpcError(geometry)) return geometry;
if (isRpcError(centre)) return centre; const centre = geometry.actionPoint;
if (throwIfAborted(deps.signal)) { if (throwIfAborted(deps.signal)) {
return { code: "cancelled", message: "click aborted" }; return { code: "cancelled", message: "click aborted" };
@@ -353,19 +367,19 @@ export async function handleHover(
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "hover"); const node = await resolveBackendNode(deps.cdp, ctx, target, params, "hover");
if (isRpcError(node)) return node; if (isRpcError(node)) return node;
try { deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId); const geometry = await resolveNodeGeometry(
const scrollErr = await scrollNodeIntoView(deps.cdp, target.tabId, node.backendNodeId); deps.cdp,
if (scrollErr) return scrollErr; target.tabId,
} catch (err) { {
return { target: node.cdpTarget,
code: "cdp_failed", backendNodeId: node.backendNodeId,
message: err instanceof Error ? err.message : String(err), ...(node.frameId ? { frameId: node.frameId } : {}),
}; },
} { scrollIntoView: true },
);
const centre = await nodeCentre(deps.cdp, target.tabId, node.backendNodeId); if (isRpcError(geometry)) return geometry;
if (isRpcError(centre)) return centre; const centre = geometry.actionPoint;
if (throwIfAborted(deps.signal)) { if (throwIfAborted(deps.signal)) {
return { code: "cancelled", message: "hover aborted" }; return { code: "cancelled", message: "hover aborted" };
@@ -585,10 +599,11 @@ export async function handleFill(
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "fill"); const node = await resolveBackendNode(deps.cdp, ctx, target, params, "fill");
if (isRpcError(node)) return node; if (isRpcError(node)) return node;
const nodeCdp = cdpRunnerForTarget(deps.cdp, node.cdpTarget);
try { try {
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId); deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
const described = await deps.cdp.send<{ node?: DescribedNode }>( const described = await nodeCdp.send<{ node?: DescribedNode }>(
target.tabId, target.tabId,
"DOM.describeNode", "DOM.describeNode",
{ {
@@ -602,12 +617,18 @@ export async function handleFill(
`element ${described.node?.nodeName ?? "?"} not fillable (need input/textarea/contenteditable)`, `element ${described.node?.nodeName ?? "?"} not fillable (need input/textarea/contenteditable)`,
); );
} }
const scrollErr = await scrollNodeIntoView(deps.cdp, target.tabId, node.backendNodeId); const scrollErr = await scrollElementAndFramesIntoView(
deps.cdp,
target.tabId,
node.cdpTarget,
node.backendNodeId,
node.frameId,
);
if (scrollErr) return scrollErr; if (scrollErr) return scrollErr;
if (throwIfAborted(deps.signal)) { if (throwIfAborted(deps.signal)) {
return { code: "cancelled", message: "fill aborted" }; return { code: "cancelled", message: "fill aborted" };
} }
await deps.cdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId }); await nodeCdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
} catch (err) { } catch (err) {
return { return {
code: "cdp_failed", code: "cdp_failed",
@@ -619,7 +640,7 @@ export async function handleFill(
return { code: "cancelled", message: "fill aborted" }; return { code: "cancelled", message: "fill aborted" };
} }
const objectIdOrErr = await backendNodeToObject(deps.cdp, target.tabId, node.backendNodeId); const objectIdOrErr = await backendNodeToObject(nodeCdp, target.tabId, node.backendNodeId);
if (isRpcError(objectIdOrErr)) return objectIdOrErr; if (isRpcError(objectIdOrErr)) return objectIdOrErr;
const objectId = objectIdOrErr; const objectId = objectIdOrErr;
const clearBefore = params.clear_before ?? true; const clearBefore = params.clear_before ?? true;
@@ -628,7 +649,7 @@ export async function handleFill(
if (clearBefore) { if (clearBefore) {
// Clear input/textarea value or wipe contenteditable innerText, // Clear input/textarea value or wipe contenteditable innerText,
// then fire `input` so frameworks observe the empty state. // then fire `input` so frameworks observe the empty state.
await deps.cdp.send(target.tabId, "Runtime.callFunctionOn", { await nodeCdp.send(target.tabId, "Runtime.callFunctionOn", {
objectId, objectId,
functionDeclaration: `function() { functionDeclaration: `function() {
if (this.isContentEditable) { this.textContent = ''; } if (this.isContentEditable) { this.textContent = ''; }
@@ -655,7 +676,7 @@ export async function handleFill(
return { code: "cancelled", message: "fill aborted" }; return { code: "cancelled", message: "fill aborted" };
} }
// Fire `input` + `change` so React / Vue controlled inputs commit. // Fire `input` + `change` so React / Vue controlled inputs commit.
await deps.cdp.send(target.tabId, "Runtime.callFunctionOn", { await nodeCdp.send(target.tabId, "Runtime.callFunctionOn", {
objectId, objectId,
functionDeclaration: `function() { functionDeclaration: `function() {
this.dispatchEvent(new Event('input', { bubbles: true })); this.dispatchEvent(new Event('input', { bubbles: true }));
@@ -853,14 +874,21 @@ export async function handlePress(
if (params.ref || params.selector) { if (params.ref || params.selector) {
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "press"); const node = await resolveBackendNode(deps.cdp, ctx, target, params, "press");
if (isRpcError(node)) return node; if (isRpcError(node)) return node;
const nodeCdp = cdpRunnerForTarget(deps.cdp, node.cdpTarget);
try { try {
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId); deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
const scrollErr = await scrollNodeIntoView(deps.cdp, target.tabId, node.backendNodeId); const scrollErr = await scrollElementAndFramesIntoView(
deps.cdp,
target.tabId,
node.cdpTarget,
node.backendNodeId,
node.frameId,
);
if (scrollErr) return scrollErr; if (scrollErr) return scrollErr;
if (throwIfAborted(deps.signal)) { if (throwIfAborted(deps.signal)) {
return { code: "cancelled", message: "press aborted" }; return { code: "cancelled", message: "press aborted" };
} }
await deps.cdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId }); await nodeCdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
} catch (err) { } catch (err) {
return { return {
code: "cdp_failed", code: "cdp_failed",
@@ -965,10 +993,11 @@ export async function handleSelect(
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "select"); const node = await resolveBackendNode(deps.cdp, ctx, target, params, "select");
if (isRpcError(node)) return node; if (isRpcError(node)) return node;
const nodeCdp = cdpRunnerForTarget(deps.cdp, node.cdpTarget);
try { try {
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId); deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
const described = await deps.cdp.send<{ node?: DescribedNode }>( const described = await nodeCdp.send<{ node?: DescribedNode }>(
target.tabId, target.tabId,
"DOM.describeNode", "DOM.describeNode",
{ backendNodeId: node.backendNodeId }, { backendNodeId: node.backendNodeId },
@@ -991,12 +1020,18 @@ export async function handleSelect(
"single-select <select> requires exactly one value", "single-select <select> requires exactly one value",
); );
} }
const scrollErr = await scrollNodeIntoView(deps.cdp, target.tabId, node.backendNodeId); const scrollErr = await scrollElementAndFramesIntoView(
deps.cdp,
target.tabId,
node.cdpTarget,
node.backendNodeId,
node.frameId,
);
if (scrollErr) return scrollErr; if (scrollErr) return scrollErr;
if (throwIfAborted(deps.signal)) { if (throwIfAborted(deps.signal)) {
return { code: "cancelled", message: "select aborted" }; return { code: "cancelled", message: "select aborted" };
} }
await deps.cdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId }); await nodeCdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
} catch (err) { } catch (err) {
return { return {
code: "cdp_failed", code: "cdp_failed",
@@ -1008,12 +1043,12 @@ export async function handleSelect(
return { code: "cancelled", message: "select aborted" }; return { code: "cancelled", message: "select aborted" };
} }
const objectIdOrErr = await backendNodeToObject(deps.cdp, target.tabId, node.backendNodeId); const objectIdOrErr = await backendNodeToObject(nodeCdp, target.tabId, node.backendNodeId);
if (isRpcError(objectIdOrErr)) return objectIdOrErr; if (isRpcError(objectIdOrErr)) return objectIdOrErr;
const objectId = objectIdOrErr; const objectId = objectIdOrErr;
try { try {
const evaluated = await deps.cdp.send<{ const evaluated = await nodeCdp.send<{
result?: { value?: SelectMutationResult | null }; result?: { value?: SelectMutationResult | null };
}>(target.tabId, "Runtime.callFunctionOn", { }>(target.tabId, "Runtime.callFunctionOn", {
objectId, objectId,
@@ -1072,9 +1107,6 @@ export async function handleSelect(
export const __testing__ = { export const __testing__ = {
DEFAULT_TIMEOUT_MS, DEFAULT_TIMEOUT_MS,
quadCentre,
boxCentre,
resolveBackendNode, resolveBackendNode,
nodeCentre,
isFillable, isFillable,
}; };
+204 -37
View File
@@ -13,6 +13,7 @@ import {
type VomScene, type VomScene,
} from "@browser-skill/vom"; } from "@browser-skill/vom";
import { ChromiumCdp } from "@/browser-driver/chromium-cdp"; import { ChromiumCdp } from "@/browser-driver/chromium-cdp";
import type { CdpTarget } from "@/browser-driver/frame-graph";
import { import {
type CaptureSuppressSendToTab, type CaptureSuppressSendToTab,
withOverlaysHiddenForCapture, withOverlaysHiddenForCapture,
@@ -30,8 +31,8 @@ import type {
SnapshotResult, SnapshotResult,
} from "@/transport/types"; } from "@/transport/types";
import { attachDialogs, markDialogCursor } from "./dialogs"; import { attachDialogs, markDialogCursor } from "./dialogs";
import { nodeBoundingRect, scrollNodeIntoView } from "./element-geometry";
import { rpcError } from "./errors"; import { rpcError } from "./errors";
import { resolveNodeGeometry } from "./frame-geometry";
import { import {
type ChromeTabsApi, type ChromeTabsApi,
enforceToolTargetScope, enforceToolTargetScope,
@@ -40,6 +41,7 @@ import {
type ResolvedTargetTab, type ResolvedTargetTab,
resolveCdpAccessibleTargetTab, resolveCdpAccessibleTargetTab,
type CdpRunner as SharedCdpRunner, type CdpRunner as SharedCdpRunner,
sendToCdpTarget,
normaliseRef as sharedNormaliseRef, normaliseRef as sharedNormaliseRef,
type ToolEffect, type ToolEffect,
} from "./shared"; } from "./shared";
@@ -50,6 +52,7 @@ import {
captureViewModel, captureViewModel,
collectOverlayExcludedBackendIds, collectOverlayExcludedBackendIds,
} from "./vom/capture"; } from "./vom/capture";
import { type CapturedFrameDocument, captureFrameData } from "./vom/frame-capture";
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// Shared helpers (legacy aliases — observation.ts kept exporting these // Shared helpers (legacy aliases — observation.ts kept exporting these
@@ -162,26 +165,30 @@ function throwIfAborted(signal: AbortSignal | undefined, tool: string): void {
async function captureElementScreenshot( async function captureElementScreenshot(
cdp: SharedCdpRunner, cdp: SharedCdpRunner,
tabId: number, tabId: number,
target: CdpTarget,
backendNodeId: number, backendNodeId: number,
frameId?: string,
signal?: AbortSignal, signal?: AbortSignal,
): Promise<{ image_base64: string; width: number; height: number } | RpcError> { ): Promise<{ image_base64: string; width: number; height: number } | RpcError> {
if (signal?.aborted) return cancelled("screenshot"); if (signal?.aborted) return cancelled("screenshot");
const scrollErr = await scrollNodeIntoView(cdp, tabId, backendNodeId); const geometry = await resolveNodeGeometry(
if (scrollErr) return scrollErr; cdp,
if (signal?.aborted) return cancelled("screenshot"); tabId,
{ target, backendNodeId, ...(frameId ? { frameId } : {}) },
const rectOrErr = await nodeBoundingRect(cdp, tabId, backendNodeId); { scrollIntoView: true },
if (isRpcError(rectOrErr)) return rectOrErr; );
if (isRpcError(geometry)) return geometry;
if (signal?.aborted) return cancelled("screenshot"); if (signal?.aborted) return cancelled("screenshot");
const rect = geometry.topBounds;
try { try {
const shot = await cdp.send<{ data?: string }>(tabId, "Page.captureScreenshot", { const shot = await cdp.send<{ data?: string }>(tabId, "Page.captureScreenshot", {
format: "png", format: "png",
clip: { clip: {
x: rectOrErr.x, x: rect.x,
y: rectOrErr.y, y: rect.y,
width: rectOrErr.width, width: rect.width,
height: rectOrErr.height, height: rect.height,
scale: 1, scale: 1,
}, },
}); });
@@ -191,8 +198,8 @@ async function captureElementScreenshot(
return { code: "cdp_failed", message: "Page.captureScreenshot returned no data" }; return { code: "cdp_failed", message: "Page.captureScreenshot returned no data" };
} }
const dims = parsePngDimensions(image_base64) ?? { const dims = parsePngDimensions(image_base64) ?? {
width: Math.round(rectOrErr.width), width: Math.round(rect.width),
height: Math.round(rectOrErr.height), height: Math.round(rect.height),
}; };
return { image_base64, width: dims.width, height: dims.height }; return { image_base64, width: dims.width, height: dims.height };
} catch (err) { } catch (err) {
@@ -296,9 +303,21 @@ export async function handleScreenshot(
await deps.cdp.ensureAttachedToUrl?.(target.tabId, target.url); await deps.cdp.ensureAttachedToUrl?.(target.tabId, target.url);
if (signal?.aborted) return cancelled("screenshot"); if (signal?.aborted) return cancelled("screenshot");
const cdp = deps.cdp; const cdp = deps.cdp;
const nodeTarget = {
tabId: target.tabId,
...(node.cdpSessionId ? { sessionId: node.cdpSessionId } : {}),
};
const captured = await withOverlaysHiddenForCapture( const captured = await withOverlaysHiddenForCapture(
target.tabId, target.tabId,
() => captureElementScreenshot(cdp, target.tabId, node.backendNodeId, signal), () =>
captureElementScreenshot(
cdp,
target.tabId,
nodeTarget,
node.backendNodeId,
node.frameId,
signal,
),
deps.sendToTab, deps.sendToTab,
); );
if (isRpcError(captured)) return captured; if (isRpcError(captured)) return captured;
@@ -352,6 +371,7 @@ export type CdpRunner = SharedCdpRunner;
/** Subset of CDP `AXNode` we care about — see `Accessibility.AXNode`. */ /** Subset of CDP `AXNode` we care about — see `Accessibility.AXNode`. */
export interface CdpAxNode { export interface CdpAxNode {
nodeId: string; nodeId: string;
frameId?: string;
parentId?: string; parentId?: string;
backendDOMNodeId?: number; backendDOMNodeId?: number;
ignored?: boolean; ignored?: boolean;
@@ -584,7 +604,7 @@ function iframeNameFor(node: CapturedNode): string | undefined {
function isRenderableIframeControl(node: CapturedNode): boolean { function isRenderableIframeControl(node: CapturedNode): boolean {
const tag = normalizeTag(node.tag); const tag = normalizeTag(node.tag);
if (!IFRAME_RENDERABLE_TAGS.has(tag)) return false; if (!IFRAME_RENDERABLE_TAGS.has(tag)) return false;
if (!node.rect) return false; if (!node.localRect && !node.rect) return false;
if (node.pointerEvents === "none") return false; if (node.pointerEvents === "none") return false;
if ((node.attrs.type ?? "").toLowerCase() === "hidden") return false; if ((node.attrs.type ?? "").toLowerCase() === "hidden") return false;
@@ -756,7 +776,12 @@ function applyCapturedSignals(
signals: VomNodeDomSignals, signals: VomNodeDomSignals,
): VomNode { ): VomNode {
if (!capturedNode) return node; if (!capturedNode) return node;
const attrs = capturedNode.attrs; const attrs =
node.sensitive && Object.prototype.hasOwnProperty.call(capturedNode.attrs, "value")
? Object.fromEntries(
Object.entries(capturedNode.attrs).filter(([name]) => name.toLowerCase() !== "value"),
)
: capturedNode.attrs;
const text = cleanAttr(capturedNode.textContent); const text = cleanAttr(capturedNode.textContent);
const nearbyText = nearbyTextFor(capturedNode, signals.childrenByParentId); const nearbyText = nearbyTextFor(capturedNode, signals.childrenByParentId);
const placeholder = cleanAttr(capturedNode.formPlaceholder) ?? cleanAttr(attrs.placeholder); const placeholder = cleanAttr(capturedNode.formPlaceholder) ?? cleanAttr(attrs.placeholder);
@@ -823,8 +848,7 @@ function vomNodeFromCaptured(
signals: VomNodeDomSignals, signals: VomNodeDomSignals,
): VomNode { ): VomNode {
const sensitive = isSensitive(capturedNode); const sensitive = isSensitive(capturedNode);
const value = const value = sensitive ? undefined : capturedNode.formValue;
capturedNode.formValue ?? (sensitive ? (capturedNode.attrs.value ?? "") : undefined);
const tag = normalizeTag(capturedNode.tag); const tag = normalizeTag(capturedNode.tag);
const node = applyCapturedSignals( const node = applyCapturedSignals(
{ {
@@ -860,8 +884,17 @@ function axVomNode(
): VomNode { ): VomNode {
const role = axString(axNode.role); const role = axString(axNode.role);
const capturedValue = capturedNode?.formValue; const capturedValue = capturedNode?.formValue;
const value = capturedValue ?? axValue(axNode.value); const sourceValue = capturedValue ?? axValue(axNode.value);
const signalsForNode = axSignals.get(axNode.nodeId); const signalsForNode = axSignals.get(axNode.nodeId);
const sensitive = isSensitive(capturedNode) || signalsForNode?.sensitive === true;
const value = sensitive ? undefined : sourceValue;
const inputState =
inputStateFor(capturedNode, sourceValue) ??
(sensitive && ["textbox", "searchbox"].includes(role?.toLowerCase() ?? "")
? sourceValue === undefined || sourceValue === ""
? "empty"
: "filled"
: undefined);
let name = FORM_CONTROL_TAGS.has(normalizeTag(capturedNode?.tag)) let name = FORM_CONTROL_TAGS.has(normalizeTag(capturedNode?.tag))
? formControlName(capturedNode, axString(axNode.name), axSiblingLabel(axNode, axById), signals) ? formControlName(capturedNode, axString(axNode.name), axSiblingLabel(axNode, axById), signals)
: (axString(axNode.name) ?? signalsForNode?.aggregatedText); : (axString(axNode.name) ?? signalsForNode?.aggregatedText);
@@ -877,8 +910,8 @@ function axVomNode(
position: capturedNode?.position || "static", position: capturedNode?.position || "static",
pointerEvents: capturedNode?.pointerEvents || "auto", pointerEvents: capturedNode?.pointerEvents || "auto",
modal: isModalSignal(axNode, capturedNode), modal: isModalSignal(axNode, capturedNode),
sensitive: isSensitive(capturedNode) || signalsForNode?.sensitive === true, sensitive,
inputState: inputStateFor(capturedNode, value), inputState,
}; };
if (role) node.role = role; if (role) node.role = role;
if (name) node.name = name; if (name) node.name = name;
@@ -1141,6 +1174,8 @@ export interface BuildVomSceneOptions {
pageUrl?: string; pageUrl?: string;
} }
export type VomFrameDocument = CapturedFrameDocument<CdpAxNode>;
function externalHrefHost( function externalHrefHost(
href: string | undefined, href: string | undefined,
pageUrl: string | undefined, pageUrl: string | undefined,
@@ -1234,6 +1269,119 @@ export function buildVomScene(
}; };
} }
export function buildFrameVomScene(
documents: VomFrameDocument[],
captured: CapturedViewModel,
options: BuildVomSceneOptions = {},
): VomScene {
const documentByFrameId = new Map(documents.map((document) => [document.frameId, document]));
const rootFrameId =
captured.rootFrameId ?? documents.find((document) => !document.parentFrameId)?.frameId;
const orderedFrameIds = [
...(rootFrameId ? [rootFrameId] : []),
...documents.map((document) => document.frameId),
].filter((frameId, index, all) => all.indexOf(frameId) === index);
const localScenes = new Map<string, VomScene>();
for (const frameId of orderedFrameIds) {
const document = documentByFrameId.get(frameId);
if (!document) continue;
localScenes.set(
frameId,
buildVomScene(
document.axNodes,
{
viewport: captured.viewport,
nodes: document.domNodes,
iframeNodes: new Map(),
surfaceProbes: frameId === rootFrameId ? captured.surfaceProbes : [],
excludedBackendNodeIds:
frameId === rootFrameId ? captured.excludedBackendNodeIds : new Set(),
},
{ pageUrl: document.url ?? options.pageUrl },
),
);
}
const sceneIdByFrameAndBackend = new Map<string, number>();
let nextSceneId = 1;
for (const [frameId, scene] of localScenes) {
for (const node of scene.nodes) {
sceneIdByFrameAndBackend.set(`${frameId}:${node.id}`, nextSceneId);
nextSceneId += 1;
}
}
const nodes: VomNode[] = [];
const surfaces: CondSurface[] = [];
const activeScopeBlocks: ActiveScopeBlock[] = [];
for (const [frameId, scene] of localScenes) {
const document = documentByFrameId.get(frameId);
const ownerBackendNodeId = document?.ownerBackendNodeId;
const parentFrameId = document?.parentFrameId;
const ownerSceneId =
ownerBackendNodeId !== undefined && parentFrameId
? sceneIdByFrameAndBackend.get(`${parentFrameId}:${ownerBackendNodeId}`)
: undefined;
if (frameId !== rootFrameId && ownerSceneId === undefined) continue;
const suppressedRoots = new Set(
frameId === rootFrameId
? []
: scene.nodes
.filter(
(node) =>
node.parentId === null &&
["rootwebarea", "webarea"].includes(node.role?.toLowerCase() ?? ""),
)
.map((node) => node.id),
);
for (const node of scene.nodes) {
if (suppressedRoots.has(node.id)) continue;
const sceneNodeId = sceneIdByFrameAndBackend.get(`${frameId}:${node.id}`) as number;
let parentId =
node.parentId !== null
? sceneIdByFrameAndBackend.get(`${frameId}:${node.parentId}`)
: ownerSceneId;
if (node.parentId !== null && suppressedRoots.has(node.parentId)) parentId = ownerSceneId;
const remapId = (id: number) => sceneIdByFrameAndBackend.get(`${frameId}:${id}`);
nodes.push({
...node,
id: sceneNodeId,
backendNodeId: node.id,
frameId,
contextScopeId: document?.contextScopeId ?? frameId,
parentId: parentId ?? null,
...(node.domParentId !== undefined
? { domParentId: node.domParentId === null ? null : (remapId(node.domParentId) ?? null) }
: {}),
...(node.domAncestorIds
? { domAncestorIds: node.domAncestorIds.flatMap((id) => remapId(id) ?? []) }
: {}),
});
}
for (const surface of scene.surfaces ?? []) {
const triggerId = sceneIdByFrameAndBackend.get(`${frameId}:${surface.triggerId}`);
if (triggerId !== undefined) surfaces.push({ ...surface, triggerId });
}
for (const block of scene.activeScopeBlocks ?? []) {
const triggerId = sceneIdByFrameAndBackend.get(`${frameId}:${block.triggerId}`);
if (triggerId !== undefined) activeScopeBlocks.push({ ...block, triggerId });
}
}
return {
viewport: captured.viewport,
nodes,
...(rootFrameId ? { rootFrameId } : {}),
...(surfaces.length > 0 ? { surfaces } : {}),
...(activeScopeBlocks.length > 0 ? { activeScopeBlocks } : {}),
};
}
export interface SnapshotDeps { export interface SnapshotDeps {
cdp: CdpRunner; cdp: CdpRunner;
tabsApi: { tabsApi: {
@@ -1322,9 +1470,15 @@ export async function handleGetHtml(
if (params.ref) { if (params.ref) {
const resolved = resolveSnapshotRef(ctx, params.ref, target.tabId); const resolved = resolveSnapshotRef(ctx, params.ref, target.tabId);
if (isRpcError(resolved)) return resolved; if (isRpcError(resolved)) return resolved;
const resp = await deps.cdp.send<{ outerHTML?: string }>(target.tabId, "DOM.getOuterHTML", { const resp = await sendToCdpTarget<{ outerHTML?: string }>(
backendNodeId: resolved.backendNodeId, deps.cdp,
}); {
tabId: target.tabId,
...(resolved.cdpSessionId ? { sessionId: resolved.cdpSessionId } : {}),
},
"DOM.getOuterHTML",
{ backendNodeId: resolved.backendNodeId },
);
throwIfAborted(signal, "get_html"); throwIfAborted(signal, "get_html");
html = resp.outerHTML ?? ""; html = resp.outerHTML ?? "";
} else { } else {
@@ -1447,15 +1601,6 @@ async function handleVomObservation(
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId); deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
await deps.cdp.ensureAttachedToUrl?.(target.tabId, target.url); await deps.cdp.ensureAttachedToUrl?.(target.tabId, target.url);
throwIfAborted(signal, toolName); throwIfAborted(signal, toolName);
await deps.cdp.send<unknown>(target.tabId, "Accessibility.enable", {});
throwIfAborted(signal, toolName);
const result = await deps.cdp.send<{ nodes: CdpAxNode[] }>(
target.tabId,
"Accessibility.getFullAXTree",
{},
);
throwIfAborted(signal, toolName);
const axNodes = result.nodes ?? [];
const effectiveConditionalSurfaceProbe = const effectiveConditionalSurfaceProbe =
deps.conditionalSurfaceProbe ?? conditionalSurfaceProbe; deps.conditionalSurfaceProbe ?? conditionalSurfaceProbe;
const captured = await captureForVom( const captured = await captureForVom(
@@ -1466,17 +1611,39 @@ async function handleVomObservation(
signal, signal,
); );
throwIfAborted(signal, toolName); throwIfAborted(signal, toolName);
const scene = buildVomScene(axNodes, captured, { pageUrl: target.url }); const frameDocuments = await captureFrameData<CdpAxNode>(
deps.cdp,
target.tabId,
captured,
signal,
);
throwIfAborted(signal, toolName);
const scene =
captured.rootFrameId && captured.frameNodes?.size
? buildFrameVomScene(frameDocuments, captured, { pageUrl: target.url })
: buildVomScene(frameDocuments[0]?.axNodes ?? [], captured, { pageUrl: target.url });
const rendered = renderVom(scene, { const rendered = renderVom(scene, {
maxDepth: params.max_depth, maxDepth: params.max_depth,
maxTokens: params.max_tokens, maxTokens: params.max_tokens,
activeRegionPolicy: true, activeRegionPolicy: true,
}); });
throwIfAborted(signal, toolName); throwIfAborted(signal, toolName);
const targetByFrameId = new Map(
frameDocuments.map((document) => [document.frameId, document.target]),
);
ctx.refStore.replace( ctx.refStore.replace(
rendered.refs.map( rendered.refs.map((ref) => {
(r) => [r.ref, { backendNodeId: r.backendNodeId, tabId: target.tabId }] as const, const refTarget = ref.frameId ? targetByFrameId.get(ref.frameId) : undefined;
), return [
ref.ref,
{
backendNodeId: ref.backendNodeId,
tabId: target.tabId,
...(ref.frameId ? { frameId: ref.frameId } : {}),
...(refTarget?.sessionId ? { cdpSessionId: refTarget.sessionId } : {}),
},
] as const;
}),
); );
return attachDialogs(deps.cdp, target.tabId, dialogCursor, { return attachDialogs(deps.cdp, target.tabId, dialogCursor, {
text: rendered.text, text: rendered.text,
+24 -2
View File
@@ -6,7 +6,8 @@
// exactly the same sandbox + visibility rules as the M6 observation // exactly the same sandbox + visibility rules as the M6 observation
// handlers (review parity). // handlers (review parity).
import type { DialogCursor } from "@/browser-driver/chromium-cdp"; import type { CdpDebuggee, DialogCursor } from "@/browser-driver/chromium-cdp";
import type { CdpFrameGraph, CdpTarget } from "@/browser-driver/frame-graph";
import type { SessionContext, SessionManager } from "@/session-manager/manager"; import type { SessionContext, SessionManager } from "@/session-manager/manager";
import { normaliseRef } from "@/session-manager/ref-store"; import { normaliseRef } from "@/session-manager/ref-store";
import type { ConsoleResult, JavaScriptDialogInfo, RpcError } from "@/transport/types"; import type { ConsoleResult, JavaScriptDialogInfo, RpcError } from "@/transport/types";
@@ -51,9 +52,11 @@ export type { DialogCursor };
*/ */
export interface CdpRunner { export interface CdpRunner {
send<T = unknown>(tabId: number, method: string, params?: object): Promise<T>; send<T = unknown>(tabId: number, method: string, params?: object): Promise<T>;
sendToTarget?<T = unknown>(target: CdpTarget, method: string, params?: object): Promise<T>;
getFrameGraph?(tabId: number): Promise<CdpFrameGraph>;
ensureAttachedToUrl?(tabId: number, expectedUrl: string | undefined): Promise<void>; ensureAttachedToUrl?(tabId: number, expectedUrl: string | undefined): Promise<void>;
trackSessionTab?(sessionId: string, tabId: number): void; trackSessionTab?(sessionId: string, tabId: number): void;
onEvent?(handler: (source: chrome.debugger.Debuggee, method: string, params: unknown) => void): { onEvent?(handler: (source: CdpDebuggee, method: string, params: unknown) => void): {
dispose(): void; dispose(): void;
}; };
dialogCursor?(tabId: number): DialogCursor; dialogCursor?(tabId: number): DialogCursor;
@@ -68,6 +71,25 @@ export interface CdpRunner {
): ConsoleResult; ): ConsoleResult;
} }
export function sendToCdpTarget<T = unknown>(
cdp: CdpRunner,
target: CdpTarget,
method: string,
params?: object,
): Promise<T> {
if (target.sessionId && cdp.sendToTarget) {
return cdp.sendToTarget<T>(target, method, params);
}
return cdp.send<T>(target.tabId, method, params);
}
export function cdpRunnerForTarget(cdp: CdpRunner, target: CdpTarget): CdpRunner {
return {
send: <T = unknown>(_tabId: number, method: string, params?: object) =>
sendToCdpTarget<T>(cdp, target, method, params),
};
}
export interface BufferedReadBounds { export interface BufferedReadBounds {
since: number | undefined; since: number | undefined;
limit: number; limit: number;
+17 -5
View File
@@ -1,5 +1,5 @@
// Snapshot ref resolution — normalise `@eN` / `eN`, look up // Snapshot ref resolution — normalise `@eN` / `eN`, look up
// `backendNodeId` via the session RefStore, and build stable // compound CDP node identity via the session RefStore, and build stable
// `ref_not_found` errors for hard-failure tool paths. // `ref_not_found` errors for hard-failure tool paths.
import type { SessionContext } from "@/session-manager/manager"; import type { SessionContext } from "@/session-manager/manager";
@@ -10,6 +10,13 @@ import { rpcError } from "./errors";
export interface SnapshotRefLookup { export interface SnapshotRefLookup {
backendNodeId: number; backendNodeId: number;
refKey: string; refKey: string;
frameId?: string;
cdpSessionId?: string;
}
function refEntryForTab(ctx: SessionContext, refKey: string, tabId: number) {
const entry = ctx.refStore.resolveEntry(refKey);
return entry?.tabId === tabId ? entry : null;
} }
/** /**
@@ -23,9 +30,14 @@ export function lookupSnapshotRef(
tabId: number, tabId: number,
): SnapshotRefLookup | null { ): SnapshotRefLookup | null {
const refKey = normaliseRef(ref); const refKey = normaliseRef(ref);
const backendNodeId = ctx.refStore.resolve(refKey, { tabId }); const entry = refEntryForTab(ctx, refKey, tabId);
if (backendNodeId === null) return null; if (!entry) return null;
return { backendNodeId, refKey }; return {
backendNodeId: entry.backendNodeId,
refKey,
...(entry.frameId ? { frameId: entry.frameId } : {}),
...(entry.cdpSessionId ? { cdpSessionId: entry.cdpSessionId } : {}),
};
} }
/** /**
@@ -37,7 +49,7 @@ export function resolveSnapshotRef(
ctx: SessionContext, ctx: SessionContext,
ref: string, ref: string,
tabId: number, tabId: number,
): { backendNodeId: number; refKey: string } | RpcError { ): SnapshotRefLookup | RpcError {
const looked = lookupSnapshotRef(ctx, ref, tabId); const looked = lookupSnapshotRef(ctx, ref, tabId);
if (looked === null) { if (looked === null) {
return rpcError( return rpcError(
@@ -18,6 +18,8 @@ function fakeSnapshotReply() {
"auto", "auto",
"type", "type",
"password", "password",
"value",
"hunter2",
]; ];
const i = (s: string) => S.indexOf(s); const i = (s: string) => S.indexOf(s);
return { return {
@@ -29,7 +31,8 @@ function fakeSnapshotReply() {
nodeType: [1, 1, 1, 1], nodeType: [1, 1, 1, 1],
nodeName: [i("html"), i("body"), i("div"), i("input")], nodeName: [i("html"), i("body"), i("div"), i("input")],
backendNodeId: [10, 11, 12, 13], backendNodeId: [10, 11, 12, 13],
attributes: [[], [], [], [i("type"), i("password")]], attributes: [[], [], [], [i("type"), i("password"), i("value"), i("hunter2")]],
inputValue: { index: [3], value: [i("hunter2")] },
}, },
layout: { layout: {
nodeIndex: [1, 2, 3], nodeIndex: [1, 2, 3],
@@ -217,7 +220,8 @@ function makeCdp(snapshot: unknown) {
{ {
state: "filled", state: "filled",
sensitive: true, sensitive: true,
defaultValue: "", defaultValue: "hunter2",
value: "hunter2",
placeholder: "secret", placeholder: "secret",
}, },
], ],
@@ -262,10 +266,11 @@ describe("captureViewModel", () => {
expect(input).toMatchObject({ expect(input).toMatchObject({
formState: "filled", formState: "filled",
formDefaultValue: "",
formPlaceholder: "secret", formPlaceholder: "secret",
attrs: { type: "password" },
}); });
expect(input?.formValue).toBeUndefined(); expect(input?.formValue).toBeUndefined();
expect(input?.formDefaultValue).toBeUndefined();
expect(cdp.send).toHaveBeenCalledWith( expect(cdp.send).toHaveBeenCalledWith(
4, 4,
"Runtime.evaluate", "Runtime.evaluate",
@@ -692,9 +697,7 @@ describe("captureViewModel", () => {
}; };
const { nodes } = await captureViewModel(cdp, 4); const { nodes } = await captureViewModel(cdp, 4);
const div = nodes.find((n) => n.backendNodeId === 12); const div = nodes.find((n) => n.backendNodeId === 12);
// local rect preserves viewport-relative scroll math.
expect(div?.localRect?.y).toBe(-200); expect(div?.localRect?.y).toBe(-200);
// exported rect is clipped to the top-level viewport before VOM consumes it.
expect(div?.rect).toMatchObject({ y: 0, h: 600 }); expect(div?.rect).toMatchObject({ y: 0, h: 600 });
}); });
@@ -839,6 +842,7 @@ describe("captureViewModel", () => {
strings: S, strings: S,
documents: [ documents: [
{ {
frameId: "main-frame",
scrollOffsetX: 0, scrollOffsetX: 0,
scrollOffsetY: 0, scrollOffsetY: 0,
nodes: { nodes: {
@@ -866,6 +870,7 @@ describe("captureViewModel", () => {
}, },
// documents[1]: the iframe's sub-document with a login input // documents[1]: the iframe's sub-document with a login input
{ {
frameId: "child-frame",
scrollOffsetX: 0, scrollOffsetX: 0,
scrollOffsetY: 0, scrollOffsetY: 0,
nodes: { nodes: {
@@ -895,7 +900,14 @@ describe("captureViewModel", () => {
throw new Error(method); throw new Error(method);
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>, }) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
}; };
const { nodes, iframeNodes } = await captureViewModel(cdp, 4); const {
nodes,
iframeNodes,
frameNodes,
frameOwnerBackendNodeIds,
frameParentIds,
rootFrameId,
} = await captureViewModel(cdp, 4);
// Main frame has 4 nodes; iframe is included // Main frame has 4 nodes; iframe is included
expect(nodes.find((n) => n.backendNodeId === 13)?.tag).toBe("iframe"); expect(nodes.find((n) => n.backendNodeId === 13)?.tag).toBe("iframe");
// iframeNodes keyed by the <iframe> element's backendNodeId=13 // iframeNodes keyed by the <iframe> element's backendNodeId=13
@@ -908,9 +920,13 @@ describe("captureViewModel", () => {
expect(input?.ownerFrameBackendNodeId).toBe(13); expect(input?.ownerFrameBackendNodeId).toBe(13);
expect(input?.localRect).toEqual({ x: 0, y: 0, w: 200, h: 40 }); expect(input?.localRect).toEqual({ x: 0, y: 0, w: 200, h: 40 });
expect(input?.rect).toEqual({ x: 100, y: 300, w: 200, h: 40 }); expect(input?.rect).toEqual({ x: 100, y: 300, w: 200, h: 40 });
expect(rootFrameId).toBe("main-frame");
expect(frameNodes?.get("child-frame")).toBe(subNodes);
expect(frameOwnerBackendNodeIds?.get("child-frame")).toBe(13);
expect(frameParentIds?.get("child-frame")).toBe("main-frame");
}); });
it("clips iframe sub-document rects to the iframe viewport in top coordinates", async () => { it("captures iframe documents without requiring owner-frame geometry", async () => {
const S = [ const S = [
"html", "html",
"body", "body",
@@ -938,16 +954,10 @@ describe("captureViewModel", () => {
contentDocumentIndex: { index: [2], value: [1] }, contentDocumentIndex: { index: [2], value: [1] },
}, },
layout: { layout: {
nodeIndex: [1, 2], nodeIndex: [1],
styles: [ styles: [[i("static"), i("auto")]],
[i("static"), i("auto")], bounds: [[0, 0, 1000, 800]],
[i("static"), i("auto")], paintOrders: [0],
],
bounds: [
[0, 0, 1000, 800],
[950, 760, 200, 100],
],
paintOrders: [0, 1],
}, },
}, },
{ {
@@ -985,7 +995,7 @@ describe("captureViewModel", () => {
const input = iframeNodes.get(13)?.find((n) => n.backendNodeId === 101); const input = iframeNodes.get(13)?.find((n) => n.backendNodeId === 101);
expect(input?.localRect).toEqual({ x: 20, y: 20, w: 120, h: 60 }); expect(input?.localRect).toEqual({ x: 20, y: 20, w: 120, h: 60 });
expect(input?.rect).toEqual({ x: 970, y: 780, w: 30, h: 20 }); expect(input?.rect).toBeNull();
}); });
it("recursively normalizes nested iframe sub-documents", async () => { it("recursively normalizes nested iframe sub-documents", async () => {
@@ -1080,8 +1090,10 @@ describe("captureViewModel", () => {
const nestedIframe = iframeNodes.get(13)?.find((n) => n.backendNodeId === 23); const nestedIframe = iframeNodes.get(13)?.find((n) => n.backendNodeId === 23);
const input = iframeNodes.get(23)?.find((n) => n.backendNodeId === 31); const input = iframeNodes.get(23)?.find((n) => n.backendNodeId === 31);
expect(nestedIframe?.localRect).toEqual({ x: 5, y: 6, w: 100, h: 80 });
expect(nestedIframe?.rect).toEqual({ x: 15, y: 26, w: 100, h: 80 }); expect(nestedIframe?.rect).toEqual({ x: 15, y: 26, w: 100, h: 80 });
expect(input?.ownerFrameBackendNodeId).toBe(23); expect(input?.ownerFrameBackendNodeId).toBe(23);
expect(input?.localRect).toEqual({ x: 1, y: 2, w: 40, h: 20 });
expect(input?.rect).toEqual({ x: 16, y: 28, w: 40, h: 20 }); expect(input?.rect).toEqual({ x: 16, y: 28, w: 40, h: 20 });
}); });
@@ -0,0 +1,135 @@
import { describe, expect, it, vi } from "vitest";
import type { CdpRunner } from "../../shared";
import type { CapturedNode, CapturedViewModel } from "../capture";
import { captureFrameData } from "../frame-capture";
function ownerNode(backendNodeId: number, x: number): CapturedNode {
return {
backendNodeId,
parentBackendNodeId: 1,
frameId: "main",
ownerFrameBackendNodeId: null,
tag: "iframe",
attrs: {},
rect: { x, y: 100, w: 300, h: 200 },
localRect: { x, y: 100, w: 300, h: 200 },
paintOrder: 1,
position: "static",
pointerEvents: "auto",
};
}
function childSnapshot(frameId: string, backendNodeId: number) {
const strings = [frameId, "body", "button", "static", "auto", "pointer"];
return {
strings,
documents: [
{
frameId,
nodes: {
parentIndex: [-1, 0],
nodeName: [1, 2],
backendNodeId: [backendNodeId - 1, backendNodeId],
attributes: [[], []],
},
layout: {
nodeIndex: [0, 1],
styles: [
[3, 4, 4],
[3, 4, 5],
],
bounds: [
[0, 0, 300, 200],
[10, 20, 100, 40],
],
paintOrders: [0, 1],
},
},
],
};
}
describe("captureFrameData", () => {
it("captures and positions multiple OOPIF documents missing from the root snapshot", async () => {
const leftOwner = ownerNode(10, 50);
const rightOwner = ownerNode(20, 500);
const captured: CapturedViewModel = {
nodes: [leftOwner, rightOwner],
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
frameNodes: new Map([["main", [leftOwner, rightOwner]]]),
frameOwnerBackendNodeIds: new Map(),
rootFrameId: "main",
excludedBackendNodeIds: new Set(),
};
const sendToTarget = vi.fn(async (target, method) => {
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 300, clientHeight: 200, pageX: 0, pageY: 0 } };
}
if (method === "DOMSnapshot.enable" || method === "Accessibility.enable") return {};
if (method === "DOMSnapshot.captureSnapshot") {
return target.sessionId === "left-session"
? childSnapshot("left", 101)
: childSnapshot("right", 201);
}
if (method === "Accessibility.getFullAXTree") return { nodes: [] };
throw new Error(`unexpected ${method}`);
});
const cdp: CdpRunner = {
send: vi.fn(async (_tabId, method, params) => {
if (method === "Accessibility.enable") return {};
if (method === "Accessibility.getFullAXTree") return { nodes: [] };
if (method === "DOM.getBoxModel") {
const backendNodeId = (params as { backendNodeId?: number })?.backendNodeId;
const x = backendNodeId === 10 ? 50 : 500;
return { model: { content: [x, 100, x + 300, 100, x + 300, 300, x, 300] } };
}
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } };
}
throw new Error(`unexpected root ${method}`);
}) as CdpRunner["send"],
sendToTarget: sendToTarget as unknown as NonNullable<CdpRunner["sendToTarget"]>,
getFrameGraph: vi.fn(async () => ({
rootFrameId: "main",
frames: [
{ frameId: "main", target: { tabId: 4 } },
{
frameId: "left",
parentFrameId: "main",
ownerBackendNodeId: 10,
target: { tabId: 4, sessionId: "left-session" },
},
{
frameId: "right",
parentFrameId: "main",
ownerBackendNodeId: 20,
target: { tabId: 4, sessionId: "right-session" },
},
],
})),
};
const trees = await captureFrameData(cdp, 4, captured);
expect(trees.map((tree) => tree.frameId)).toEqual(["main", "left", "right"]);
expect(
captured.frameNodes?.get("left")?.find((node) => node.backendNodeId === 101)?.rect,
).toEqual({ x: 60, y: 120, w: 100, h: 40 });
expect(
captured.frameNodes?.get("right")?.find((node) => node.backendNodeId === 201)?.rect,
).toEqual({ x: 510, y: 120, w: 100, h: 40 });
expect(captured.frameOwnerBackendNodeIds).toEqual(
new Map([
["left", 10],
["right", 20],
]),
);
expect(captured.frameParentIds).toEqual(
new Map([
["left", "main"],
["right", "main"],
]),
);
});
});
@@ -0,0 +1,174 @@
import { describe, expect, it } from "vitest";
import type { CdpFrameGraph } from "@/browser-driver/frame-graph";
import type { CapturedNode, CapturedViewModel } from "../capture";
import { buildFrameDocuments, type FrameAxBatch, type FrameOwnedAxNode } from "../frame-document";
function domNode(frameId: string, backendNodeId: number): CapturedNode {
return {
backendNodeId,
parentBackendNodeId: null,
frameId,
tag: "div",
attrs: {},
rect: { x: 0, y: 0, w: 20, h: 20 },
paintOrder: 0,
position: "static",
pointerEvents: "auto",
};
}
function captured(frameNodes: Map<string, CapturedNode[]>): CapturedViewModel {
return {
nodes: frameNodes.get("main") ?? [],
viewport: { width: 800, height: 600 },
iframeNodes: new Map(),
frameNodes,
rootFrameId: "main",
excludedBackendNodeIds: new Set(),
};
}
describe("buildFrameDocuments", () => {
it("partitions mixed AX results by node ownership and cuts cross-frame parent edges", () => {
const graph: CdpFrameGraph = {
rootFrameId: "main",
frames: [
{ frameId: "main", target: { tabId: 4 } },
{ frameId: "child", parentFrameId: "main", target: { tabId: 4 } },
],
};
const batches: FrameAxBatch<FrameOwnedAxNode>[] = [
{
frame: graph.frames[0],
nodes: [
{ nodeId: "root", frameId: "main", backendDOMNodeId: 1, childIds: ["h"] },
{
nodeId: "h",
frameId: "child",
parentId: "root",
backendDOMNodeId: 100,
childIds: ["virtual"],
},
{ nodeId: "virtual", parentId: "h" },
],
},
];
const documents = buildFrameDocuments(
graph,
batches,
captured(
new Map([
["main", [domNode("main", 1)]],
["child", [domNode("child", 100)]],
]),
),
);
expect(documents.find((document) => document.frameId === "main")?.axNodes).toHaveLength(1);
const childNodes = documents.find((document) => document.frameId === "child")?.axNodes ?? [];
expect(childNodes.map((node) => node.nodeId)).toEqual(["h", "virtual"]);
expect(childNodes.find((node) => node.nodeId === "h")?.parentId).toBeUndefined();
expect(childNodes.find((node) => node.nodeId === "virtual")?.frameId).toBe("child");
});
it("deduplicates overlapping same-target AX results using the strongest ownership", () => {
const graph: CdpFrameGraph = {
rootFrameId: "main",
frames: [
{ frameId: "main", target: { tabId: 4 } },
{ frameId: "child", parentFrameId: "main", target: { tabId: 4 } },
],
};
const childNode = { nodeId: "shared", backendDOMNodeId: 100 };
const batches: FrameAxBatch<FrameOwnedAxNode>[] = [
{
frame: graph.frames[0],
nodes: [{ ...childNode, frameId: "child" }],
},
{
frame: graph.frames[1],
nodes: [childNode],
},
];
const documents = buildFrameDocuments(
graph,
batches,
captured(
new Map([
["main", [domNode("main", 1)]],
["child", [domNode("child", 100)]],
]),
),
);
expect(documents.find((document) => document.frameId === "main")?.axNodes).toEqual([]);
expect(documents.find((document) => document.frameId === "child")?.axNodes).toHaveLength(1);
});
it("uses target-scoped backend ownership when OOPIF backend ids collide", () => {
const graph: CdpFrameGraph = {
rootFrameId: "main",
frames: [
{ frameId: "main", target: { tabId: 4 } },
{
frameId: "left",
parentFrameId: "main",
target: { tabId: 4, sessionId: "left-session" },
},
{
frameId: "right",
parentFrameId: "main",
target: { tabId: 4, sessionId: "right-session" },
},
],
};
const batches: FrameAxBatch<FrameOwnedAxNode>[] = graph.frames.slice(1).map((frame) => ({
frame,
nodes: [{ nodeId: `${frame.frameId}-node`, backendDOMNodeId: 7 }],
}));
const documents = buildFrameDocuments(
graph,
batches,
captured(
new Map([
["main", []],
["left", [domNode("left", 7)]],
["right", [domNode("right", 7)]],
]),
),
);
expect(documents.find((document) => document.frameId === "left")?.axNodes[0]?.nodeId).toBe(
"left-node",
);
expect(documents.find((document) => document.frameId === "right")?.axNodes[0]?.nodeId).toBe(
"right-node",
);
});
it("uses captured frame parent identity instead of guessing from backend ids", () => {
const frameNodes = new Map([
["main", [domNode("main", 1)]],
["left", [domNode("left", 7)]],
["right", [domNode("right", 7)]],
["child", [domNode("child", 9)]],
]);
const capture = captured(frameNodes);
capture.frameOwnerBackendNodeIds = new Map([["child", 7]]);
capture.frameParentIds = new Map([["child", "left"]]);
const frames = ["main", "left", "right", "child"].map((frameId) => ({
frame: { frameId, target: { tabId: 4 } },
nodes: [],
}));
const documents = buildFrameDocuments(null, frames, capture);
expect(documents.find((document) => document.frameId === "child")).toMatchObject({
parentFrameId: "left",
ownerBackendNodeId: 7,
});
});
});
+138 -56
View File
@@ -7,6 +7,7 @@
import type { Rect, Viewport } from "@browser-skill/vom"; import type { Rect, Viewport } from "@browser-skill/vom";
import { evaluateHoverTrigger } from "@/lib/hover-trigger-policy"; import { evaluateHoverTrigger } from "@/lib/hover-trigger-policy";
import { isOverlayHostNode, OVERLAY_HOST_SELECTOR } from "../../lib/overlay-bridge"; import { isOverlayHostNode, OVERLAY_HOST_SELECTOR } from "../../lib/overlay-bridge";
import { childFrameProjection, type GeometryProjection, projectRectToViewport } from "../geometry";
import type { CdpRunner } from "../shared"; import type { CdpRunner } from "../shared";
const REQUESTED_STYLES = ["position", "pointer-events", "cursor"] as const; const REQUESTED_STYLES = ["position", "pointer-events", "cursor"] as const;
@@ -17,13 +18,14 @@ const STYLE_COL = Object.fromEntries(
export interface CapturedNode { export interface CapturedNode {
backendNodeId: number; backendNodeId: number;
parentBackendNodeId: number | null; parentBackendNodeId: number | null;
frameId?: string;
/** Owning iframe backend node id; `null` for the top-level document. */ /** Owning iframe backend node id; `null` for the top-level document. */
ownerFrameBackendNodeId?: number | null; ownerFrameBackendNodeId?: number | null;
tag: string; tag: string;
attrs: Record<string, string>; attrs: Record<string, string>;
/** Top-level viewport-relative CSS px, clipped to the owning frame viewport. */ /** Top-level viewport-relative CSS px, clipped to the owning frame viewport. */
rect: Rect | null; rect: Rect | null;
/** Frame-local viewport-relative CSS px before top-level translation/clipping. */ /** Frame-local viewport-relative CSS px before top-level projection. */
localRect?: Rect | null; localRect?: Rect | null;
paintOrder: number; paintOrder: number;
position: string; position: string;
@@ -57,6 +59,11 @@ export interface CapturedViewModel {
nodes: CapturedNode[]; nodes: CapturedNode[];
viewport: Viewport; viewport: Viewport;
iframeNodes: CapturedIframeNodes; iframeNodes: CapturedIframeNodes;
frameNodes?: Map<string, CapturedNode[]>;
frameOwnerBackendNodeIds?: Map<string, number>;
/** Explicit DOMSnapshot frame ancestry; never inferred from backend node ids. */
frameParentIds?: Map<string, string>;
rootFrameId?: string;
surfaceProbes?: CapturedSurfaceProbe[]; surfaceProbes?: CapturedSurfaceProbe[];
/** Backend node ids belonging to the agent overlay host + its shadow subtree. */ /** Backend node ids belonging to the agent overlay host + its shadow subtree. */
excludedBackendNodeIds: Set<number>; excludedBackendNodeIds: Set<number>;
@@ -92,7 +99,12 @@ interface SparseArray {
value: number[]; value: number[];
} }
interface RareBooleanData {
index: number[];
}
interface SnapshotDocument { interface SnapshotDocument {
frameId?: string | number;
scrollOffsetX?: number; scrollOffsetX?: number;
scrollOffsetY?: number; scrollOffsetY?: number;
nodes?: { nodes?: {
@@ -107,6 +119,10 @@ interface SnapshotDocument {
nodeValue?: number[]; nodeValue?: number[];
/** Maps node array index → index into `documents[]` for frame content. */ /** Maps node array index → index into `documents[]` for frame content. */
contentDocumentIndex?: SparseArray; contentDocumentIndex?: SparseArray;
inputValue?: SparseArray;
textValue?: SparseArray;
inputChecked?: RareBooleanData;
optionSelected?: RareBooleanData;
}; };
layout?: { layout?: {
nodeIndex?: number[]; nodeIndex?: number[];
@@ -116,6 +132,12 @@ interface SnapshotDocument {
}; };
} }
function snapshotFrameId(document: SnapshotDocument, strings: string[]): string | undefined {
if (typeof document.frameId === "string") return document.frameId || undefined;
if (typeof document.frameId === "number") return str(strings, document.frameId) || undefined;
return undefined;
}
interface SnapshotReply { interface SnapshotReply {
strings?: string[]; strings?: string[];
documents?: SnapshotDocument[]; documents?: SnapshotDocument[];
@@ -135,6 +157,25 @@ function isFormControlTag(tag: string): boolean {
return tag === "input" || tag === "textarea" || tag === "select"; return tag === "input" || tag === "textarea" || tag === "select";
} }
function isSensitiveFormControl(node: Pick<CapturedNode, "tag" | "attrs">): boolean {
return node.tag === "input" && (node.attrs.type ?? "").toLowerCase() === "password";
}
function snapshotFormState(
value: string | undefined,
defaultValue: string,
hasDefaultValue: boolean,
sensitive: boolean,
): CapturedNode["formState"] {
if (sensitive) {
if (value === undefined && !hasDefaultValue) return undefined;
return (value ?? defaultValue) === "" ? "empty" : "filled";
}
if (value === undefined) return undefined;
if (value === "") return "empty";
return value === defaultValue ? "default" : "filled";
}
const MAX_FORM_ENRICH_CONTROLS = 250; const MAX_FORM_ENRICH_CONTROLS = 250;
interface CapturedFormState { interface CapturedFormState {
@@ -166,9 +207,8 @@ function formStateBatchExpression(maxControls: number): string {
return { return {
state, state,
sensitive, sensitive,
defaultValue,
placeholder, placeholder,
...(sensitive ? {} : { value: rawValue }), ...(sensitive ? {} : { value: rawValue, defaultValue }),
}; };
}; };
const collect = (doc) => { const collect = (doc) => {
@@ -210,10 +250,14 @@ function applyFormStates(nodes: CapturedNode[], states: CapturedFormState[]): vo
index += 1; index += 1;
if (!state) continue; if (!state) continue;
node.formState = state.state; node.formState = state.state;
node.formDefaultValue = state.defaultValue ?? "";
node.formPlaceholder = state.placeholder ?? ""; node.formPlaceholder = state.placeholder ?? "";
if (!state.sensitive && state.value !== undefined) { if (state.sensitive || isSensitiveFormControl(node)) {
node.formValue = state.value; delete node.formValue;
delete node.formDefaultValue;
delete node.attrs.value;
} else {
node.formDefaultValue = state.defaultValue ?? "";
if (state.value !== undefined) node.formValue = state.value;
} }
} }
} }
@@ -272,9 +316,9 @@ interface ParseDocumentResult {
} }
interface FrameContext { interface FrameContext {
frameId?: string;
ownerFrameBackendNodeId: number | null; ownerFrameBackendNodeId: number | null;
originInTop: { x: number; y: number }; projection: GeometryProjection | null;
clipRectInTop: Rect;
scrollX: number; scrollX: number;
scrollY: number; scrollY: number;
} }
@@ -293,38 +337,6 @@ function devicePixelRatio(metrics: LayoutMetricsReply): number {
return dpr >= 1 ? dpr : 1; return dpr >= 1 ? dpr : 1;
} }
function viewportRect(viewport: Viewport): Rect {
return { x: 0, y: 0, w: Math.max(0, viewport.width), h: Math.max(0, viewport.height) };
}
function intersectRects(a: Rect, b: Rect): Rect | null {
const x1 = Math.max(a.x, b.x);
const y1 = Math.max(a.y, b.y);
const x2 = Math.min(a.x + a.w, b.x + b.w);
const y2 = Math.min(a.y + a.h, b.y + b.h);
if (x2 <= x1 || y2 <= y1) return null;
return { x: x1, y: y1, w: x2 - x1, h: y2 - y1 };
}
function rectInTop(localRect: Rect, context: FrameContext): Rect | null {
const translated = {
x: context.originInTop.x + localRect.x,
y: context.originInTop.y + localRect.y,
w: localRect.w,
h: localRect.h,
};
return intersectRects(translated, context.clipRectInTop);
}
function unclipRectInTop(localRect: Rect, context: FrameContext): Rect {
return {
x: context.originInTop.x + localRect.x,
y: context.originInTop.y + localRect.y,
w: localRect.w,
h: localRect.h,
};
}
function sparseIndexMap(sparse: SparseArray | undefined): Map<number, number> { function sparseIndexMap(sparse: SparseArray | undefined): Map<number, number> {
const out = new Map<number, number>(); const out = new Map<number, number>();
if (!sparse?.index || !sparse.value) return out; if (!sparse?.index || !sparse.value) return out;
@@ -805,6 +817,10 @@ function parseDocumentNodes(
const posCol = STYLE_COL.position; const posCol = STYLE_COL.position;
const peCol = STYLE_COL["pointer-events"]; const peCol = STYLE_COL["pointer-events"];
const cursorCol = STYLE_COL.cursor; const cursorCol = STYLE_COL.cursor;
const inputValues = sparseIndexMap(dn.inputValue);
const textValues = sparseIndexMap(dn.textValue);
const checkedInputs = new Set(dn.inputChecked?.index ?? []);
const selectedOptions = new Set(dn.optionSelected?.index ?? []);
// Collect visible text from #text child nodes so element CapturedNodes // Collect visible text from #text child nodes so element CapturedNodes
// carry a textContent value usable as a button/link label fallback. // carry a textContent value usable as a button/link label fallback.
@@ -855,7 +871,10 @@ function parseDocumentNodes(
w: b[2] / dpr, w: b[2] / dpr,
h: b[3] / dpr, h: b[3] / dpr,
}; };
rect = rectInTop(localRect, context); const bounds = context.projection
? projectRectToViewport(localRect, context.projection)
: null;
rect = bounds ? { x: bounds.x, y: bounds.y, w: bounds.width, h: bounds.height } : null;
} }
paintOrder = dl?.paintOrders?.[li] ?? 0; paintOrder = dl?.paintOrders?.[li] ?? 0;
const styleRow = dl?.styles?.[li] ?? []; const styleRow = dl?.styles?.[li] ?? [];
@@ -870,9 +889,24 @@ function parseDocumentNodes(
const textContent = nodeTextContent.get(n); const textContent = nodeTextContent.get(n);
const rawFormValueIndex = tag === "textarea" ? textValues.get(n) : inputValues.get(n);
const rawFormValue =
rawFormValueIndex !== undefined ? str(strings, rawFormValueIndex) : undefined;
const sensitive = isSensitiveFormControl({ tag, attrs });
const formDefaultValue = attrs.value ?? "";
const formValue = sensitive ? undefined : rawFormValue;
const formState = snapshotFormState(
rawFormValue,
formDefaultValue,
Object.prototype.hasOwnProperty.call(attrs, "value"),
sensitive,
);
if (sensitive) delete attrs.value;
nodes.push({ nodes.push({
backendNodeId, backendNodeId,
parentBackendNodeId, parentBackendNodeId,
...(context.frameId ? { frameId: context.frameId } : {}),
ownerFrameBackendNodeId: context.ownerFrameBackendNodeId, ownerFrameBackendNodeId: context.ownerFrameBackendNodeId,
tag, tag,
attrs, attrs,
@@ -883,6 +917,16 @@ function parseDocumentNodes(
pointerEvents, pointerEvents,
cursor, cursor,
textContent, textContent,
...(formValue !== undefined ? { formValue } : {}),
...(tag === "input" || tag === "textarea"
? {
formPlaceholder: attrs.placeholder ?? "",
...(!sensitive ? { formDefaultValue } : {}),
...(formState ? { formState } : {}),
}
: {}),
...(checkedInputs.has(n) ? { formValue: "true", formState: "filled" } : {}),
...(selectedOptions.has(n) ? { formValue: attrs.value ?? textContent ?? "" } : {}),
}); });
} }
return { nodes, excludedBackendNodeIds }; return { nodes, excludedBackendNodeIds };
@@ -890,6 +934,8 @@ function parseDocumentNodes(
interface ParseFrameDocumentsResult { interface ParseFrameDocumentsResult {
iframeNodes: CapturedIframeNodes; iframeNodes: CapturedIframeNodes;
frameOwnerBackendNodeIds: Map<string, number>;
frameParentIds: Map<string, string>;
excludedBackendNodeIds: Set<number>; excludedBackendNodeIds: Set<number>;
} }
@@ -903,10 +949,14 @@ function parseChildFrameDocuments(
visited = new Set<number>(), visited = new Set<number>(),
): ParseFrameDocumentsResult { ): ParseFrameDocumentsResult {
const iframeNodes: CapturedIframeNodes = new Map(); const iframeNodes: CapturedIframeNodes = new Map();
const frameOwnerBackendNodeIds = new Map<string, number>();
const frameParentIds = new Map<string, string>();
const excludedBackendNodeIds = new Set<number>(); const excludedBackendNodeIds = new Set<number>();
const parentDoc = documents[parentDocIndex]; const parentDoc = documents[parentDocIndex];
const cdi = sparseIndexMap(parentDoc?.nodes?.contentDocumentIndex); const cdi = sparseIndexMap(parentDoc?.nodes?.contentDocumentIndex);
if (cdi.size === 0) return { iframeNodes, excludedBackendNodeIds }; if (cdi.size === 0) {
return { iframeNodes, frameOwnerBackendNodeIds, frameParentIds, excludedBackendNodeIds };
}
const parentBackendIds = parentDoc?.nodes?.backendNodeId ?? []; const parentBackendIds = parentDoc?.nodes?.backendNodeId ?? [];
const parentNodeByBackendId = new Map(parentNodes.map((node) => [node.backendNodeId, node])); const parentNodeByBackendId = new Map(parentNodes.map((node) => [node.backendNodeId, node]));
@@ -918,19 +968,15 @@ function parseChildFrameDocuments(
const iframeBackendId = parentBackendIds[nodeArrayIdx]; const iframeBackendId = parentBackendIds[nodeArrayIdx];
if (iframeBackendId === undefined) continue; if (iframeBackendId === undefined) continue;
const iframeNode = parentNodeByBackendId.get(iframeBackendId); const iframeNode = parentNodeByBackendId.get(iframeBackendId);
if (!iframeNode?.localRect) continue; const projection =
parentContext.projection && iframeNode?.localRect
const iframeRectInTop = unclipRectInTop(iframeNode.localRect, parentContext); ? childFrameProjection(parentContext.projection, iframeNode.localRect)
const childClip = intersectRects(iframeRectInTop, parentContext.clipRectInTop); : null;
if (!childClip) {
iframeNodes.set(iframeBackendId, []);
continue;
}
const childContext: FrameContext = { const childContext: FrameContext = {
frameId: snapshotFrameId(childDoc, strings),
ownerFrameBackendNodeId: iframeBackendId, ownerFrameBackendNodeId: iframeBackendId,
originInTop: { x: iframeRectInTop.x, y: iframeRectInTop.y }, projection,
clipRectInTop: childClip,
scrollX: childDoc.scrollOffsetX ?? 0, scrollX: childDoc.scrollOffsetX ?? 0,
scrollY: childDoc.scrollOffsetY ?? 0, scrollY: childDoc.scrollOffsetY ?? 0,
}; };
@@ -938,6 +984,10 @@ function parseChildFrameDocuments(
nextVisited.add(childDocIndex); nextVisited.add(childDocIndex);
const parsed = parseDocumentNodes(childDoc, strings, dpr, childContext); const parsed = parseDocumentNodes(childDoc, strings, dpr, childContext);
iframeNodes.set(iframeBackendId, parsed.nodes); iframeNodes.set(iframeBackendId, parsed.nodes);
if (childContext.frameId) {
frameOwnerBackendNodeIds.set(childContext.frameId, iframeBackendId);
if (parentContext.frameId) frameParentIds.set(childContext.frameId, parentContext.frameId);
}
for (const id of parsed.excludedBackendNodeIds) excludedBackendNodeIds.add(id); for (const id of parsed.excludedBackendNodeIds) excludedBackendNodeIds.add(id);
const nested = parseChildFrameDocuments( const nested = parseChildFrameDocuments(
@@ -952,10 +1002,16 @@ function parseChildFrameDocuments(
for (const [nestedFrameId, nestedNodes] of nested.iframeNodes) { for (const [nestedFrameId, nestedNodes] of nested.iframeNodes) {
iframeNodes.set(nestedFrameId, nestedNodes); iframeNodes.set(nestedFrameId, nestedNodes);
} }
for (const [frameId, ownerBackendNodeId] of nested.frameOwnerBackendNodeIds) {
frameOwnerBackendNodeIds.set(frameId, ownerBackendNodeId);
}
for (const [frameId, parentFrameId] of nested.frameParentIds) {
frameParentIds.set(frameId, parentFrameId);
}
for (const id of nested.excludedBackendNodeIds) excludedBackendNodeIds.add(id); for (const id of nested.excludedBackendNodeIds) excludedBackendNodeIds.add(id);
} }
return { iframeNodes, excludedBackendNodeIds }; return { iframeNodes, frameOwnerBackendNodeIds, frameParentIds, excludedBackendNodeIds };
} }
export async function captureViewModel( export async function captureViewModel(
@@ -998,9 +1054,13 @@ export async function captureViewModel(
} }
const topContext: FrameContext = { const topContext: FrameContext = {
frameId: snapshotFrameId(doc0, strings),
ownerFrameBackendNodeId: null, ownerFrameBackendNodeId: null,
originInTop: { x: 0, y: 0 }, projection: {
clipRectInTop: viewportRect(viewport), sourceClips: [],
edges: [],
topViewport: viewport,
},
scrollX, scrollX,
scrollY, scrollY,
}; };
@@ -1022,5 +1082,27 @@ export async function captureViewModel(
: []; : [];
throwIfAborted(options.signal); throwIfAborted(options.signal);
return { nodes, viewport, iframeNodes, surfaceProbes, excludedBackendNodeIds }; const frameNodes = new Map<string, CapturedNode[]>();
if (topContext.frameId) frameNodes.set(topContext.frameId, nodes);
for (const iframeFrameNodes of iframeNodes.values()) {
const frameId = iframeFrameNodes.find((node) => node.frameId)?.frameId;
if (frameId) frameNodes.set(frameId, iframeFrameNodes);
}
for (const [frameId, ownerBackendNodeId] of frameParsed.frameOwnerBackendNodeIds) {
if (!frameNodes.has(frameId)) {
frameNodes.set(frameId, iframeNodes.get(ownerBackendNodeId) ?? []);
}
}
return {
nodes,
viewport,
iframeNodes,
frameNodes,
frameOwnerBackendNodeIds: frameParsed.frameOwnerBackendNodeIds,
frameParentIds: frameParsed.frameParentIds,
...(topContext.frameId ? { rootFrameId: topContext.frameId } : {}),
surfaceProbes,
excludedBackendNodeIds,
};
} }
@@ -0,0 +1,179 @@
import type { CdpFrame, CdpFrameGraph } from "@/browser-driver/frame-graph";
import { resolveFrameProjection } from "../frame-geometry";
import { type GeometryProjection, projectRectToViewport } from "../geometry";
import { type CdpRunner, cdpRunnerForTarget, sendToCdpTarget } from "../shared";
import { type CapturedNode, type CapturedViewModel, captureViewModel } from "./capture";
import {
buildFrameDocuments,
type FrameAxBatch,
type FrameDocument,
type FrameOwnedAxNode,
} from "./frame-document";
export type FrameAxNode = FrameOwnedAxNode;
export type CapturedFrameDocument<T extends FrameAxNode> = FrameDocument<T>;
function throwIfAborted(signal?: AbortSignal): void {
if (signal?.aborted) throw new DOMException("observation aborted", "AbortError");
}
function isAbortError(err: unknown): boolean {
return err instanceof Error && err.name === "AbortError";
}
async function discoverFrameGraph(cdp: CdpRunner, tabId: number): Promise<CdpFrameGraph | null> {
if (!cdp.getFrameGraph) return null;
try {
return await cdp.getFrameGraph(tabId);
} catch (err) {
console.debug("[bsk observation] frame graph capture failed", err);
return null;
}
}
function transformFrameNodes(
nodes: CapturedNode[],
projection: GeometryProjection,
): CapturedNode[] {
return nodes.map((node) => ({
...node,
rect: (() => {
const bounds = projectRectToViewport(node.rect, projection);
return bounds ? { x: bounds.x, y: bounds.y, w: bounds.width, h: bounds.height } : null;
})(),
}));
}
function frameOwnerNode(captured: CapturedViewModel, frame: CdpFrame): CapturedNode | undefined {
if (frame.ownerBackendNodeId === undefined) return undefined;
const parentNodes = frame.parentFrameId
? captured.frameNodes?.get(frame.parentFrameId)
: captured.nodes;
return parentNodes?.find((node) => node.backendNodeId === frame.ownerBackendNodeId);
}
async function captureMissingFrameDocuments(
cdp: CdpRunner,
tabId: number,
graph: CdpFrameGraph,
captured: CapturedViewModel,
signal?: AbortSignal,
): Promise<void> {
if (!captured.frameNodes) captured.frameNodes = new Map();
if (!captured.frameOwnerBackendNodeIds) captured.frameOwnerBackendNodeIds = new Map();
if (!captured.frameParentIds) captured.frameParentIds = new Map();
for (const frame of graph.frames) {
if (!frame.target.sessionId || captured.frameNodes.has(frame.frameId)) continue;
const owner = frameOwnerNode(captured, frame);
if (!owner?.rect || frame.ownerBackendNodeId === undefined) continue;
try {
const child = await captureViewModel(cdpRunnerForTarget(cdp, frame.target), tabId, {
signal,
});
const projection = await resolveFrameProjection(cdp, graph, frame.frameId);
if (!projection) continue;
const rootFrameId = child.rootFrameId ?? frame.frameId;
const childFrames = child.frameNodes ?? new Map<string, CapturedNode[]>();
if (!childFrames.has(rootFrameId)) childFrames.set(rootFrameId, child.nodes);
if (rootFrameId !== frame.frameId && !childFrames.has(frame.frameId)) {
childFrames.set(
frame.frameId,
child.nodes.map((node) => ({ ...node, frameId: frame.frameId })),
);
}
for (const [childFrameId, nodes] of childFrames) {
captured.frameNodes.set(childFrameId, transformFrameNodes(nodes, projection));
}
captured.iframeNodes.set(
frame.ownerBackendNodeId,
captured.frameNodes.get(frame.frameId) ?? [],
);
captured.frameOwnerBackendNodeIds.set(frame.frameId, frame.ownerBackendNodeId);
if (frame.parentFrameId) captured.frameParentIds.set(frame.frameId, frame.parentFrameId);
for (const [childFrameId, ownerBackendNodeId] of child.frameOwnerBackendNodeIds ?? []) {
captured.frameOwnerBackendNodeIds.set(childFrameId, ownerBackendNodeId);
}
for (const [childFrameId, parentFrameId] of child.frameParentIds ?? []) {
captured.frameParentIds.set(childFrameId, parentFrameId);
}
} catch (err) {
if (isAbortError(err)) throw err;
console.debug("[bsk observation] child frame DOM capture failed", {
frameId: frame.frameId,
err,
});
}
}
}
async function captureAxTrees<T extends FrameAxNode>(
cdp: CdpRunner,
tabId: number,
discoveredFrames: CdpFrame[],
captured: CapturedViewModel,
signal?: AbortSignal,
): Promise<FrameAxBatch<T>[]> {
let frames = [...discoveredFrames];
const capturedFrameIds = [...(captured.frameNodes?.keys() ?? [])];
if (frames.length === 0) {
const rootFrameId = captured.rootFrameId ?? capturedFrameIds[0] ?? "root";
frames = [{ frameId: rootFrameId, target: { tabId } }];
}
for (const frameId of capturedFrameIds) {
if (!frames.some((frame) => frame.frameId === frameId)) {
frames.push({ frameId, target: { tabId } });
}
}
let firstFailure: unknown;
const trees = await Promise.all(
frames.map(async (frame): Promise<FrameAxBatch<T>> => {
try {
throwIfAborted(signal);
await sendToCdpTarget(cdp, frame.target, "Accessibility.enable", {});
const result = await sendToCdpTarget<{ nodes?: T[] }>(
cdp,
frame.target,
"Accessibility.getFullAXTree",
frame.frameId === "root" ? {} : { frameId: frame.frameId },
);
throwIfAborted(signal);
return {
frame,
nodes: result.nodes ?? [],
};
} catch (err) {
if (isAbortError(err)) throw err;
firstFailure ??= err;
console.debug("[bsk observation] frame AX capture failed", {
frameId: frame.frameId,
err,
});
return { frame, nodes: [] };
}
}),
);
const capturedNodeCount = [...captured.iframeNodes.values()].reduce(
(count, nodes) => count + nodes.length,
captured.nodes.length,
);
if (firstFailure && capturedNodeCount === 0 && trees.every((tree) => tree.nodes.length === 0)) {
throw firstFailure;
}
return trees;
}
export async function captureFrameData<T extends FrameAxNode>(
cdp: CdpRunner,
tabId: number,
captured: CapturedViewModel,
signal?: AbortSignal,
): Promise<CapturedFrameDocument<T>[]> {
const graph = await discoverFrameGraph(cdp, tabId);
const frames = graph?.frames ?? [];
if (graph) await captureMissingFrameDocuments(cdp, tabId, graph, captured, signal);
throwIfAborted(signal);
const batches = await captureAxTrees<T>(cdp, tabId, frames, captured, signal);
return buildFrameDocuments(graph, batches, captured);
}
@@ -0,0 +1,170 @@
import {
type CdpFrame,
type CdpFrameGraph,
type CdpTarget,
cdpTargetKey,
} from "@/browser-driver/frame-graph";
import type { CapturedNode, CapturedViewModel } from "./capture";
export interface FrameOwnedAxNode {
nodeId: string;
frameId?: string;
parentId?: string;
childIds?: string[];
backendDOMNodeId?: number;
}
export interface FrameAxBatch<T extends FrameOwnedAxNode> {
frame: CdpFrame;
nodes: T[];
}
export interface FrameDocument<T extends FrameOwnedAxNode> extends CdpFrame {
contextScopeId: string;
axNodes: T[];
domNodes: CapturedNode[];
}
interface Ownership {
frameId: string;
strength: number;
}
interface OwnedCandidate<T> {
node: T;
target: CdpTarget;
ownership: Ownership;
}
function targetNodeKey(target: CdpTarget, nodeId: string): string {
return `${cdpTargetKey(target)}:${nodeId}`;
}
function targetBackendKey(target: CdpTarget, backendNodeId: number): string {
return `${cdpTargetKey(target)}:${backendNodeId}`;
}
function frameList<T extends FrameOwnedAxNode>(
graph: CdpFrameGraph | null,
batches: FrameAxBatch<T>[],
captured: CapturedViewModel,
): CdpFrame[] {
const frames = new Map<string, CdpFrame>();
for (const frame of graph?.frames ?? []) frames.set(frame.frameId, frame);
for (const batch of batches) {
if (!frames.has(batch.frame.frameId)) frames.set(batch.frame.frameId, batch.frame);
}
for (const frameId of captured.frameNodes?.keys() ?? []) {
if (!frames.has(frameId))
frames.set(frameId, { frameId, target: { tabId: batches[0]?.frame.target.tabId ?? 0 } });
}
return [...frames.values()].map((frame) => {
const ownerBackendNodeId =
frame.ownerBackendNodeId ?? captured.frameOwnerBackendNodeIds?.get(frame.frameId);
const parentFrameId = frame.parentFrameId ?? captured.frameParentIds?.get(frame.frameId);
return {
...frame,
...(ownerBackendNodeId !== undefined ? { ownerBackendNodeId } : {}),
...(parentFrameId ? { parentFrameId } : {}),
};
});
}
export function buildFrameDocuments<T extends FrameOwnedAxNode>(
graph: CdpFrameGraph | null,
batches: FrameAxBatch<T>[],
captured: CapturedViewModel,
): FrameDocument<T>[] {
const frames = frameList(graph, batches, captured);
const frameById = new Map(frames.map((frame) => [frame.frameId, frame]));
const backendOwner = new Map<string, string>();
for (const frame of frames) {
for (const node of captured.frameNodes?.get(frame.frameId) ?? []) {
backendOwner.set(targetBackendKey(frame.target, node.backendNodeId), frame.frameId);
}
}
const candidates = new Map<string, OwnedCandidate<T>>();
for (const batch of batches) {
const nodeById = new Map(batch.nodes.map((node) => [node.nodeId, node]));
const ownershipByNodeId = new Map<string, Ownership>();
const resolving = new Set<string>();
const resolveOwnership = (node: T): Ownership => {
const cached = ownershipByNodeId.get(node.nodeId);
if (cached) return cached;
if (node.frameId && frameById.has(node.frameId)) {
const ownership = { frameId: node.frameId, strength: 4 };
ownershipByNodeId.set(node.nodeId, ownership);
return ownership;
}
if (typeof node.backendDOMNodeId === "number") {
const frameId = backendOwner.get(
targetBackendKey(batch.frame.target, node.backendDOMNodeId),
);
if (frameId) {
const ownership = { frameId, strength: 3 };
ownershipByNodeId.set(node.nodeId, ownership);
return ownership;
}
}
if (node.parentId && !resolving.has(node.nodeId)) {
const parent = nodeById.get(node.parentId);
if (parent) {
resolving.add(node.nodeId);
const inherited = resolveOwnership(parent);
resolving.delete(node.nodeId);
const ownership = {
frameId: inherited.frameId,
strength: Math.min(2, inherited.strength),
};
ownershipByNodeId.set(node.nodeId, ownership);
return ownership;
}
}
const ownership = { frameId: batch.frame.frameId, strength: 1 };
ownershipByNodeId.set(node.nodeId, ownership);
return ownership;
};
for (const node of batch.nodes) {
const ownership = resolveOwnership(node);
const key = targetNodeKey(batch.frame.target, node.nodeId);
const existing = candidates.get(key);
if (!existing || ownership.strength > existing.ownership.strength) {
candidates.set(key, { node, target: batch.frame.target, ownership });
}
}
}
const ownershipByTargetNode = new Map(
[...candidates].map(([key, candidate]) => [key, candidate.ownership.frameId]),
);
const axNodesByFrame = new Map<string, T[]>();
for (const candidate of candidates.values()) {
const { node, target, ownership } = candidate;
const parentFrameId = node.parentId
? ownershipByTargetNode.get(targetNodeKey(target, node.parentId))
: undefined;
const childIds = node.childIds?.filter(
(childId) => ownershipByTargetNode.get(targetNodeKey(target, childId)) === ownership.frameId,
);
const ownedNode = {
...node,
frameId: ownership.frameId,
...(parentFrameId === ownership.frameId
? { parentId: node.parentId }
: { parentId: undefined }),
...(childIds ? { childIds } : {}),
};
const nodes = axNodesByFrame.get(ownership.frameId) ?? [];
nodes.push(ownedNode);
axNodesByFrame.set(ownership.frameId, nodes);
}
return frames.map((frame) => ({
...frame,
contextScopeId: frame.frameId,
axNodes: axNodesByFrame.get(frame.frameId) ?? [],
domNodes: captured.frameNodes?.get(frame.frameId) ?? [],
}));
}
+2 -4
View File
@@ -25,10 +25,8 @@ export default defineConfig({
name: "BrowserSkill", name: "BrowserSkill",
description: description:
"Let AI agents use your logged-in browser in a separate Agent Window—without interrupting your work. Powered by the bsk CLI.", "Let AI agents use your logged-in browser in a separate Agent Window—without interrupting your work. Powered by the bsk CLI.",
// Chrome 116 is the first version where WebSocket activity resets the // Flat debugger sessions are required to address out-of-process iframes.
// service-worker idle timer, which the 20s heartbeat relies on to minimum_chrome_version: "125",
// keep the daemon link alive.
minimum_chrome_version: "116",
permissions: [ permissions: [
"alarms", "alarms",
"debugger", "debugger",
+37 -11
View File
@@ -18,6 +18,12 @@ use crate::daemon::state::PROTOCOL_VERSION;
const EXTENSION_STORE_URL: &str = const EXTENSION_STORE_URL: &str =
"https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi"; "https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi";
/// Edge Add-ons listing for the browser-skill extension.
const EXTENSION_STORE_URL_EDGE: &str = "https://microsoftedge.microsoft.com/addons/detail/browserskill/emacgiaaaiojkkpkddmmdfhmokgmnikg";
/// Store listings highlighted in repair hints, in the order they appear.
const EXTENSION_STORE_URLS: [&str; 2] = [EXTENSION_STORE_URL, EXTENSION_STORE_URL_EDGE];
/// Status of a single doctor check. `Ok` / `Fail` are the legacy two /// Status of a single doctor check. `Ok` / `Fail` are the legacy two
/// states; `NotApplicable` (review M2) is reported as "N/A" in human /// states; `NotApplicable` (review M2) is reported as "N/A" in human
/// output and as `"status": "na"` in `--json` output, so a check that /// output and as `"status": "na"` in `--json` output, so a check that
@@ -420,7 +426,10 @@ fn check_extension_connected(status: Option<&StatusResult>) -> CheckResult {
CheckResult::fail( CheckResult::fail(
name, name,
"0 browsers connected", "0 browsers connected",
format!("install the extension from {EXTENSION_STORE_URL} and load it in Chromium"), format!(
"install the extension from {EXTENSION_STORE_URL} (Chrome) \
or {EXTENSION_STORE_URL_EDGE} (Edge) and load it in the browser"
),
) )
} }
} }
@@ -428,17 +437,14 @@ fn check_extension_connected(status: Option<&StatusResult>) -> CheckResult {
/// Highlight known URLs in repair hints for terminal output. Plain /// Highlight known URLs in repair hints for terminal output. Plain
/// text is preserved in `--json` and in stored [`CheckResult::hint`]. /// text is preserved in `--json` and in stored [`CheckResult::hint`].
fn style_hint(hint: &str) -> String { fn style_hint(hint: &str) -> String {
if !hint.contains(EXTENSION_STORE_URL) { let mut styled = hint.to_string();
return hint.to_string(); for url in EXTENSION_STORE_URLS {
if !styled.contains(url) {
continue;
}
styled = styled.replace(url, &style(url).cyan().bold().underlined().to_string());
} }
hint.replace( styled
EXTENSION_STORE_URL,
&style(EXTENSION_STORE_URL)
.cyan()
.bold()
.underlined()
.to_string(),
)
} }
fn render_human(checks: &[CheckResult]) { fn render_human(checks: &[CheckResult]) {
@@ -504,6 +510,26 @@ mod m2_tests {
hint.contains(EXTENSION_STORE_URL), hint.contains(EXTENSION_STORE_URL),
"hint should include Chrome Web Store URL: {hint}" "hint should include Chrome Web Store URL: {hint}"
); );
assert!(
hint.contains(EXTENSION_STORE_URL_EDGE),
"hint should include Edge Add-ons URL: {hint}"
);
}
#[test]
fn style_hint_preserves_every_store_url() {
let hint = check_extension_connected(Some(&fake_status(Vec::new(), Vec::new())))
.hint
.expect("extension disconnected should include a hint");
let styled = style_hint(&hint);
// Whether or not the terminal accepts colors, styling must never drop or
// mangle a URL the user has to click.
for url in EXTENSION_STORE_URLS {
assert!(
styled.contains(url),
"styled hint should still contain {url}: {styled}"
);
}
} }
#[test] #[test]
+8 -7
View File
@@ -6,7 +6,7 @@ A [DeepSeek Harness](https://github.com/deepseek-ai/deepseek-harness) (dsh) tool
[BrowserSkill](https://github.com/Tencent/BrowserSkill) (`bsk`) browser automation to the model. [BrowserSkill](https://github.com/Tencent/BrowserSkill) (`bsk`) browser automation to the model.
Each tool maps to one `bsk <cmd> --json` invocation: the plugin spawns the bsk CLI, parses its Each tool maps to one `bsk <cmd> --json` invocation: the plugin spawns the bsk CLI, parses its
structured JSON output, and returns a canonical typed value. The bsk daemon, browser, and Chrome structured JSON output, and returns a canonical typed value. The bsk daemon, browser, and browser
extension keep owning the actual browser control — this package is a thin, well-typed bridge. extension keep owning the actual browser control — this package is a thin, well-typed bridge.
## Tools ## Tools
@@ -64,9 +64,10 @@ dsh plugin --profile <name> add @wxg-prc-cpg/browser-skill-dsh-plugin
dsh --profile <name> dsh --profile <name>
``` ```
Prerequisite: the `bsk` CLI must be installed and on `PATH`, and the BrowserSkill Chrome extension Prerequisite: the `bsk` CLI must be installed and on `PATH`, and the BrowserSkill browser extension
must be connected — see the [BrowserSkill README](https://github.com/Tencent/BrowserSkill). When bsk (Chrome or Edge) must be connected — see the
is missing, tool calls fail with install guidance instead of a bare spawn error. [BrowserSkill README](https://github.com/Tencent/BrowserSkill). When bsk is missing, tool calls fail
with install guidance instead of a bare spawn error.
## Configuration ## Configuration
@@ -76,7 +77,7 @@ All fields are optional and validated through the plugin's Schemastery `Config`:
# cordis.patch.yml override example # cordis.patch.yml override example
- insert: - insert:
- id: browserskill - id: browserskill
name: dsh-plugin-browserskill name: "@wxg-prc-cpg/browser-skill-dsh-plugin"
config: config:
bskPath: bsk # path to the bsk binary (default: resolve from PATH) bskPath: bsk # path to the bsk binary (default: resolve from PATH)
defaultTimeoutMs: 120000 defaultTimeoutMs: 120000
@@ -186,8 +187,8 @@ git tag dsh-plugin-v0.1.0
git push origin dsh-plugin-v0.1.0 git push origin dsh-plugin-v0.1.0
``` ```
Or run the workflow from the Actions tab (`workflow_dispatch`). The job reads Or run the workflow from the Actions tab (`workflow_dispatch`). The job reads the
`NPM_TOKEN` from the GitHub Environment of the same name. `NPM_TOKEN` secret from the `npm-publish` GitHub Environment.
## License ## License
@@ -1,3 +1,5 @@
- insert: - insert:
- id: browserskill - id: browserskill
name: dsh-plugin-browserskill # Must be the published package name — the loader imports this specifier.
# Quoted because YAML reserves a leading '@' in plain scalars.
name: "@wxg-prc-cpg/browser-skill-dsh-plugin"
@@ -1,7 +1,7 @@
{ {
"name": "@wxg-prc-cpg/browser-skill-dsh-plugin", "name": "@wxg-prc-cpg/browser-skill-dsh-plugin",
"description": "DeepSeek Harness tool plugin that exposes BrowserSkill (bsk) browser automation to the model", "description": "DeepSeek Harness tool plugin that exposes BrowserSkill (bsk) browser automation to the model",
"version": "0.1.0", "version": "0.1.1",
"license": "MIT", "license": "MIT",
"publishConfig": { "publishConfig": {
"access": "public", "access": "public",
@@ -20,7 +20,7 @@ export interface EventSourceLike {
export interface ObservationClientDeps { export interface ObservationClientDeps {
fetchFn: ( fetchFn: (
url: string, url: string,
init?: { method?: string; body?: string }, init?: { method?: string; body?: string; headers?: Record<string, string> },
) => Promise<{ ) => Promise<{
ok: boolean; ok: boolean;
json(): Promise<unknown>; json(): Promise<unknown>;
@@ -282,6 +282,7 @@ export class ObservationClientStore {
try { try {
const res = await this.deps.fetchFn(INTERRUPT_URL, { const res = await this.deps.fetchFn(INTERRUPT_URL, {
method: "POST", method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify(sessionId === undefined ? {} : { sessionId }), body: JSON.stringify(sessionId === undefined ? {} : { sessionId }),
}); });
if (!res.ok) return false; if (!res.ok) return false;
@@ -1,16 +1,18 @@
/** /**
* ObservationService: per-owned-session live observation state for the PiP * ObservationService: per-owned-session live observation state for the PiP
* overlay — current action, page url, and a breathing thumbnail carried by * overlay — current action, page url, and a breathing thumbnail identified by
* attachment reference. Strict ownership boundary applies throughout: only * an ephemeral frame id. Frames live in a per-session 2-slot ring in process
* sessions in the plugin's SessionRegistry (owned by construction) ever get * memory (never the durable attachment store). Strict ownership boundary
* an observation entry, and interrupt/kill paths can only reach children this * applies throughout: only sessions in the plugin's SessionRegistry (owned by
* plugin spawned. * construction) ever get an observation entry, and interrupt/kill paths can
* only reach children this plugin spawned.
* *
* Observation traffic isolation: thumbnail captures run through the runner * Observation traffic isolation: thumbnail captures run through the runner
* directly (never through the tool-level instrumentation), emit no action * directly (never through the tool-level instrumentation), emit no action
* events, and never move the registry's current pointer. * events, and never move the registry's current pointer.
*/ */
import { createHash } from "node:crypto";
import { readFile, unlink } from "node:fs/promises"; import { readFile, unlink } from "node:fs/promises";
import { tmpdir } from "node:os"; import { tmpdir } from "node:os";
import { join } from "node:path"; import { join } from "node:path";
@@ -28,7 +30,7 @@ export interface SessionObservation {
action: string; action: string;
/** Epoch ms when the current action started (elapsed time is client-side). */ /** Epoch ms when the current action started (elapsed time is client-side). */
since: number; since: number;
/** Latest thumbnail, by attachment-store reference (never bytes on the wire). */ /** Latest thumbnail id (ephemeral, plugin-owned; never bytes on the wire). */
thumbnailAttachmentId?: string; thumbnailAttachmentId?: string;
/** Summary of the most recent failed action (drives the red status dot). */ /** Summary of the most recent failed action (drives the red status dot). */
lastError?: string; lastError?: string;
@@ -52,28 +54,6 @@ export interface ObservationOptions {
idleIntervalMs: number; idleIntervalMs: number;
} }
/** Structural view of the optional attachment seam (absent in headless compositions). */
interface AttachmentLike {
saveImage(input: {
data: Uint8Array;
mediaType: string;
name?: string;
}): Promise<ImageAttachmentRefLike>;
readImage(
ref: ImageAttachmentRefLike,
): Promise<{ data: Uint8Array; attachment: ImageAttachmentRefLike }>;
}
/** The attachment reference fields the store verifies reads against. */
export interface ImageAttachmentRefLike {
attachmentId: unknown;
mediaType: string;
bytes: number;
width: number;
height: number;
name?: string;
}
/** Injectable clock/scheduler bits for tests. */ /** Injectable clock/scheduler bits for tests. */
export interface ObservationScheduler { export interface ObservationScheduler {
setTimeout: (fn: () => void, ms: number) => unknown; setTimeout: (fn: () => void, ms: number) => unknown;
@@ -94,6 +74,8 @@ const DEFAULT_SCHEDULER: ObservationScheduler = {
/** Max consecutive capture failures before a session drops to the idle cadence. */ /** Max consecutive capture failures before a session drops to the idle cadence. */
const FAILURE_BACKOFF_THRESHOLD = 3; const FAILURE_BACKOFF_THRESHOLD = 3;
/** Current + previous frame, so an in-flight HTTP fetch of the old id still lands. */
const FRAME_RING_SIZE = 2;
export class ObservationService { export class ObservationService {
private readonly observations = new Map<string, SessionObservation>(); private readonly observations = new Map<string, SessionObservation>();
@@ -102,8 +84,14 @@ export class ObservationService {
private readonly captureInFlight = new Set<string>(); private readonly captureInFlight = new Set<string>();
private readonly captureFailures = new Map<string, number>(); private readonly captureFailures = new Map<string, number>();
private readonly lastActivity = new Map<string, number>(); private readonly lastActivity = new Map<string, number>();
/** attachmentId → full reference (the store requires it for verified reads). */ /** Ephemeral frame id → PNG bytes. Only the live ring members are present. */
private readonly thumbRefs = new Map<string, ImageAttachmentRefLike>(); private readonly frames = new Map<string, { data: Uint8Array; mediaType: string }>();
/** sessionId → oldest-first frame ids, length ≤ FRAME_RING_SIZE. */
private readonly rings = new Map<string, string[]>();
/** sessionId → sha256 of the current frame (skip republish when unchanged). */
private readonly hashes = new Map<string, string>();
/** sessionId → last issued sequence number. */
private readonly seqs = new Map<string, number>();
/** Consecutive capture failures across ALL sessions (daemon-level signal). */ /** Consecutive capture failures across ALL sessions (daemon-level signal). */
private globalFailures = 0; private globalFailures = 0;
private available = true; private available = true;
@@ -179,10 +167,7 @@ export class ObservationService {
this.cancelCapture(sessionId); this.cancelCapture(sessionId);
this.captureFailures.delete(sessionId); this.captureFailures.delete(sessionId);
this.lastActivity.delete(sessionId); this.lastActivity.delete(sessionId);
const entry = this.observations.get(sessionId); this.dropSessionFrames(sessionId);
if (entry?.thumbnailAttachmentId !== undefined) {
this.thumbRefs.delete(entry.thumbnailAttachmentId);
}
if (this.observations.delete(sessionId)) { if (this.observations.delete(sessionId)) {
this.emit({ this.emit({
type: "remove", type: "remove",
@@ -243,24 +228,17 @@ export class ObservationService {
} }
/** /**
* Read one captured thumbnail back through the store's verified path. * Read one captured thumbnail from the in-process ring. Powers the plugin's
* Powers the plugin's own HTTP thumbnail route — frames are plugin-owned * own HTTP thumbnail route — frames are plugin-owned runtime data, never
* runtime data, never referenced by any session log, so the * referenced by any session log, so the session-authorized client RPC
* session-authorized client RPC cannot serve them. * cannot serve them.
*/ */
async readThumbnail( async readThumbnail(
attachmentId: string, attachmentId: string,
): Promise<{ data: Uint8Array; mediaType: string } | undefined> { ): Promise<{ data: Uint8Array; mediaType: string } | undefined> {
const ref = this.thumbRefs.get(attachmentId); const frame = this.frames.get(attachmentId);
if (ref === undefined) return undefined; if (frame === undefined) return undefined;
const attachments = this.deps.ctx.get("attachments") as AttachmentLike | undefined; return { data: frame.data, mediaType: frame.mediaType };
if (attachments === undefined) return undefined;
try {
const stored = await attachments.readImage(ref);
return { data: stored.data, mediaType: ref.mediaType };
} catch {
return undefined;
}
} }
/** Tear down all state and timers (plugin dispose). */ /** Tear down all state and timers (plugin dispose). */
@@ -271,7 +249,10 @@ export class ObservationService {
this.captureInFlight.clear(); this.captureInFlight.clear();
this.captureFailures.clear(); this.captureFailures.clear();
this.lastActivity.clear(); this.lastActivity.clear();
this.thumbRefs.clear(); this.frames.clear();
this.rings.clear();
this.hashes.clear();
this.seqs.clear();
this.observations.clear(); this.observations.clear();
this.emit({ type: "reset" }); this.emit({ type: "reset" });
this.listeners.clear(); this.listeners.clear();
@@ -313,8 +294,8 @@ export class ObservationService {
/** /**
* Capture one frame: `bsk screenshot --json` through the runner, bytes into * Capture one frame: `bsk screenshot --json` through the runner, bytes into
* the attachment store, reference onto the observation. Runs OUTSIDE the * the in-process ring, id onto the observation. Runs OUTSIDE the tool
* tool instrumentation on purpose — no action events, no registry writes. * instrumentation on purpose — no action events, no registry writes.
* Failures keep the previous frame and back off silently. * Failures keep the previous frame and back off silently.
*/ */
private async capture(sessionId: string): Promise<void> { private async capture(sessionId: string): Promise<void> {
@@ -322,11 +303,6 @@ export class ObservationService {
const current = this.observations.get(sessionId); const current = this.observations.get(sessionId);
if (current === undefined || current.dead === true) return; if (current === undefined || current.dead === true) return;
if (this.captureInFlight.has(sessionId)) return; if (this.captureInFlight.has(sessionId)) return;
const attachments = this.deps.ctx.get("attachments") as AttachmentLike | undefined;
if (attachments === undefined) {
// No attachment store (headless composition): thumbnails stay absent.
return;
}
this.captureInFlight.add(sessionId); this.captureInFlight.add(sessionId);
// One fixed scratch path per session (overwritten each frame) and always // One fixed scratch path per session (overwritten each frame) and always
// unlinked after the bytes are read — /tmp must not grow without bound. // unlinked after the bytes are read — /tmp must not grow without bound.
@@ -355,23 +331,12 @@ export class ObservationService {
const reply = JSON.parse(result.stdout) as { path?: string }; const reply = JSON.parse(result.stdout) as { path?: string };
writtenPath = reply.path ?? outPath; writtenPath = reply.path ?? outPath;
const data = await readFile(writtenPath); const data = await readFile(writtenPath);
const ref = await attachments.saveImage({
data,
mediaType: "image/png",
name: `observation-${sessionId}.png`,
});
const entry = this.observations.get(sessionId); const entry = this.observations.get(sessionId);
if (entry === undefined) return; if (entry === undefined) return;
this.captureFailures.delete(sessionId); this.captureFailures.delete(sessionId);
this.globalFailures = 0; this.globalFailures = 0;
this.setAvailable(true); this.setAvailable(true);
// Drop the replaced frame's reference — the store keeps the bytes, but this.publishFrame(sessionId, entry, data);
// this map (and the old id's read path) must not grow without bound.
if (entry.thumbnailAttachmentId !== undefined) {
this.thumbRefs.delete(entry.thumbnailAttachmentId);
}
this.thumbRefs.set(String(ref.attachmentId), ref);
this.put({ ...entry, thumbnailAttachmentId: String(ref.attachmentId) });
} catch { } catch {
// Silent by design: keep the previous frame, count toward backoff. // Silent by design: keep the previous frame, count toward backoff.
this.captureFailures.set(sessionId, (this.captureFailures.get(sessionId) ?? 0) + 1); this.captureFailures.set(sessionId, (this.captureFailures.get(sessionId) ?? 0) + 1);
@@ -383,6 +348,33 @@ export class ObservationService {
if (!this.disposed && this.observations.has(sessionId)) this.scheduleCapture(sessionId); if (!this.disposed && this.observations.has(sessionId)) this.scheduleCapture(sessionId);
} }
} }
/** Insert a new frame, or no-op when the PNG bytes match the current one. */
private publishFrame(sessionId: string, entry: SessionObservation, data: Uint8Array): void {
const hash = createHash("sha256").update(data).digest("hex");
if (this.hashes.get(sessionId) === hash) return;
const seq = (this.seqs.get(sessionId) ?? 0) + 1;
this.seqs.set(sessionId, seq);
const id = `obs-${sessionId}-${seq}`;
this.frames.set(id, { data, mediaType: "image/png" });
this.hashes.set(sessionId, hash);
const ring = this.rings.get(sessionId) ?? [];
ring.push(id);
while (ring.length > FRAME_RING_SIZE) {
const evicted = ring.shift();
if (evicted !== undefined) this.frames.delete(evicted);
}
this.rings.set(sessionId, ring);
this.put({ ...entry, thumbnailAttachmentId: id });
}
private dropSessionFrames(sessionId: string): void {
for (const id of this.rings.get(sessionId) ?? []) this.frames.delete(id);
this.rings.delete(sessionId);
this.hashes.delete(sessionId);
this.seqs.delete(sessionId);
}
/** Mark a session dead: grey it out, stop asking for frames, keep the entry for the strip. */ /** Mark a session dead: grey it out, stop asking for frames, keep the entry for the strip. */
private markDead(sessionId: string): void { private markDead(sessionId: string): void {
const entry = this.observations.get(sessionId); const entry = this.observations.get(sessionId);
@@ -9,15 +9,23 @@ const OBS_IDLE: SessionObservation = { sessionId: "s1", action: "idle", since: 1
const OBS_BUSY: SessionObservation = { sessionId: "s1", action: "clicking", since: 2000 }; const OBS_BUSY: SessionObservation = { sessionId: "s1", action: "clicking", since: 2000 };
function harness(state: SessionObservation[] = []) { function harness(state: SessionObservation[] = []) {
const fetches: { url: string; init?: { method?: string; body?: string } }[] = []; const fetches: {
url: string;
init?: { method?: string; body?: string; headers?: Record<string, string> };
}[] = [];
const esInstances: EventSourceLike[] = []; const esInstances: EventSourceLike[] = [];
const fetchFn = vi.fn(async (url: string, init?: { method?: string; body?: string }) => { const fetchFn = vi.fn(
fetches.push({ url, init }); async (
if (url === "/bsk-observation/state") { url: string,
return { ok: true, json: async () => ({ sessions: state }) }; init?: { method?: string; body?: string; headers?: Record<string, string> },
} ) => {
return { ok: true, json: async () => ({ interrupted: true }) }; fetches.push({ url, init });
}); if (url === "/bsk-observation/state") {
return { ok: true, json: async () => ({ sessions: state }) };
}
return { ok: true, json: async () => ({ interrupted: true }) };
},
);
const eventSourceFactory = (url: string) => { const eventSourceFactory = (url: string) => {
const es: EventSourceLike = { onmessage: null, close: vi.fn() }; const es: EventSourceLike = { onmessage: null, close: vi.fn() };
esInstances.push(es); esInstances.push(es);
@@ -89,7 +97,9 @@ describe("ObservationClientStore", () => {
expect(await store.interrupt()).toBe(true); expect(await store.interrupt()).toBe(true);
expect(await store.interrupt("s1")).toBe(true); expect(await store.interrupt("s1")).toBe(true);
const calls = fetches.filter((f) => f.url === "/bsk-observation/interrupt"); const calls = fetches.filter((f) => f.url === "/bsk-observation/interrupt");
expect(calls[0].init?.headers?.["content-type"]).toBe("application/json");
expect(JSON.parse(calls[0].init?.body ?? "")).toEqual({}); expect(JSON.parse(calls[0].init?.body ?? "")).toEqual({});
expect(calls[1].init?.headers?.["content-type"]).toBe("application/json");
expect(JSON.parse(calls[1].init?.body ?? "")).toEqual({ sessionId: "s1" }); expect(JSON.parse(calls[1].init?.body ?? "")).toEqual({ sessionId: "s1" });
}); });
@@ -58,7 +58,12 @@ interface FakeRunner extends BskRunner {
/** Runner that answers screenshot with a real temp PNG and records kills. */ /** Runner that answers screenshot with a real temp PNG and records kills. */
function fakeRunner( function fakeRunner(
opts: { screenshotFails?: boolean; screenshotNotFound?: boolean; killCount?: number } = {}, opts: {
screenshotFails?: boolean;
screenshotNotFound?: boolean;
killCount?: number;
screenshotBytes?: () => Uint8Array;
} = {},
): FakeRunner { ): FakeRunner {
const calls: FakeRunner["calls"] = []; const calls: FakeRunner["calls"] = [];
const killed: string[] = []; const killed: string[] = [];
@@ -82,7 +87,7 @@ function fakeRunner(
} }
const outIndex = args.indexOf("--out") + 1; const outIndex = args.indexOf("--out") + 1;
const out = args[outIndex]; const out = args[outIndex];
writeFileSync(out, Buffer.from([0x89, 0x50, 0x4e, 0x47])); writeFileSync(out, opts.screenshotBytes?.() ?? Buffer.from([0x89, 0x50, 0x4e, 0x47]));
return { return {
code: 0, code: 0,
stdout: JSON.stringify({ stdout: JSON.stringify({
@@ -108,38 +113,20 @@ function fakeRunner(
}; };
} }
function fakeCtx(attachments?: unknown) { function fakeCtx() {
return { get: (key: string) => (key === "attachments" ? attachments : undefined) } as never; return { get: () => undefined } as never;
} }
function setup(opts: { function setup(opts: {
runner?: FakeRunner; runner?: FakeRunner;
attachments?: unknown;
registry?: SessionRegistry; registry?: SessionRegistry;
scheduler?: ReturnType<typeof fakeScheduler>; scheduler?: ReturnType<typeof fakeScheduler>;
}) { }) {
const registry = opts.registry ?? new SessionRegistry(5); const registry = opts.registry ?? new SessionRegistry(5);
const runner = opts.runner ?? fakeRunner(); const runner = opts.runner ?? fakeRunner();
const scheduler = opts.scheduler ?? fakeScheduler(); const scheduler = opts.scheduler ?? fakeScheduler();
const attachments =
"attachments" in opts
? opts.attachments
: {
saveImage: async (input: { data: Uint8Array; name?: string }) => ({
attachmentId: "att-1",
mediaType: "image/png",
bytes: input.data.byteLength,
width: 4,
height: 4,
name: input.name,
}),
readImage: async (ref: { attachmentId: unknown }) => ({
data: new Uint8Array([1, 2, 3]),
attachment: ref,
}),
};
const service = new ObservationService({ const service = new ObservationService({
ctx: fakeCtx(attachments), ctx: fakeCtx(),
runner, runner,
registry, registry,
queue: new KeyedExecutor(), queue: new KeyedExecutor(),
@@ -157,13 +144,8 @@ function own(registry: SessionRegistry, sessionId: string): void {
registry.completeStart({ sessionId, startedAtMs: 1 }); registry.completeStart({ sessionId, startedAtMs: 1 });
} }
async function flushMicrotasks(): Promise<void> {
// setImmediate yields to the poll phase so real fs reads inside capture() settle.
for (let i = 0; i < 20; i++) await new Promise((resolve) => setImmediate(resolve));
}
/** Poll until the condition holds (capture completion is real async I/O). */ /** Poll until the condition holds (capture completion is real async I/O). */
async function waitFor(cond: () => boolean, timeoutMs = 2000): Promise<void> { async function waitFor(cond: () => boolean, timeoutMs = 10_000): Promise<void> {
const start = Date.now(); const start = Date.now();
while (!cond()) { while (!cond()) {
if (Date.now() - start > timeoutMs) throw new Error("waitFor timeout"); if (Date.now() - start > timeoutMs) throw new Error("waitFor timeout");
@@ -218,7 +200,7 @@ describe("thumbnail cadence", () => {
expect(scheduler.pending()).toEqual([0]); expect(scheduler.pending()).toEqual([0]);
scheduler.runNext(); scheduler.runNext();
await waitFor(() => captures() === 1 && scheduler.pending().length === 1); await waitFor(() => captures() === 1 && scheduler.pending().length === 1);
expect(service.getState()[0].thumbnailAttachmentId).toBe("att-1"); expect(service.getState()[0].thumbnailAttachmentId).toBe("obs-s1-1");
// Just captured with fresh activity: next frame on the fast cadence. // Just captured with fresh activity: next frame on the fast cadence.
expect(scheduler.pending()).toEqual([1500]); expect(scheduler.pending()).toEqual([1500]);
scheduler.runNext(); scheduler.runNext();
@@ -259,14 +241,14 @@ describe("thumbnail cadence", () => {
expect(service.getState()[0].thumbnailAttachmentId).toBeUndefined(); expect(service.getState()[0].thumbnailAttachmentId).toBeUndefined();
}); });
it("stays silent without an attachment store (headless composition)", async () => { it("captures frames without an attachment store", async () => {
const scheduler = fakeScheduler(); const scheduler = fakeScheduler();
const runner = fakeRunner(); const runner = fakeRunner();
const { service } = setup({ runner, scheduler, attachments: undefined }); const { service } = setup({ runner, scheduler });
service.addSession("s1"); service.addSession("s1");
scheduler.runNext(); scheduler.runNext();
await flushMicrotasks(); await waitFor(() => service.getState()[0]?.thumbnailAttachmentId === "obs-s1-1");
expect(runner.calls.filter((c) => c.args[0] === "screenshot")).toHaveLength(0); expect((await service.readThumbnail("obs-s1-1"))?.mediaType).toBe("image/png");
}); });
}); });
@@ -286,7 +268,7 @@ describe("observation traffic isolation", () => {
const captureEvent = captureEvents[0]; const captureEvent = captureEvents[0];
if (captureEvent.type !== "upsert") throw new Error("unreachable"); if (captureEvent.type !== "upsert") throw new Error("unreachable");
expect(captureEvent.session?.action).toBe("idle"); expect(captureEvent.session?.action).toBe("idle");
expect(captureEvent.session?.thumbnailAttachmentId).toBe("att-1"); expect(captureEvent.session?.thumbnailAttachmentId).toBe("obs-s1-1");
// …and the registry's current pointer never moved through observation. // …and the registry's current pointer never moved through observation.
expect(registry.current()).toBe("s1"); expect(registry.current()).toBe("s1");
expect(registry.list()[0].startedAtMs).toBe(1); expect(registry.list()[0].startedAtMs).toBe(1);
@@ -543,22 +525,6 @@ describe("availability and dead sessions", () => {
}); });
}); });
describe("thumbnail read-back", () => {
it("serves captured frames through readThumbnail and 404s unknown ids", async () => {
const scheduler = fakeScheduler();
const runner = fakeRunner();
const { service } = setup({ runner, scheduler });
service.addSession("s1");
scheduler.runNext();
await waitFor(() => service.getState()[0]?.thumbnailAttachmentId !== undefined);
const frame = await service.readThumbnail("att-1");
expect(frame?.mediaType).toBe("image/png");
expect([...(frame?.data ?? [])]).toEqual([1, 2, 3]);
expect(await service.readThumbnail("nope")).toBeUndefined();
service.dispose();
});
});
describe("actionForLabel", () => { describe("actionForLabel", () => {
it("maps tool labels onto observation verbs", () => { it("maps tool labels onto observation verbs", () => {
expect(actionForLabel("navigate")).toBe("navigating"); expect(actionForLabel("navigate")).toBe("navigating");
@@ -598,47 +564,65 @@ describe("scratch files and thumbnail references", () => {
service.dispose(); service.dispose();
}); });
it("drops the replaced frame reference and clears it on remove", async () => { it("keeps the last two distinct frames and evicts the third", async () => {
let counter = 0; let n = 0;
const attachments = { const runner = fakeRunner({
saveImage: async (input: { data: Uint8Array; name?: string }) => { screenshotBytes: () => Uint8Array.from([0x89, 0x50, 0x4e, 0x47, n++]),
counter += 1; });
return {
attachmentId: `att-${counter}`,
mediaType: "image/png",
bytes: input.data.byteLength,
width: 4,
height: 4,
name: input.name,
};
},
readImage: async (ref: { attachmentId: unknown }) => ({
data: new Uint8Array([1, 2, 3]),
attachment: ref,
}),
};
const scheduler = fakeScheduler(); const scheduler = fakeScheduler();
const { service } = setup({ scheduler, attachments }); const { service } = setup({ scheduler, runner });
service.addSession("s1"); service.addSession("s1");
scheduler.runNext(); scheduler.runNext();
await waitFor( await waitFor(
() => () =>
service.getState()[0]?.thumbnailAttachmentId === "att-1" && service.getState()[0]?.thumbnailAttachmentId === "obs-s1-1" &&
scheduler.pending().length === 1, scheduler.pending().length === 1,
); );
service.endAction("s1"); service.endAction("s1");
scheduler.runNext(); scheduler.runNext();
await waitFor( await waitFor(
() => () =>
service.getState()[0]?.thumbnailAttachmentId === "att-2" && service.getState()[0]?.thumbnailAttachmentId === "obs-s1-2" &&
scheduler.pending().length === 1, scheduler.pending().length === 1,
); );
// the replaced frame's read path is gone expect((await service.readThumbnail("obs-s1-1"))?.mediaType).toBe("image/png");
expect(await service.readThumbnail("att-1")).toBeUndefined(); expect((await service.readThumbnail("obs-s1-2"))?.mediaType).toBe("image/png");
expect((await service.readThumbnail("att-2"))?.mediaType).toBe("image/png"); service.endAction("s1");
// removing the session clears its live reference too scheduler.runNext();
await waitFor(
() =>
service.getState()[0]?.thumbnailAttachmentId === "obs-s1-3" &&
scheduler.pending().length === 1,
);
expect(await service.readThumbnail("obs-s1-1")).toBeUndefined();
expect((await service.readThumbnail("obs-s1-2"))?.mediaType).toBe("image/png");
expect((await service.readThumbnail("obs-s1-3"))?.mediaType).toBe("image/png");
service.removeSession("s1"); service.removeSession("s1");
expect(await service.readThumbnail("att-2")).toBeUndefined(); expect(await service.readThumbnail("obs-s1-2")).toBeUndefined();
expect(await service.readThumbnail("obs-s1-3")).toBeUndefined();
service.dispose();
});
it("reuses the current frame id when the PNG bytes are unchanged", async () => {
const scheduler = fakeScheduler();
const runner = fakeRunner();
const { service } = setup({ scheduler, runner });
service.addSession("s1");
scheduler.runNext();
await waitFor(
() =>
service.getState()[0]?.thumbnailAttachmentId === "obs-s1-1" &&
scheduler.pending().length === 1,
);
service.endAction("s1");
scheduler.runNext();
await waitFor(
() =>
runner.calls.filter((c) => c.args[0] === "screenshot").length >= 2 &&
scheduler.pending().length === 1,
);
expect(service.getState()[0]?.thumbnailAttachmentId).toBe("obs-s1-1");
expect((await service.readThumbnail("obs-s1-1"))?.mediaType).toBe("image/png");
service.dispose(); service.dispose();
}); });
}); });
@@ -733,6 +733,10 @@ describe("observation action instrumentation timing", () => {
const navigateGate = new Promise<void>((resolve) => { const navigateGate = new Promise<void>((resolve) => {
releaseNavigate = resolve; releaseNavigate = resolve;
}); });
let releaseSnapshot!: () => void;
const snapshotGate = new Promise<void>((resolve) => {
releaseSnapshot = resolve;
});
const runner: BskRunner = { const runner: BskRunner = {
async run(args: string[]): Promise<BskRunResult> { async run(args: string[]): Promise<BskRunResult> {
if (args[0] === "session") { if (args[0] === "session") {
@@ -754,6 +758,21 @@ describe("observation action instrumentation timing", () => {
aborted: false, aborted: false,
}; };
} }
if (args[0] === "snapshot") {
await snapshotGate;
return {
code: 0,
stdout: JSON.stringify({ text: "ok", ref_count: 0, tab_id: 7, truncated: false }),
stderr: "",
timedOut: false,
aborted: false,
};
}
// Observation captures share the session queue; fail them fast so they
// cannot stall the action-label assertions.
if (args[0] === "screenshot") {
return { code: 1, stdout: "", stderr: "skip", timedOut: false, aborted: false };
}
return { return {
code: 0, code: 0,
stdout: JSON.stringify({ text: "ok", ref_count: 0, tab_id: 7, truncated: false }), stdout: JSON.stringify({ text: "ok", ref_count: 0, tab_id: 7, truncated: false }),
@@ -792,6 +811,7 @@ describe("observation action instrumentation timing", () => {
await vi.waitFor(() => await vi.waitFor(() =>
expect(observation.getState().find((s) => s.sessionId === "s1")?.action).toBe("snapshotting"), expect(observation.getState().find((s) => s.sessionId === "s1")?.action).toBe("snapshotting"),
); );
releaseSnapshot();
await b; await b;
await vi.waitFor(() => await vi.waitFor(() =>
expect(observation.getState().find((s) => s.sessionId === "s1")?.action).toBe("idle"), expect(observation.getState().find((s) => s.sessionId === "s1")?.action).toBe("idle"),
@@ -2,6 +2,7 @@ import { readFile } from "node:fs/promises";
import { basename, dirname, resolve as resolvePath } from "node:path"; import { basename, dirname, resolve as resolvePath } from "node:path";
import { transform } from "lightningcss"; import { transform } from "lightningcss";
import { defineConfig, type UserConfig } from "tsdown"; import { defineConfig, type UserConfig } from "tsdown";
import pkg from "./package.json" with { type: "json" };
/** /**
* Two build faces of the dual-face package: * Two build faces of the dual-face package:
@@ -33,7 +34,10 @@ const VENDORED_LIBRARY = /^@deepseek-ai\/(cosmokit|schemastery)(\/|$)/;
const CSS_VIRTUAL_PREFIX = "\0bsk-css:"; const CSS_VIRTUAL_PREFIX = "\0bsk-css:";
const CSS_VIRTUAL_SUFFIX = ".mjs"; const CSS_VIRTUAL_SUFFIX = ".mjs";
const CLIENT_ID = "dsh-plugin-browserskill"; // The web shell resolves client bundles from its module table by package name
// (dsh's own bundles register the same way), so this must never drift from
// package.json.
const CLIENT_ID = pkg.name;
const client: UserConfig = { const client: UserConfig = {
name: `${CLIENT_ID}/client`, name: `${CLIENT_ID}/client`,
+196 -1
View File
@@ -25,6 +25,85 @@ function scene(nodes: VomNode[]): VomScene {
} }
describe("renderVom single-layer page", () => { describe("renderVom single-layer page", () => {
it("does not derive handle context across frame scopes", () => {
const out = renderVom(
scene([
node({
id: 1,
role: "RootWebArea",
frameId: "main",
contextScopeId: "main",
}),
node({
id: 2,
parentId: 1,
role: "StaticText",
name: "创建于",
frameId: "main",
contextScopeId: "main",
}),
node({
id: 3,
parentId: 1,
role: "Iframe",
frameId: "main",
contextScopeId: "main",
}),
node({
id: 4,
parentId: 3,
role: "button",
name: "表格视图",
frameId: "child",
contextScopeId: "child",
}),
node({
id: 5,
parentId: 1,
role: "button",
name: "表格视图",
frameId: "main",
contextScopeId: "main",
}),
]),
);
expect(out.text).toContain('@e1 button "表格视图"');
expect(out.text).not.toContain('@e1 button "表格视图" [ctx: 创建于]');
});
it("preserves handle context within a frame scope", () => {
const out = renderVom(
scene([
node({ id: 1, role: "RootWebArea", contextScopeId: "main" }),
node({ id: 2, parentId: 1, role: "Iframe", contextScopeId: "main" }),
node({
id: 3,
parentId: 2,
role: "StaticText",
name: "工具栏",
contextScopeId: "child",
}),
node({
id: 4,
parentId: 2,
role: "button",
name: "更多",
contextScopeId: "child",
}),
node({
id: 5,
parentId: 1,
role: "button",
name: "更多",
contextScopeId: "main",
}),
]),
);
expect(out.text).toContain('@e1 button "更多" [ctx: 工具栏]');
});
it("always emits a complete @vom single-layer document when there is no blocker", () => { it("always emits a complete @vom single-layer document when there is no blocker", () => {
const out = renderVom( const out = renderVom(
scene([ scene([
@@ -157,7 +236,7 @@ describe("renderVom single-layer page", () => {
expect(out.text).not.toContain("hunter2"); expect(out.text).not.toContain("hunter2");
}); });
it("does not imply a sensitive textbox has a value when the value is missing", () => { it("renders a sensitive filled state without retaining its clear text value", () => {
const out = renderVom( const out = renderVom(
scene([ scene([
node({ id: 1, role: "RootWebArea", name: "Login" }), node({ id: 1, role: "RootWebArea", name: "Login" }),
@@ -168,6 +247,26 @@ describe("renderVom single-layer page", () => {
name: "密码", name: "密码",
tag: "input", tag: "input",
sensitive: true, sensitive: true,
inputState: "filled",
}),
]),
);
expect(out.text).toContain('@e1 textbox "密码" [filled] ="•••"');
});
it("does not imply an empty sensitive textbox has a value", () => {
const out = renderVom(
scene([
node({ id: 1, role: "RootWebArea", name: "Login" }),
node({
id: 2,
parentId: 1,
role: "textbox",
name: "密码",
tag: "input",
sensitive: true,
inputState: "empty",
}), }),
]), ]),
); );
@@ -389,6 +488,102 @@ describe("renderVom single-layer page", () => {
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 4 }]); expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 4 }]);
}); });
it("keeps paint-order comparisons inside their frame document", () => {
const out = renderVom(
{
...scene([
node({ id: 1, role: "RootWebArea", frameId: "main" }),
node({
id: 2,
parentId: 1,
role: "button",
name: "Top-level action",
tag: "button",
rect: { x: 120, y: 120, w: 120, h: 40 },
paintOrder: 2,
frameId: "main",
}),
node({
id: 3,
parentId: 1,
role: "Iframe",
tag: "iframe",
rect: { x: 0, y: 0, w: 1000, h: 700 },
paintOrder: 1,
frameId: "main",
}),
node({
id: 4,
parentId: 3,
role: "generic",
rect: { x: 0, y: 0, w: 1000, h: 700 },
paintOrder: 100,
position: "fixed",
frameId: "child",
}),
node({
id: 5,
parentId: 4,
role: "button",
name: "Child action",
tag: "button",
rect: { x: 120, y: 120, w: 120, h: 40 },
paintOrder: 101,
frameId: "child",
}),
]),
rootFrameId: "main",
},
{ activeRegionPolicy: true },
);
expect(out.text).toContain("@layers 1 focus=L1");
expect(out.text).toContain('button "Top-level action"');
expect(out.text).toContain('button "Child action"');
});
it("uses the iframe owner when a parent-frame region covers child content", () => {
const out = renderVom(
{
...scene([
node({ id: 1, role: "RootWebArea", frameId: "main" }),
node({
id: 2,
parentId: 1,
role: "Iframe",
tag: "iframe",
rect: { x: 100, y: 100, w: 400, h: 300 },
paintOrder: 1,
frameId: "main",
}),
node({
id: 3,
parentId: 2,
role: "button",
name: "Covered child action",
tag: "button",
rect: { x: 150, y: 150, w: 120, h: 40 },
paintOrder: 100,
frameId: "child",
}),
node({
id: 4,
parentId: 1,
role: "generic",
rect: { x: 120, y: 120, w: 200, h: 120 },
paintOrder: 2,
position: "fixed",
frameId: "main",
}),
]),
rootFrameId: "main",
},
{ activeRegionPolicy: true },
);
expect(out.text).not.toContain("Covered child action");
});
it("renders conditional surface items inline on the trigger", () => { it("renders conditional surface items inline on the trigger", () => {
const out = renderVom({ const out = renderVom({
...scene([ ...scene([
+1
View File
@@ -8,6 +8,7 @@ export type {
Viewport, Viewport,
VomNode, VomNode,
VomOptions, VomOptions,
VomRef,
VomResult, VomResult,
VomScene, VomScene,
} from "./types"; } from "./types";
+12 -2
View File
@@ -50,10 +50,18 @@ function classifyLayer(
return blockerCoverage >= MASK_COVERAGE_THRESHOLD ? "mask" : "modal"; return blockerCoverage >= MASK_COVERAGE_THRESHOLD ? "mask" : "modal";
} }
export function detectBlockingLayer(nodes: VomNode[], vp: Viewport): BlockingLayer | null { export function detectBlockingLayer(
nodes: VomNode[],
vp: Viewport,
rootFrameId?: string,
): BlockingLayer | null {
let blocker: { node: VomNode; coverage: number } | null = null; let blocker: { node: VomNode; coverage: number } | null = null;
for (const node of nodes) { for (const node of nodes) {
// A document's paint order is local to its iframe stacking context. Only
// the root document can establish a page-level blocking layer; child
// documents are constrained by their iframe owner in the parent document.
if (rootFrameId !== undefined && node.frameId !== rootFrameId) continue;
if (!isBlockingCandidate(node)) continue; if (!isBlockingCandidate(node)) continue;
const cov = coverage(node.rect, vp); const cov = coverage(node.rect, vp);
const qualifies = cov >= BLOCK_COVERAGE_THRESHOLD || (hasModalFeature(node) && cov >= 0.15); const qualifies = cov >= BLOCK_COVERAGE_THRESHOLD || (hasModalFeature(node) && cov >= 0.15);
@@ -72,7 +80,9 @@ export function detectBlockingLayer(nodes: VomNode[], vp: Viewport): BlockingLay
const threshold = blocker.node.paintOrder; const threshold = blocker.node.paintOrder;
const members = new Set<number>(); const members = new Set<number>();
for (const node of nodes) { for (const node of nodes) {
if (node.paintOrder >= threshold) members.add(node.id); if (node.frameId === blocker.node.frameId && node.paintOrder >= threshold) {
members.add(node.id);
}
} }
return { return {
+73 -11
View File
@@ -6,6 +6,7 @@ import type {
Rect, Rect,
VomNode, VomNode,
VomOptions, VomOptions,
VomRef,
VomResult, VomResult,
VomScene, VomScene,
} from "./types"; } from "./types";
@@ -503,12 +504,18 @@ function isContextBoundary(node: VomNode): boolean {
); );
} }
function sharesContextScope(node: VomNode, candidate: VomNode): boolean {
return node.contextScopeId === candidate.contextScopeId;
}
function collectWeakLabelsFromSubtree( function collectWeakLabelsFromSubtree(
node: VomNode, node: VomNode,
target: VomNode,
nodeName: string, nodeName: string,
state: RenderState, state: RenderState,
labels: string[], labels: string[],
): void { ): void {
if (!sharesContextScope(target, node)) return;
if (isVomReferenceNode(node)) return; if (isVomReferenceNode(node)) return;
const label = weakLabelContext(node, nodeName); const label = weakLabelContext(node, nodeName);
if (label) { if (label) {
@@ -516,7 +523,7 @@ function collectWeakLabelsFromSubtree(
return; return;
} }
for (const child of state.children.get(node.id) ?? []) { for (const child of state.children.get(node.id) ?? []) {
collectWeakLabelsFromSubtree(child, nodeName, state, labels); collectWeakLabelsFromSubtree(child, target, nodeName, state, labels);
if (labels.length >= MAX_HANDLE_CONTEXT_ITEMS) return; if (labels.length >= MAX_HANDLE_CONTEXT_ITEMS) return;
} }
} }
@@ -532,6 +539,7 @@ function collectOwnedContext(
while (current !== null && guard <= state.parentMap.size) { while (current !== null && guard <= state.parentMap.size) {
const parent = state.nodesById.get(current); const parent = state.nodesById.get(current);
if (!parent) break; if (!parent) break;
if (!sharesContextScope(node, parent)) break;
const label = contextLabel(parent, nodeName); const label = contextLabel(parent, nodeName);
if (label) contexts.push({ text: label, source: "owned", order: contexts.length }); if (label) contexts.push({ text: label, source: "owned", order: contexts.length });
current = state.parentMap.get(current) ?? null; current = state.parentMap.get(current) ?? null;
@@ -553,12 +561,12 @@ function collectSameContainerContext(
const siblings = state.children.get(parentId) ?? []; const siblings = state.children.get(parentId) ?? [];
for (const sibling of siblings) { for (const sibling of siblings) {
if (sibling.id === childId) break; if (sibling.id === childId) break;
collectWeakLabelsFromSubtree(sibling, nodeName, state, labels); collectWeakLabelsFromSubtree(sibling, node, nodeName, state, labels);
while (labels.length > MAX_HANDLE_CONTEXT_ITEMS) labels.shift(); while (labels.length > MAX_HANDLE_CONTEXT_ITEMS) labels.shift();
} }
const parent = state.nodesById.get(parentId); const parent = state.nodesById.get(parentId);
if (!parent || isContextBoundary(parent)) break; if (!parent || !sharesContextScope(node, parent) || isContextBoundary(parent)) break;
childId = parentId; childId = parentId;
parentId = state.parentMap.get(parentId) ?? null; parentId = state.parentMap.get(parentId) ?? null;
guard += 1; guard += 1;
@@ -578,6 +586,7 @@ function collectDomContext(
for (const candidate of state.nodesById.values()) { for (const candidate of state.nodesById.values()) {
if (candidate.id === node.id) continue; if (candidate.id === node.id) continue;
if (!sharesContextScope(node, candidate)) continue;
if (isContextBoundary(candidate) && !ancestorIds.includes(candidate.id)) continue; if (isContextBoundary(candidate) && !ancestorIds.includes(candidate.id)) continue;
const parentId = candidate.domParentId; const parentId = candidate.domParentId;
if (parentId === undefined || parentId === null || !ancestors.has(parentId)) continue; if (parentId === undefined || parentId === null || !ancestors.has(parentId)) continue;
@@ -627,8 +636,10 @@ function renderNodeLine(
} }
const placeholder = cleaned(node.placeholder); const placeholder = cleaned(node.placeholder);
if (placeholder) line += ` placeholder=${JSON.stringify(placeholder)}`; if (placeholder) line += ` placeholder=${JSON.stringify(placeholder)}`;
const sensitiveHasValue =
rawValue !== undefined || node.inputState === "filled" || node.inputState === "default";
const value = node.sensitive const value = node.sensitive
? rawValue !== undefined ? sensitiveHasValue
? SENSITIVE_MASK ? SENSITIVE_MASK
: undefined : undefined
: rawValue?.slice(0, MAX_VALUE_LENGTH); : rawValue?.slice(0, MAX_VALUE_LENGTH);
@@ -684,7 +695,7 @@ function shouldSkipRedundantChildren(node: VomNode, state: RenderState): boolean
interface RenderState { interface RenderState {
lines: string[]; lines: string[];
refs: Array<{ ref: string; backendNodeId: number }>; refs: VomRef[];
nextRef: number; nextRef: number;
tokens: number; tokens: number;
maxDepth: number; maxDepth: number;
@@ -764,7 +775,11 @@ function renderTree(
state.lines.push(line); state.lines.push(line);
state.tokens = nextTokens; state.tokens = nextTokens;
if (ref) { if (ref) {
state.refs.push({ ref, backendNodeId: node.id }); state.refs.push({
ref,
backendNodeId: node.backendNodeId ?? node.id,
...(node.frameId ? { frameId: node.frameId } : {}),
});
state.nextRef += 1; state.nextRef += 1;
} }
@@ -855,29 +870,75 @@ function activeRegionCandidatePriority(node: VomNode, viewportCoverage: number):
return 1; return 1;
} }
function paintLineageByFrame(
node: VomNode,
parentMap: Map<number, number | null>,
nodesById: Map<number, VomNode>,
): Map<string | undefined, VomNode> {
const lineage = new Map<string | undefined, VomNode>();
let current: VomNode | undefined = node;
let guard = 0;
while (current && guard <= parentMap.size) {
if (!lineage.has(current.frameId)) lineage.set(current.frameId, current);
const parentId: number | null = parentMap.get(current.id) ?? null;
current = parentId === null ? undefined : nodesById.get(parentId);
guard += 1;
}
return lineage;
}
/** Resolve both nodes to the nearest document where their paint order is comparable. */
function comparablePaintNodes(
target: VomNode,
blocker: VomNode,
parentMap: Map<number, number | null>,
nodesById: Map<number, VomNode>,
): { target: VomNode; blocker: VomNode } | null {
if (target.frameId === blocker.frameId) return { target, blocker };
const targetLineage = paintLineageByFrame(target, parentMap, nodesById);
let blockerInFrame: VomNode | undefined = blocker;
let guard = 0;
while (blockerInFrame && guard <= parentMap.size) {
const targetInFrame = targetLineage.get(blockerInFrame.frameId);
if (targetInFrame) return { target: targetInFrame, blocker: blockerInFrame };
const parentId: number | null = parentMap.get(blockerInFrame.id) ?? null;
blockerInFrame = parentId === null ? undefined : nodesById.get(parentId);
guard += 1;
}
return null;
}
function isBlockedByRegion( function isBlockedByRegion(
target: VomNode, target: VomNode,
blocker: VomNode, blocker: VomNode,
parentMap: Map<number, number | null>, parentMap: Map<number, number | null>,
nodesById: Map<number, VomNode>,
viewportCoverage: number, viewportCoverage: number,
): boolean { ): boolean {
if (target.id === blocker.id) return false; if (target.id === blocker.id) return false;
if (isAncestorOf(parentMap, blocker.id, target.id)) return false; if (isAncestorOf(parentMap, blocker.id, target.id)) return false;
if (isAncestorOf(parentMap, target.id, blocker.id)) return false; if (isAncestorOf(parentMap, target.id, blocker.id)) return false;
if (!target.rect || !blocker.rect) return false; if (!target.rect || !blocker.rect) return false;
const paintNodes = comparablePaintNodes(target, blocker, parentMap, nodesById);
if (!paintNodes) return false;
const points = interactionPoints(target.rect); const points = interactionPoints(target.rect);
if ( if (
viewportCoverage < 0.9 && viewportCoverage < 0.9 &&
target.paintOrder >= blocker.paintOrder && paintNodes.target.paintOrder >= paintNodes.blocker.paintOrder &&
points.some(([x, y]) => rectContains(blocker.rect as Rect, x, y)) points.some(([x, y]) => rectContains(blocker.rect as Rect, x, y))
) { ) {
return false; return false;
} }
if (blocker.paintOrder < target.paintOrder) return false; if (paintNodes.blocker.paintOrder < paintNodes.target.paintOrder) return false;
if (blocker.paintOrder === target.paintOrder && blocker.id <= target.id) return false; if (
paintNodes.blocker.paintOrder === paintNodes.target.paintOrder &&
paintNodes.blocker.id <= paintNodes.target.id
) {
return false;
}
if (viewportCoverage >= 0.9) return true; if (viewportCoverage >= 0.9) return true;
return points.some(([x, y]) => rectContains(blocker.rect as Rect, x, y)); return points.some(([x, y]) => rectContains(blocker.rect as Rect, x, y));
@@ -901,6 +962,7 @@ function collectDescendantsOfIds(nodes: VomNode[], roots: Set<number>): Set<numb
function applyActiveRegionPolicy(nodes: VomNode[], scene: VomScene): VomNode[] { function applyActiveRegionPolicy(nodes: VomNode[], scene: VomScene): VomNode[] {
const parentMap = buildParentMap(nodes); const parentMap = buildParentMap(nodes);
const nodesById = new Map(nodes.map((node) => [node.id, node]));
const candidates = nodes const candidates = nodes
.filter(isPositionedRegionCandidate) .filter(isPositionedRegionCandidate)
.map((node) => { .map((node) => {
@@ -917,7 +979,7 @@ function applyActiveRegionPolicy(nodes: VomNode[], scene: VomScene): VomNode[] {
for (const target of nodes) { for (const target of nodes) {
if (!isVomReferenceNode(target)) continue; if (!isVomReferenceNode(target)) continue;
const blocked = candidates.some((candidate) => const blocked = candidates.some((candidate) =>
isBlockedByRegion(target, candidate.node, parentMap, candidate.viewportCoverage), isBlockedByRegion(target, candidate.node, parentMap, nodesById, candidate.viewportCoverage),
); );
if (blocked) blockedRoots.add(target.id); if (blocked) blockedRoots.add(target.id);
} }
@@ -972,7 +1034,7 @@ function renderDoubleLayer(scene: VomScene, layer: BlockingLayer, options: VomOp
export function renderVom(scene: VomScene, options: VomOptions = {}): VomResult { export function renderVom(scene: VomScene, options: VomOptions = {}): VomResult {
const nodes = applyVomInteractionRecovery(scene.nodes); const nodes = applyVomInteractionRecovery(scene.nodes);
const renderScene = { ...scene, nodes }; const renderScene = { ...scene, nodes };
const layer = detectBlockingLayer(nodes, scene.viewport); const layer = detectBlockingLayer(nodes, scene.viewport, scene.rootFrameId);
if (layer) return renderDoubleLayer(renderScene, layer, options); if (layer) return renderDoubleLayer(renderScene, layer, options);
const visibleNodes = options.activeRegionPolicy ? applyActiveRegionPolicy(nodes, scene) : nodes; const visibleNodes = options.activeRegionPolicy ? applyActiveRegionPolicy(nodes, scene) : nodes;
+12 -1
View File
@@ -21,6 +21,9 @@ export type LayerKind = "page" | "modal" | "mask";
export interface VomNode { export interface VomNode {
id: number; id: number;
parentId: number | null; parentId: number | null;
backendNodeId?: number;
frameId?: string;
contextScopeId?: string;
role?: string; role?: string;
name?: string; name?: string;
@@ -52,6 +55,8 @@ export interface VomNode {
export interface VomScene { export interface VomScene {
viewport: Viewport; viewport: Viewport;
nodes: VomNode[]; nodes: VomNode[];
/** Root document whose paint order defines page-level blocking layers. */
rootFrameId?: string;
surfaces?: CondSurface[]; surfaces?: CondSurface[];
activeScopeBlocks?: ActiveScopeBlock[]; activeScopeBlocks?: ActiveScopeBlock[];
} }
@@ -79,9 +84,15 @@ export interface ActiveScopeBlock {
lines: string[]; lines: string[];
} }
export interface VomRef {
ref: string;
backendNodeId: number;
frameId?: string;
}
export interface VomResult { export interface VomResult {
text: string; text: string;
refs: Array<{ ref: string; backendNodeId: number }>; refs: VomRef[];
truncated: boolean; truncated: boolean;
} }